diff options
Diffstat (limited to 'drivers/mmc/host')
| -rw-r--r-- | drivers/mmc/host/Kconfig | 42 | ||||
| -rw-r--r-- | drivers/mmc/host/Makefile | 4 | ||||
| -rw-r--r-- | drivers/mmc/host/cmdq_hci.c | 1362 | ||||
| -rw-r--r-- | drivers/mmc/host/cmdq_hci.h | 251 | ||||
| -rw-r--r-- | drivers/mmc/host/sdhci-msm-ice.c | 565 | ||||
| -rw-r--r-- | drivers/mmc/host/sdhci-msm-ice.h | 174 | ||||
| -rw-r--r-- | drivers/mmc/host/sdhci-msm.c | 4883 | ||||
| -rw-r--r-- | drivers/mmc/host/sdhci-msm.h | 245 | ||||
| -rw-r--r-- | drivers/mmc/host/sdhci.c | 1215 | ||||
| -rw-r--r-- | drivers/mmc/host/sdhci.h | 152 |
10 files changed, 8549 insertions, 344 deletions
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 2e6d2fff1096..01959bd2d523 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -406,18 +406,39 @@ config MMC_ATMELMCI If unsure, say N. config MMC_SDHCI_MSM - tristate "Qualcomm SDHCI Controller Support" - depends on ARCH_QCOM || (ARM && COMPILE_TEST) + tristate "Qualcomm Technologies, Inc. SDHCI Controller Support" + depends on ARCH_QCOM || ARCH_MSM || (ARM && COMPILE_TEST) depends on MMC_SDHCI_PLTFM + select PM_DEVFREQ + select DEVFREQ_GOV_SIMPLE_ONDEMAND help This selects the Secure Digital Host Controller Interface (SDHCI) - support present in Qualcomm SOCs. The controller supports - SD/MMC/SDIO devices. + support present in Qualcomm Technologies, Inc. SOCs. The controller + supports SD/MMC/SDIO devices. If you have a controller with this interface, say Y or M here. If unsure, say N. +config MMC_SDHCI_MSM_ICE + bool "Qualcomm Technologies, Inc Inline Crypto Engine for SDHCI core" + depends on MMC_SDHCI_MSM && CRYPTO_DEV_QCOM_ICE + help + This selects the QTI specific additions to support Inline Crypto + Engine (ICE). ICE accelerates the crypto operations and maintains + the high SDHCI performance. + + Select this if you have ICE supported for SDHCI on QTI chipset. + If unsure, say N. + +config MMC_MSM + tristate "Qualcomm SDCC Controller Support" + depends on MMC && (ARCH_MSM7X00A || ARCH_MSM7X30 || ARCH_QSD8X50) + help + This provides support for the SD/MMC cell found in the + MSM and QSD SOCs from Qualcomm. The controller also has + support for SDIO devices. + config MMC_MXC tristate "Freescale i.MX21/27/31 or MPC512x Multimedia Card support" depends on ARCH_MXC || PPC_MPC512x @@ -773,6 +794,19 @@ config MMC_SUNXI This selects support for the SD/MMC Host Controller on Allwinner sunxi SoCs. +config MMC_CQ_HCI + tristate "Command Queue Support" + depends on HAS_DMA + help + This selects the Command Queue Host Controller Interface (CQHCI) + support present in host controllers of Qualcomm Technologies, Inc + amongst others. + This controller supports eMMC devices with command queue support. + + If you have a controller with this interface, say Y or M here. + + If unsure, say N. + config MMC_TOSHIBA_PCI tristate "Toshiba Type A SD/MMC Card Interface Driver" depends on PCI diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile index 3595f83e89dd..b9cbe592f5e3 100644 --- a/drivers/mmc/host/Makefile +++ b/drivers/mmc/host/Makefile @@ -72,9 +72,11 @@ obj-$(CONFIG_MMC_SDHCI_OF_ESDHC) += sdhci-of-esdhc.o obj-$(CONFIG_MMC_SDHCI_OF_HLWD) += sdhci-of-hlwd.o obj-$(CONFIG_MMC_SDHCI_BCM_KONA) += sdhci-bcm-kona.o obj-$(CONFIG_MMC_SDHCI_BCM2835) += sdhci-bcm2835.o -obj-$(CONFIG_MMC_SDHCI_IPROC) += sdhci-iproc.o obj-$(CONFIG_MMC_SDHCI_MSM) += sdhci-msm.o +obj-$(CONFIG_MMC_SDHCI_MSM_ICE) += sdhci-msm-ice.o +obj-$(CONFIG_MMC_SDHCI_IPROC) += sdhci-iproc.o obj-$(CONFIG_MMC_SDHCI_ST) += sdhci-st.o +obj-$(CONFIG_MMC_CQ_HCI) += cmdq_hci.o ifeq ($(CONFIG_CB710_DEBUG),y) CFLAGS-cb710-mmc += -DDEBUG diff --git a/drivers/mmc/host/cmdq_hci.c b/drivers/mmc/host/cmdq_hci.c new file mode 100644 index 000000000000..3f741f83a436 --- /dev/null +++ b/drivers/mmc/host/cmdq_hci.c @@ -0,0 +1,1362 @@ +/* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/delay.h> +#include <linux/highmem.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/slab.h> +#include <linux/scatterlist.h> +#include <linux/platform_device.h> +#include <linux/blkdev.h> + +#include <linux/mmc/mmc.h> +#include <linux/mmc/host.h> +#include <linux/mmc/card.h> +#include <linux/pm_runtime.h> +#include <linux/workqueue.h> + +#include "cmdq_hci.h" +#include "sdhci.h" +#include "sdhci-msm.h" + +#define DCMD_SLOT 31 +#define NUM_SLOTS 32 + +/* 10 sec */ +#define HALT_TIMEOUT_MS 10000 + +static int cmdq_halt_poll(struct mmc_host *mmc, bool halt); +static int cmdq_halt(struct mmc_host *mmc, bool halt); + +#ifdef CONFIG_PM_RUNTIME +static int cmdq_runtime_pm_get(struct cmdq_host *host) +{ + return pm_runtime_get_sync(host->mmc->parent); +} +static int cmdq_runtime_pm_put(struct cmdq_host *host) +{ + pm_runtime_mark_last_busy(host->mmc->parent); + return pm_runtime_put_autosuspend(host->mmc->parent); +} +#else +static inline int cmdq_runtime_pm_get(struct cmdq_host *host) +{ + return 0; +} +static inline int cmdq_runtime_pm_put(struct cmdq_host *host) +{ + return 0; +} +#endif +static inline struct mmc_request *get_req_by_tag(struct cmdq_host *cq_host, + unsigned int tag) +{ + return cq_host->mrq_slot[tag]; +} + +static inline u8 *get_desc(struct cmdq_host *cq_host, u8 tag) +{ + return cq_host->desc_base + (tag * cq_host->slot_sz); +} + +static inline u8 *get_link_desc(struct cmdq_host *cq_host, u8 tag) +{ + u8 *desc = get_desc(cq_host, tag); + + return desc + cq_host->task_desc_len; +} + +static inline dma_addr_t get_trans_desc_dma(struct cmdq_host *cq_host, u8 tag) +{ + return cq_host->trans_desc_dma_base + + (cq_host->mmc->max_segs * tag * + cq_host->trans_desc_len); +} + +static inline u8 *get_trans_desc(struct cmdq_host *cq_host, u8 tag) +{ + return cq_host->trans_desc_base + + (cq_host->trans_desc_len * cq_host->mmc->max_segs * tag); +} + +static void setup_trans_desc(struct cmdq_host *cq_host, u8 tag) +{ + u8 *link_temp; + dma_addr_t trans_temp; + + link_temp = get_link_desc(cq_host, tag); + trans_temp = get_trans_desc_dma(cq_host, tag); + + memset(link_temp, 0, cq_host->link_desc_len); + if (cq_host->link_desc_len > 8) + *(link_temp + 8) = 0; + + if (tag == DCMD_SLOT) { + *link_temp = VALID(0) | ACT(0) | END(1); + return; + } + + *link_temp = VALID(1) | ACT(0x6) | END(0); + + if (cq_host->dma64) { + __le64 *data_addr = (__le64 __force *)(link_temp + 4); + data_addr[0] = cpu_to_le64(trans_temp); + } else { + __le32 *data_addr = (__le32 __force *)(link_temp + 4); + data_addr[0] = cpu_to_le32(trans_temp); + } +} + +static void cmdq_set_halt_irq(struct cmdq_host *cq_host, bool enable) +{ + u32 ier; + + ier = cmdq_readl(cq_host, CQISTE); + if (enable) { + cmdq_writel(cq_host, ier | HALT, CQISTE); + cmdq_writel(cq_host, ier | HALT, CQISGE); + } else { + cmdq_writel(cq_host, ier & ~HALT, CQISTE); + cmdq_writel(cq_host, ier & ~HALT, CQISGE); + } +} + +static void cmdq_clear_set_irqs(struct cmdq_host *cq_host, u32 clear, u32 set) +{ + u32 ier; + + ier = cmdq_readl(cq_host, CQISTE); + ier &= ~clear; + ier |= set; + cmdq_writel(cq_host, ier, CQISTE); + cmdq_writel(cq_host, ier, CQISGE); + /* ensure the writes are done */ + mb(); +} + +static int cmdq_clear_task_poll(struct cmdq_host *cq_host, unsigned int tag) +{ + int retries = 100; + + cmdq_clear_set_irqs(cq_host, CQIS_TCL, 0); + cmdq_writel(cq_host, 1<<tag, CQTCLR); + while (retries) { + /* + * Task Clear register and doorbell, + * both should indicate that task is cleared + */ + if ((cmdq_readl(cq_host, CQTCLR) & 1<<tag) || + (cmdq_readl(cq_host, CQTDBR) & 1<<tag)) { + udelay(5); + retries--; + continue; + } else + break; + } + + cmdq_clear_set_irqs(cq_host, 0, CQIS_TCL); + return retries ? 0 : -ETIMEDOUT; +} + +#define DRV_NAME "cmdq-host" + +static void cmdq_dump_task_history(struct cmdq_host *cq_host) +{ + int i; + + if (likely(!cq_host->mmc->cmdq_thist_enabled)) + return; + + if (!cq_host->thist) { + pr_err("%s: %s: CMDQ task history buffer not allocated\n", + mmc_hostname(cq_host->mmc), __func__); + return; + } + + pr_err("---- Circular Task History ----\n"); + pr_err(DRV_NAME ": Last entry index: %d", cq_host->thist_idx - 1); + + for (i = 0; i < cq_host->num_slots; i++) { + pr_err(DRV_NAME ": [%02d]%s Task: 0x%08x | Args: 0x%08x\n", i, + (cq_host->thist[i].is_dcmd) ? "DCMD" : "DATA", + lower_32_bits(cq_host->thist[i].task), + upper_32_bits(cq_host->thist[i].task)); + } + pr_err("-------------------------\n"); +} + +static void cmdq_dump_adma_mem(struct cmdq_host *cq_host) +{ + struct mmc_host *mmc = cq_host->mmc; + dma_addr_t desc_dma; + int tag = 0; + unsigned long data_active_reqs = + mmc->cmdq_ctx.data_active_reqs; + unsigned long desc_size = + (cq_host->mmc->max_segs * cq_host->trans_desc_len); + + for_each_set_bit(tag, &data_active_reqs, cq_host->num_slots) { + desc_dma = get_trans_desc_dma(cq_host, tag); + pr_err("%s: %s: tag = %d, trans_dma(phys) = %pad, trans_desc(virt) = 0x%p\n", + mmc_hostname(mmc), __func__, tag, + &desc_dma, get_trans_desc(cq_host, tag)); + print_hex_dump(KERN_ERR, "cmdq-adma:", DUMP_PREFIX_ADDRESS, + 32, 8, get_trans_desc(cq_host, tag), + (desc_size), false); + } +} + +static void cmdq_dumpregs(struct cmdq_host *cq_host) +{ + struct mmc_host *mmc = cq_host->mmc; + int offset = 0; + + if (cq_host->offset_changed) + offset = CQ_V5_VENDOR_CFG; + + MMC_TRACE(mmc, + "%s: 0x0C=0x%08x 0x10=0x%08x 0x14=0x%08x 0x18=0x%08x 0x28=0x%08x 0x2C=0x%08x 0x30=0x%08x 0x34=0x%08x 0x54=0x%08x 0x58=0x%08x 0x5C=0x%08x 0x48=0x%08x\n", + __func__, cmdq_readl(cq_host, CQCTL), cmdq_readl(cq_host, CQIS), + cmdq_readl(cq_host, CQISTE), cmdq_readl(cq_host, CQISGE), + cmdq_readl(cq_host, CQTDBR), cmdq_readl(cq_host, CQTCN), + cmdq_readl(cq_host, CQDQS), cmdq_readl(cq_host, CQDPT), + cmdq_readl(cq_host, CQTERRI), cmdq_readl(cq_host, CQCRI), + cmdq_readl(cq_host, CQCRA), cmdq_readl(cq_host, CQCRDCT)); + pr_err(DRV_NAME ": ========== REGISTER DUMP (%s)==========\n", + mmc_hostname(mmc)); + + pr_err(DRV_NAME ": Caps: 0x%08x | Version: 0x%08x\n", + cmdq_readl(cq_host, CQCAP), + cmdq_readl(cq_host, CQVER)); + pr_err(DRV_NAME ": Queing config: 0x%08x | Queue Ctrl: 0x%08x\n", + cmdq_readl(cq_host, CQCFG), + cmdq_readl(cq_host, CQCTL)); + pr_err(DRV_NAME ": Int stat: 0x%08x | Int enab: 0x%08x\n", + cmdq_readl(cq_host, CQIS), + cmdq_readl(cq_host, CQISTE)); + pr_err(DRV_NAME ": Int sig: 0x%08x | Int Coal: 0x%08x\n", + cmdq_readl(cq_host, CQISGE), + cmdq_readl(cq_host, CQIC)); + pr_err(DRV_NAME ": TDL base: 0x%08x | TDL up32: 0x%08x\n", + cmdq_readl(cq_host, CQTDLBA), + cmdq_readl(cq_host, CQTDLBAU)); + pr_err(DRV_NAME ": Doorbell: 0x%08x | Comp Notif: 0x%08x\n", + cmdq_readl(cq_host, CQTDBR), + cmdq_readl(cq_host, CQTCN)); + pr_err(DRV_NAME ": Dev queue: 0x%08x | Dev Pend: 0x%08x\n", + cmdq_readl(cq_host, CQDQS), + cmdq_readl(cq_host, CQDPT)); + pr_err(DRV_NAME ": Task clr: 0x%08x | Send stat 1: 0x%08x\n", + cmdq_readl(cq_host, CQTCLR), + cmdq_readl(cq_host, CQSSC1)); + pr_err(DRV_NAME ": Send stat 2: 0x%08x | DCMD resp: 0x%08x\n", + cmdq_readl(cq_host, CQSSC2), + cmdq_readl(cq_host, CQCRDCT)); + pr_err(DRV_NAME ": Resp err mask: 0x%08x | Task err: 0x%08x\n", + cmdq_readl(cq_host, CQRMEM), + cmdq_readl(cq_host, CQTERRI)); + pr_err(DRV_NAME ": Resp idx 0x%08x | Resp arg: 0x%08x\n", + cmdq_readl(cq_host, CQCRI), + cmdq_readl(cq_host, CQCRA)); + pr_err(DRV_NAME": Vendor cfg 0x%08x\n", + cmdq_readl(cq_host, CQ_VENDOR_CFG + offset)); + pr_err(DRV_NAME ": ===========================================\n"); + + cmdq_dump_task_history(cq_host); + if (cq_host->ops->dump_vendor_regs) + cq_host->ops->dump_vendor_regs(mmc); +} + +/** + * The allocated descriptor table for task, link & transfer descritors + * looks like: + * |----------| + * |task desc | |->|----------| + * |----------| | |trans desc| + * |link desc-|->| |----------| + * |----------| . + * . . + * no. of slots max-segs + * . |----------| + * |----------| + * The idea here is to create the [task+trans] table and mark & point the + * link desc to the transfer desc table on a per slot basis. + */ +static int cmdq_host_alloc_tdl(struct cmdq_host *cq_host) +{ + + size_t desc_size; + size_t data_size; + int i = 0; + + /* task descriptor can be 64/128 bit irrespective of arch */ + if (cq_host->caps & CMDQ_TASK_DESC_SZ_128) { + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCFG) | + CQ_TASK_DESC_SZ, CQCFG); + cq_host->task_desc_len = 16; + } else { + cq_host->task_desc_len = 8; + } + + /* + * 96 bits length of transfer desc instead of 128 bits which means + * ADMA would expect next valid descriptor at the 96th bit + * or 128th bit + */ + if (cq_host->dma64) { + if (cq_host->quirks & CMDQ_QUIRK_SHORT_TXFR_DESC_SZ) + cq_host->trans_desc_len = 12; + else + cq_host->trans_desc_len = 16; + cq_host->link_desc_len = 16; + } else { + cq_host->trans_desc_len = 8; + cq_host->link_desc_len = 8; + } + + /* total size of a slot: 1 task & 1 transfer (link) */ + cq_host->slot_sz = cq_host->task_desc_len + cq_host->link_desc_len; + + desc_size = cq_host->slot_sz * cq_host->num_slots; + + data_size = cq_host->trans_desc_len * cq_host->mmc->max_segs * + (cq_host->num_slots - 1); + + pr_info("%s: desc_size: %d data_sz: %d slot-sz: %d\n", __func__, + (int)desc_size, (int)data_size, cq_host->slot_sz); + + /* + * allocate a dma-mapped chunk of memory for the descriptors + * allocate a dma-mapped chunk of memory for link descriptors + * setup each link-desc memory offset per slot-number to + * the descriptor table. + */ + cq_host->desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), + desc_size, + &cq_host->desc_dma_base, + GFP_KERNEL); + cq_host->trans_desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), + data_size, + &cq_host->trans_desc_dma_base, + GFP_KERNEL); + cq_host->thist = devm_kzalloc(mmc_dev(cq_host->mmc), + (sizeof(*cq_host->thist) * + cq_host->num_slots), + GFP_KERNEL); + if (!cq_host->desc_base || !cq_host->trans_desc_base) + return -ENOMEM; + + pr_info("desc-base: 0x%p trans-base: 0x%p\n desc_dma 0x%llx trans_dma: 0x%llx\n", + cq_host->desc_base, cq_host->trans_desc_base, + (unsigned long long)cq_host->desc_dma_base, + (unsigned long long) cq_host->trans_desc_dma_base); + + for (; i < (cq_host->num_slots); i++) + setup_trans_desc(cq_host, i); + + return 0; +} + +static int cmdq_enable(struct mmc_host *mmc) +{ + int err = 0; + u32 cqcfg; + u32 cqcap = 0; + bool dcmd_enable; + struct cmdq_host *cq_host = mmc_cmdq_private(mmc); + + if (!cq_host || !mmc->card || !mmc_card_cmdq(mmc->card)) { + err = -EINVAL; + goto out; + } + + if (cq_host->enabled) + goto out; + + cmdq_runtime_pm_get(cq_host); + cqcfg = cmdq_readl(cq_host, CQCFG); + if (cqcfg & 0x1) { + pr_info("%s: %s: cq_host is already enabled\n", + mmc_hostname(mmc), __func__); + WARN_ON(1); + goto pm_ref_count; + } + + if (cq_host->quirks & CMDQ_QUIRK_NO_DCMD) + dcmd_enable = false; + else + dcmd_enable = true; + + cqcfg = ((cq_host->caps & CMDQ_TASK_DESC_SZ_128 ? CQ_TASK_DESC_SZ : 0) | + (dcmd_enable ? CQ_DCMD : 0)); + + cqcap = cmdq_readl(cq_host, CQCAP); + if (cqcap & CQCAP_CS) { + /* + * In case host controller supports cryptographic operations + * then, it uses 128bit task descriptor. Upper 64 bits of task + * descriptor would be used to pass crypto specific informaton. + */ + cq_host->caps |= CMDQ_CAP_CRYPTO_SUPPORT | + CMDQ_TASK_DESC_SZ_128; + cqcfg |= CQ_ICE_ENABLE; + /* + * For SDHC v5.0 onwards, ICE 3.0 specific registers are added + * in CQ register space, due to which few CQ registers are + * shifted. Set offset_changed boolean to use updated address. + */ + cq_host->offset_changed = true; + } + + cmdq_writel(cq_host, cqcfg, CQCFG); + /* enable CQ_HOST */ + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCFG) | CQ_ENABLE, + CQCFG); + + if (!cq_host->desc_base || + !cq_host->trans_desc_base) { + err = cmdq_host_alloc_tdl(cq_host); + if (err) + goto pm_ref_count; + } + + cmdq_writel(cq_host, lower_32_bits(cq_host->desc_dma_base), CQTDLBA); + cmdq_writel(cq_host, upper_32_bits(cq_host->desc_dma_base), CQTDLBAU); + + /* + * disable all vendor interrupts + * enable CMDQ interrupts + * enable the vendor error interrupts + */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + + cmdq_clear_set_irqs(cq_host, 0x0, CQ_INT_ALL); + + /* cq_host would use this rca to address the card */ + cmdq_writel(cq_host, mmc->card->rca, CQSSC2); + + /* send QSR at lesser intervals than the default */ + cmdq_writel(cq_host, SEND_QSR_INTERVAL, CQSSC1); + + /* enable bkops exception indication */ + if (mmc_card_configured_manual_bkops(mmc->card) && + !mmc_card_configured_auto_bkops(mmc->card)) + cmdq_writel(cq_host, cmdq_readl(cq_host, CQRMEM) | CQ_EXCEPTION, + CQRMEM); + + /* ensure the writes are done before enabling CQE */ + mb(); + + cq_host->enabled = true; + mmc_host_clr_cq_disable(mmc); + + if (cq_host->ops->set_transfer_params) + cq_host->ops->set_transfer_params(mmc); + + if (cq_host->ops->set_block_size) + cq_host->ops->set_block_size(cq_host->mmc); + + if (cq_host->ops->set_data_timeout) + cq_host->ops->set_data_timeout(mmc, 0xf); + + if (cq_host->ops->clear_set_dumpregs) + cq_host->ops->clear_set_dumpregs(mmc, 1); + + if (cq_host->ops->enhanced_strobe_mask) + cq_host->ops->enhanced_strobe_mask(mmc, true); + +pm_ref_count: + cmdq_runtime_pm_put(cq_host); +out: + MMC_TRACE(mmc, "%s: CQ enabled err: %d\n", __func__, err); + return err; +} + +static void cmdq_disable_nosync(struct mmc_host *mmc, bool soft) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + + if (soft) { + cmdq_writel(cq_host, cmdq_readl( + cq_host, CQCFG) & ~(CQ_ENABLE), + CQCFG); + } + if (cq_host->ops->enhanced_strobe_mask) + cq_host->ops->enhanced_strobe_mask(mmc, false); + + cq_host->enabled = false; + mmc_host_set_cq_disable(mmc); + MMC_TRACE(mmc, "%s: CQ disabled\n", __func__); +} + +static void cmdq_disable(struct mmc_host *mmc, bool soft) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + + cmdq_runtime_pm_get(cq_host); + cmdq_disable_nosync(mmc, soft); + cmdq_runtime_pm_put(cq_host); +} + +static void cmdq_reset(struct mmc_host *mmc, bool soft) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + unsigned int cqcfg; + unsigned int tdlba; + unsigned int tdlbau; + unsigned int rca; + int ret; + + cmdq_runtime_pm_get(cq_host); + cqcfg = cmdq_readl(cq_host, CQCFG); + tdlba = cmdq_readl(cq_host, CQTDLBA); + tdlbau = cmdq_readl(cq_host, CQTDLBAU); + rca = cmdq_readl(cq_host, CQSSC2); + + cmdq_disable(mmc, true); + + if (cq_host->ops->reset) { + ret = cq_host->ops->reset(mmc); + if (ret) { + pr_crit("%s: reset CMDQ controller: failed\n", + mmc_hostname(mmc)); + BUG(); + } + } + + cmdq_writel(cq_host, tdlba, CQTDLBA); + cmdq_writel(cq_host, tdlbau, CQTDLBAU); + + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + + cmdq_clear_set_irqs(cq_host, 0x0, CQ_INT_ALL); + + /* cq_host would use this rca to address the card */ + cmdq_writel(cq_host, rca, CQSSC2); + + /* ensure the writes are done before enabling CQE */ + mb(); + + cmdq_writel(cq_host, cqcfg, CQCFG); + cmdq_runtime_pm_put(cq_host); + cq_host->enabled = true; + mmc_host_clr_cq_disable(mmc); +} + +static void cmdq_prep_task_desc(struct mmc_request *mrq, + u64 *data, bool intr, bool qbr) +{ + struct mmc_cmdq_req *cmdq_req = mrq->cmdq_req; + u32 req_flags = cmdq_req->cmdq_req_flags; + + pr_debug("%s: %s: data-tag: 0x%08x - dir: %d - prio: %d - cnt: 0x%08x - addr: 0x%llx\n", + mmc_hostname(mrq->host), __func__, + !!(req_flags & DAT_TAG), !!(req_flags & DIR), + !!(req_flags & PRIO), cmdq_req->data.blocks, + (u64)mrq->cmdq_req->blk_addr); + + *data = VALID(1) | + END(1) | + INT(intr) | + ACT(0x5) | + FORCED_PROG(!!(req_flags & FORCED_PRG)) | + CONTEXT(mrq->cmdq_req->ctx_id) | + DATA_TAG(!!(req_flags & DAT_TAG)) | + DATA_DIR(!!(req_flags & DIR)) | + PRIORITY(!!(req_flags & PRIO)) | + QBAR(qbr) | + REL_WRITE(!!(req_flags & REL_WR)) | + BLK_COUNT(mrq->cmdq_req->data.blocks) | + BLK_ADDR((u64)mrq->cmdq_req->blk_addr); + + MMC_TRACE(mrq->host, + "%s: Task: 0x%08x | Args: 0x%08x | cnt: 0x%08x\n", __func__, + lower_32_bits(*data), + upper_32_bits(*data), + mrq->cmdq_req->data.blocks); +} + +static int cmdq_dma_map(struct mmc_host *host, struct mmc_request *mrq) +{ + int sg_count; + struct mmc_data *data = mrq->data; + + if (!data) + return -EINVAL; + + sg_count = dma_map_sg(mmc_dev(host), data->sg, + data->sg_len, + (data->flags & MMC_DATA_WRITE) ? + DMA_TO_DEVICE : DMA_FROM_DEVICE); + if (!sg_count) { + pr_err("%s: sg-len: %d\n", __func__, data->sg_len); + return -ENOMEM; + } + + return sg_count; +} + +static void cmdq_set_tran_desc(u8 *desc, dma_addr_t addr, int len, + bool end, bool is_dma64) +{ + __le32 *attr = (__le32 __force *)desc; + + *attr = (VALID(1) | + END(end ? 1 : 0) | + INT(0) | + ACT(0x4) | + DAT_LENGTH(len)); + + if (is_dma64) { + __le64 *dataddr = (__le64 __force *)(desc + 4); + + dataddr[0] = cpu_to_le64(addr); + } else { + __le32 *dataddr = (__le32 __force *)(desc + 4); + + dataddr[0] = cpu_to_le32(addr); + } +} + +static int cmdq_prep_tran_desc(struct mmc_request *mrq, + struct cmdq_host *cq_host, int tag) +{ + struct mmc_data *data = mrq->data; + int i, sg_count, len; + bool end = false; + dma_addr_t addr; + u8 *desc; + struct scatterlist *sg; + + sg_count = cmdq_dma_map(mrq->host, mrq); + if (sg_count < 0) { + pr_err("%s: %s: unable to map sg lists, %d\n", + mmc_hostname(mrq->host), __func__, sg_count); + return sg_count; + } + + desc = get_trans_desc(cq_host, tag); + memset(desc, 0, cq_host->trans_desc_len * cq_host->mmc->max_segs); + + for_each_sg(data->sg, sg, sg_count, i) { + addr = sg_dma_address(sg); + len = sg_dma_len(sg); + + if ((i+1) == sg_count) + end = true; + cmdq_set_tran_desc(desc, addr, len, end, cq_host->dma64); + desc += cq_host->trans_desc_len; + } + + pr_debug("%s: req: 0x%p tag: %d calc_trans_des: 0x%p sg-cnt: %d\n", + __func__, mrq->req, tag, desc, sg_count); + + return 0; +} + +static void cmdq_log_task_desc_history(struct cmdq_host *cq_host, u64 task, + bool is_dcmd) +{ + if (likely(!cq_host->mmc->cmdq_thist_enabled)) + return; + + if (!cq_host->thist) { + pr_err("%s: %s: CMDQ task history buffer not allocated\n", + mmc_hostname(cq_host->mmc), __func__); + return; + } + + if (cq_host->thist_idx >= cq_host->num_slots) + cq_host->thist_idx = 0; + + cq_host->thist[cq_host->thist_idx].is_dcmd = is_dcmd; + memcpy(&cq_host->thist[cq_host->thist_idx++].task, + &task, cq_host->task_desc_len); +} + +static void cmdq_prep_dcmd_desc(struct mmc_host *mmc, + struct mmc_request *mrq) +{ + u64 *task_desc = NULL; + u64 data = 0; + u8 resp_type; + u8 *desc; + __le64 *dataddr; + struct cmdq_host *cq_host = mmc_cmdq_private(mmc); + u8 timing; + + if (!(mrq->cmd->flags & MMC_RSP_PRESENT)) { + resp_type = 0x0; + timing = 0x1; + } else { + if (mrq->cmd->flags & MMC_RSP_BUSY) { + resp_type = 0x3; + timing = 0x0; + } else { + resp_type = 0x2; + timing = 0x1; + } + } + + task_desc = (__le64 __force *)get_desc(cq_host, cq_host->dcmd_slot); + memset(task_desc, 0, cq_host->task_desc_len); + data |= (VALID(1) | + END(1) | + INT(1) | + QBAR(1) | + ACT(0x5) | + CMD_INDEX(mrq->cmd->opcode) | + CMD_TIMING(timing) | RESP_TYPE(resp_type)); + *task_desc |= data; + desc = (u8 *)task_desc; + pr_debug("cmdq: dcmd: cmd: %d timing: %d resp: %d\n", + mrq->cmd->opcode, timing, resp_type); + dataddr = (__le64 __force *)(desc + 4); + dataddr[0] = cpu_to_le64((u64)mrq->cmd->arg); + cmdq_log_task_desc_history(cq_host, *task_desc, true); + MMC_TRACE(mrq->host, + "%s: DCMD: Task: 0x%08x | Args: 0x%08x\n", + __func__, + lower_32_bits(*task_desc), + upper_32_bits(*task_desc)); +} + +static inline +void cmdq_prep_crypto_desc(struct cmdq_host *cq_host, u64 *task_desc, + u64 ice_ctx) +{ + u64 *ice_desc = NULL; + + if (cq_host->caps & CMDQ_CAP_CRYPTO_SUPPORT) { + /* + * Get the address of ice context for the given task descriptor. + * ice context is present in the upper 64bits of task descriptor + * ice_conext_base_address = task_desc + 8-bytes + */ + ice_desc = (__le64 __force *)((u8 *)task_desc + + CQ_TASK_DESC_TASK_PARAMS_SIZE); + memset(ice_desc, 0, CQ_TASK_DESC_ICE_PARAMS_SIZE); + + /* + * Assign upper 64bits data of task descritor with ice context + */ + if (ice_ctx) + *ice_desc = cpu_to_le64(ice_ctx); + } +} + +static void cmdq_pm_qos_vote(struct sdhci_host *host, struct mmc_request *mrq) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + sdhci_msm_pm_qos_cpu_vote(host, + msm_host->pdata->pm_qos_data.cmdq_latency, mrq->req->cpu); +} + +static void cmdq_pm_qos_unvote(struct sdhci_host *host, struct mmc_request *mrq) +{ + /* use async as we're inside an atomic context (soft-irq) */ + sdhci_msm_pm_qos_cpu_unvote(host, mrq->req->cpu, true); +} + +static int cmdq_request(struct mmc_host *mmc, struct mmc_request *mrq) +{ + int err = 0; + u64 data = 0; + u64 *task_desc = NULL; + u32 tag = mrq->cmdq_req->tag; + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + struct sdhci_host *host = mmc_priv(mmc); + u64 ice_ctx = 0; + + if (!cq_host->enabled) { + pr_err("%s: CMDQ host not enabled yet !!!\n", + mmc_hostname(mmc)); + err = -EINVAL; + goto out; + } + + cmdq_runtime_pm_get(cq_host); + + if (mrq->cmdq_req->cmdq_req_flags & DCMD) { + cmdq_prep_dcmd_desc(mmc, mrq); + cq_host->mrq_slot[DCMD_SLOT] = mrq; + /* DCMD's are always issued on a fixed slot */ + tag = DCMD_SLOT; + goto ring_doorbell; + } + + if (cq_host->ops->crypto_cfg) { + err = cq_host->ops->crypto_cfg(mmc, mrq, tag, &ice_ctx); + if (err) { + pr_err("%s: failed to configure crypto: err %d tag %d\n", + mmc_hostname(mmc), err, tag); + goto out; + } + } + + task_desc = (__le64 __force *)get_desc(cq_host, tag); + + cmdq_prep_task_desc(mrq, &data, 1, + (mrq->cmdq_req->cmdq_req_flags & QBR)); + *task_desc = cpu_to_le64(data); + + cmdq_prep_crypto_desc(cq_host, task_desc, ice_ctx); + + cmdq_log_task_desc_history(cq_host, *task_desc, false); + + err = cmdq_prep_tran_desc(mrq, cq_host, tag); + if (err) { + pr_err("%s: %s: failed to setup tx desc: %d\n", + mmc_hostname(mmc), __func__, err); + goto out; + } + + cq_host->mrq_slot[tag] = mrq; + + /* PM QoS */ + sdhci_msm_pm_qos_irq_vote(host); + cmdq_pm_qos_vote(host, mrq); +ring_doorbell: + /* Ensure the task descriptor list is flushed before ringing doorbell */ + wmb(); + if (cmdq_readl(cq_host, CQTDBR) & (1 << tag)) { + cmdq_dumpregs(cq_host); + BUG_ON(1); + } + MMC_TRACE(mmc, "%s: tag: %d\n", __func__, tag); + cmdq_writel(cq_host, 1 << tag, CQTDBR); + /* Commit the doorbell write immediately */ + wmb(); + +out: + return err; +} + +static void cmdq_finish_data(struct mmc_host *mmc, unsigned int tag) +{ + struct mmc_request *mrq; + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + int offset = 0; + int err = 0; + + if (cq_host->offset_changed) + offset = CQ_V5_VENDOR_CFG; + mrq = get_req_by_tag(cq_host, tag); + if (tag == cq_host->dcmd_slot) + mrq->cmd->resp[0] = cmdq_readl(cq_host, CQCRDCT); + + if (mrq->cmdq_req->cmdq_req_flags & DCMD) + cmdq_writel(cq_host, + cmdq_readl(cq_host, CQ_VENDOR_CFG + offset) | + CMDQ_SEND_STATUS_TRIGGER, CQ_VENDOR_CFG + offset); + + cmdq_runtime_pm_put(cq_host); + + if (cq_host->ops->crypto_cfg_end) { + err = cq_host->ops->crypto_cfg_end(mmc, mrq); + if (err) { + pr_err("%s: failed to end ice config: err %d tag %d\n", + mmc_hostname(mmc), err, tag); + } + } + if (!(cq_host->caps & CMDQ_CAP_CRYPTO_SUPPORT) && + cq_host->ops->crypto_cfg_reset) + cq_host->ops->crypto_cfg_reset(mmc, tag); + mrq->done(mrq); +} + +irqreturn_t cmdq_irq(struct mmc_host *mmc, int err) +{ + u32 status; + unsigned long tag = 0, comp_status; + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + unsigned long err_info = 0; + struct mmc_request *mrq; + int ret; + u32 dbr_set = 0; + u32 dev_pend_set = 0; + int stat_err = 0; + + status = cmdq_readl(cq_host, CQIS); + + if (!status && !err) + return IRQ_NONE; + MMC_TRACE(mmc, "%s: CQIS: 0x%x err: %d\n", + __func__, status, err); + + stat_err = status & (CQIS_RED | CQIS_GCE | CQIS_ICCE); + + if (err || stat_err) { + err_info = cmdq_readl(cq_host, CQTERRI); + pr_err("%s: err: %d status: 0x%08x task-err-info (0x%08lx)\n", + mmc_hostname(mmc), err, status, err_info); + + /* + * Need to halt CQE in case of error in interrupt context itself + * otherwise CQE may proceed with sending CMD to device even if + * CQE/card is in error state. + * CMDQ error handling will make sure that it is unhalted after + * handling all the errors. + */ + ret = cmdq_halt_poll(mmc, true); + if (ret) + pr_err("%s: %s: halt failed ret=%d\n", + mmc_hostname(mmc), __func__, ret); + + /* + * Clear the CQIS after halting incase of error. This is done + * because if CQIS is cleared before halting, the CQ will + * continue with issueing commands for rest of requests with + * Doorbell rung. This will overwrite the Resp Arg register. + * So CQ must be halted first and then CQIS cleared incase + * of error + */ + cmdq_writel(cq_host, status, CQIS); + + cmdq_dumpregs(cq_host); + + if (!err_info) { + /* + * It may so happen sometimes for few errors(like ADMA) + * that HW cannot give CQTERRI info. + * Thus below is a HW WA for recovering from such + * scenario. + * - To halt/disable CQE and do reset_all. + * Since there is no way to know which tag would + * have caused such error, so check for any first + * bit set in doorbell and proceed with an error. + */ + dbr_set = cmdq_readl(cq_host, CQTDBR); + if (!dbr_set) { + pr_err("%s: spurious/force error interrupt\n", + mmc_hostname(mmc)); + cmdq_halt_poll(mmc, false); + mmc_host_clr_halt(mmc); + return IRQ_HANDLED; + } + + tag = ffs(dbr_set) - 1; + pr_err("%s: error tag selected: tag = %lu\n", + mmc_hostname(mmc), tag); + mrq = get_req_by_tag(cq_host, tag); + if (mrq->data) + mrq->data->error = err; + else + mrq->cmd->error = err; + /* + * Get ADMA descriptor memory in case of ADMA + * error for debug. + */ + if (err == -EIO) + cmdq_dump_adma_mem(cq_host); + goto skip_cqterri; + } + + if (err_info & CQ_RMEFV) { + tag = GET_CMD_ERR_TAG(err_info); + pr_err("%s: CMD err tag: %lu\n", __func__, tag); + + mrq = get_req_by_tag(cq_host, tag); + /* CMD44/45/46/47 will not have a valid cmd */ + if (mrq->cmd) + mrq->cmd->error = err; + else + mrq->data->error = err; + } else if (err_info & CQ_DTEFV) { + tag = GET_DAT_ERR_TAG(err_info); + pr_err("%s: Dat err tag: %lu\n", __func__, tag); + mrq = get_req_by_tag(cq_host, tag); + mrq->data->error = err; + } + +skip_cqterri: + /* + * If CQE halt fails then, disable CQE + * from processing any further requests + */ + if (ret) { + cmdq_disable_nosync(mmc, true); + /* + * Enable legacy interrupts as CQE halt has failed. + * This is needed to send legacy commands like status + * cmd as part of error handling work. + */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, false); + } + + /* + * CQE detected a reponse error from device + * In most cases, this would require a reset. + */ + if (stat_err & CQIS_RED) { + /* + * will check if the RED error is due to a bkops + * exception once the queue is empty + */ + BUG_ON(!mmc->card); + if (mmc_card_configured_manual_bkops(mmc->card) || + mmc_card_configured_auto_bkops(mmc->card)) + mmc->card->bkops.needs_check = true; + + mrq->cmdq_req->resp_err = true; + pr_err("%s: Response error (0x%08x) from card !!!", + mmc_hostname(mmc), cmdq_readl(cq_host, CQCRA)); + + } else { + mrq->cmdq_req->resp_idx = cmdq_readl(cq_host, CQCRI); + mrq->cmdq_req->resp_arg = cmdq_readl(cq_host, CQCRA); + } + + /* + * Generic Crypto error detected by CQE. + * Its a fatal, would require cmdq reset. + */ + if (stat_err & CQIS_GCE) { + if (mrq->data) + mrq->data->error = -EIO; + pr_err("%s: Crypto generic error while processing task %lu!", + mmc_hostname(mmc), tag); + MMC_TRACE(mmc, "%s: GCE error detected with tag %lu\n", + __func__, tag); + } + /* + * Invalid crypto config error detected by CQE, clear the task. + * Task can be cleared only when CQE is halt state. + */ + if (stat_err & CQIS_ICCE) { + /* + * Invalid Crypto Config Error is detected at the + * beginning of the transfer before the actual execution + * started. So just clear the task in CQE. No need to + * clear in device. Only the task which caused ICCE has + * to be cleared. Other tasks can be continue processing + * The first task which is about to be prepared would + * cause ICCE Error. + */ + dbr_set = cmdq_readl(cq_host, CQTDBR); + dev_pend_set = cmdq_readl(cq_host, CQDPT); + if (dbr_set ^ dev_pend_set) + tag = ffs(dbr_set ^ dev_pend_set) - 1; + mrq = get_req_by_tag(cq_host, tag); + pr_err("%s: Crypto config error while processing task %lu!", + mmc_hostname(mmc), tag); + MMC_TRACE(mmc, "%s: ICCE error with tag %lu\n", + __func__, tag); + if (mrq->data) + mrq->data->error = -EIO; + else if (mrq->cmd) + mrq->cmd->error = -EIO; + /* + * If CQE is halted and tag is valid then clear the task + * then un-halt CQE and set flag to skip error recovery. + * If any of the condtions is not met thene it will + * enter into default error recovery path. + */ + if (!ret && (dbr_set ^ dev_pend_set)) { + ret = cmdq_clear_task_poll(cq_host, tag); + if (ret) { + pr_err("%s: %s: task[%lu] clear failed ret=%d\n", + mmc_hostname(mmc), + __func__, tag, ret); + } else if (!cmdq_halt_poll(mmc, false)) { + mrq->cmdq_req->skip_err_handling = true; + } + } + } + cmdq_finish_data(mmc, tag); + } else { + cmdq_writel(cq_host, status, CQIS); + } + + if (status & CQIS_TCC) { + /* read CQTCN and complete the request */ + comp_status = cmdq_readl(cq_host, CQTCN); + if (!comp_status) + goto out; + /* + * The CQTCN must be cleared before notifying req completion + * to upper layers to avoid missing completion notification + * of new requests with the same tag. + */ + cmdq_writel(cq_host, comp_status, CQTCN); + /* + * A write memory barrier is necessary to guarantee that CQTCN + * gets cleared first before next doorbell for the same tag is + * set but that is already achieved by the barrier present + * before setting doorbell, hence one is not needed here. + */ + for_each_set_bit(tag, &comp_status, cq_host->num_slots) { + mrq = get_req_by_tag(cq_host, tag); + if (!((mrq->cmd && mrq->cmd->error) || + mrq->cmdq_req->resp_err || + (mrq->data && mrq->data->error))) { + /* complete the corresponding mrq */ + pr_debug("%s: completing tag -> %lu\n", + mmc_hostname(mmc), tag); + MMC_TRACE(mmc, "%s: completing tag -> %lu\n", + __func__, tag); + cmdq_finish_data(mmc, tag); + } + } + } + + if (status & CQIS_HAC) { + if (cq_host->ops->post_cqe_halt) + cq_host->ops->post_cqe_halt(mmc); + /* halt done: re-enable legacy interrupts */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, false); + /* halt is completed, wakeup waiting thread */ + complete(&cq_host->halt_comp); + } + +out: + return IRQ_HANDLED; +} +EXPORT_SYMBOL(cmdq_irq); + +/* cmdq_halt_poll - Halting CQE using polling method. + * @mmc: struct mmc_host + * @halt: bool halt + * This is used mainly from interrupt context to halt/unhalt + * CQE engine. + */ +static int cmdq_halt_poll(struct mmc_host *mmc, bool halt) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + int retries = 100; + + if (!halt) { + if (cq_host->ops->set_data_timeout) + cq_host->ops->set_data_timeout(mmc, 0xf); + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) & ~HALT, + CQCTL); + mmc_host_clr_halt(mmc); + return 0; + } + + cmdq_set_halt_irq(cq_host, false); + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) | HALT, CQCTL); + while (retries) { + if (!(cmdq_readl(cq_host, CQCTL) & HALT)) { + udelay(5); + retries--; + continue; + } else { + if (cq_host->ops->post_cqe_halt) + cq_host->ops->post_cqe_halt(mmc); + /* halt done: re-enable legacy interrupts */ + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, + false); + mmc_host_set_halt(mmc); + break; + } + } + cmdq_set_halt_irq(cq_host, true); + return retries ? 0 : -ETIMEDOUT; +} + +/* May sleep */ +static int cmdq_halt(struct mmc_host *mmc, bool halt) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + u32 ret = 0; + u32 config = 0; + int retries = 3; + + cmdq_runtime_pm_get(cq_host); + if (halt) { + while (retries) { + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) | HALT, + CQCTL); + ret = wait_for_completion_timeout(&cq_host->halt_comp, + msecs_to_jiffies(HALT_TIMEOUT_MS)); + if (!ret) { + pr_warn("%s: %s: HAC int timeout\n", + mmc_hostname(mmc), __func__); + if ((cmdq_readl(cq_host, CQCTL) & HALT)) { + /* + * Don't retry if CQE is halted but irq + * is not triggered in timeout period. + * And since we are returning error, + * un-halt CQE. Since irq was not fired + * yet, no need to set other params + */ + retries = 0; + config = cmdq_readl(cq_host, CQCTL); + config &= ~HALT; + cmdq_writel(cq_host, config, CQCTL); + } else { + pr_warn("%s: %s: retryng halt (%d)\n", + mmc_hostname(mmc), __func__, + retries); + retries--; + continue; + } + } else { + MMC_TRACE(mmc, "%s: halt done , retries: %d\n", + __func__, retries); + break; + } + } + ret = retries ? 0 : -ETIMEDOUT; + } else { + if (cq_host->ops->set_transfer_params) + cq_host->ops->set_transfer_params(mmc); + if (cq_host->ops->set_block_size) + cq_host->ops->set_block_size(mmc); + if (cq_host->ops->set_data_timeout) + cq_host->ops->set_data_timeout(mmc, 0xf); + if (cq_host->ops->clear_set_irqs) + cq_host->ops->clear_set_irqs(mmc, true); + MMC_TRACE(mmc, "%s: unhalt done\n", __func__); + cmdq_writel(cq_host, cmdq_readl(cq_host, CQCTL) & ~HALT, + CQCTL); + } + cmdq_runtime_pm_put(cq_host); + return ret; +} + +static void cmdq_post_req(struct mmc_host *mmc, int tag, int err) +{ + struct cmdq_host *cq_host; + struct mmc_request *mrq; + struct mmc_data *data; + struct sdhci_host *sdhci_host = mmc_priv(mmc); + + if (WARN_ON(!mmc)) + return; + + cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + mrq = get_req_by_tag(cq_host, tag); + data = mrq->data; + + if (data) { + data->error = err; + dma_unmap_sg(mmc_dev(mmc), data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (err) + data->bytes_xfered = 0; + else + data->bytes_xfered = blk_rq_bytes(mrq->req); + + /* we're in atomic context (soft-irq) so unvote async. */ + sdhci_msm_pm_qos_irq_unvote(sdhci_host, true); + cmdq_pm_qos_unvote(sdhci_host, mrq); + } +} + +static void cmdq_dumpstate(struct mmc_host *mmc) +{ + struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc); + cmdq_runtime_pm_get(cq_host); + cmdq_dumpregs(cq_host); + cmdq_runtime_pm_put(cq_host); +} + +static int cmdq_late_init(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + /* + * TODO: This should basically move to something like "sdhci-cmdq-msm" + * for msm specific implementation. + */ + sdhci_msm_pm_qos_irq_init(host); + + if (msm_host->pdata->pm_qos_data.cmdq_valid) + sdhci_msm_pm_qos_cpu_init(host, + msm_host->pdata->pm_qos_data.cmdq_latency); + return 0; +} + +static const struct mmc_cmdq_host_ops cmdq_host_ops = { + .init = cmdq_late_init, + .enable = cmdq_enable, + .disable = cmdq_disable, + .request = cmdq_request, + .post_req = cmdq_post_req, + .halt = cmdq_halt, + .reset = cmdq_reset, + .dumpstate = cmdq_dumpstate, +}; + +struct cmdq_host *cmdq_pltfm_init(struct platform_device *pdev) +{ + struct cmdq_host *cq_host; + struct resource *cmdq_memres = NULL; + + /* check and setup CMDQ interface */ + cmdq_memres = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "cmdq_mem"); + if (!cmdq_memres) { + dev_dbg(&pdev->dev, "CMDQ not supported\n"); + return ERR_PTR(-EINVAL); + } + + cq_host = kzalloc(sizeof(*cq_host), GFP_KERNEL); + if (!cq_host) { + dev_err(&pdev->dev, "failed to allocate memory for CMDQ\n"); + return ERR_PTR(-ENOMEM); + } + cq_host->mmio = devm_ioremap(&pdev->dev, + cmdq_memres->start, + resource_size(cmdq_memres)); + if (!cq_host->mmio) { + dev_err(&pdev->dev, "failed to remap cmdq regs\n"); + kfree(cq_host); + return ERR_PTR(-EBUSY); + } + dev_dbg(&pdev->dev, "CMDQ ioremap: done\n"); + + return cq_host; +} +EXPORT_SYMBOL(cmdq_pltfm_init); + +int cmdq_init(struct cmdq_host *cq_host, struct mmc_host *mmc, + bool dma64) +{ + int err = 0; + + cq_host->dma64 = dma64; + cq_host->mmc = mmc; + cq_host->mmc->cmdq_private = cq_host; + + cq_host->num_slots = NUM_SLOTS; + cq_host->dcmd_slot = DCMD_SLOT; + + mmc->cmdq_ops = &cmdq_host_ops; + mmc->num_cq_slots = NUM_SLOTS; + mmc->dcmd_cq_slot = DCMD_SLOT; + + cq_host->mrq_slot = kzalloc(sizeof(cq_host->mrq_slot) * + cq_host->num_slots, GFP_KERNEL); + if (!cq_host->mrq_slot) + return -ENOMEM; + + init_completion(&cq_host->halt_comp); + return err; +} +EXPORT_SYMBOL(cmdq_init); diff --git a/drivers/mmc/host/cmdq_hci.h b/drivers/mmc/host/cmdq_hci.h new file mode 100644 index 000000000000..ee5e6549fa4a --- /dev/null +++ b/drivers/mmc/host/cmdq_hci.h @@ -0,0 +1,251 @@ +/* Copyright (c) 2015-2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef LINUX_MMC_CQ_HCI_H +#define LINUX_MMC_CQ_HCI_H +#include <linux/mmc/core.h> + +/* registers */ +/* version */ +#define CQVER 0x00 +/* capabilities */ +#define CQCAP 0x04 +#define CQCAP_CS (1 << 28) +/* configuration */ +#define CQCFG 0x08 +#define CQ_DCMD 0x00001000 +#define CQ_TASK_DESC_SZ 0x00000100 +#define CQ_ENABLE 0x00000001 +#define CQ_ICE_ENABLE 0x00000002 + +/* control */ +#define CQCTL 0x0C +#define CLEAR_ALL_TASKS 0x00000100 +#define HALT 0x00000001 + +/* interrupt status */ +#define CQIS 0x10 +#define CQIS_HAC (1 << 0) +#define CQIS_TCC (1 << 1) +#define CQIS_RED (1 << 2) +#define CQIS_TCL (1 << 3) +#define CQIS_GCE (1 << 4) +#define CQIS_ICCE (1 << 5) + +/* interrupt status enable */ +#define CQISTE 0x14 + +/* interrupt signal enable */ +#define CQISGE 0x18 + +/* interrupt coalescing */ +#define CQIC 0x1C +#define CQIC_ENABLE (1 << 31) +#define CQIC_RESET (1 << 16) +#define CQIC_ICCTHWEN (1 << 15) +#define CQIC_ICCTH(x) ((x & 0x1F) << 8) +#define CQIC_ICTOVALWEN (1 << 7) +#define CQIC_ICTOVAL(x) (x & 0x7F) + +/* task list base address */ +#define CQTDLBA 0x20 + +/* task list base address upper */ +#define CQTDLBAU 0x24 + +/* door-bell */ +#define CQTDBR 0x28 + +/* task completion notification */ +#define CQTCN 0x2C + +/* device queue status */ +#define CQDQS 0x30 + +/* device pending tasks */ +#define CQDPT 0x34 + +/* task clear */ +#define CQTCLR 0x38 + +/* send status config 1 */ +#define CQSSC1 0x40 +/* + * Value n means CQE would send CMD13 during the transfer of data block + * BLOCK_CNT-n + */ +#define SEND_QSR_INTERVAL 0x70001 + +/* send status config 2 */ +#define CQSSC2 0x44 + +/* response for dcmd */ +#define CQCRDCT 0x48 + +/* response mode error mask */ +#define CQRMEM 0x50 +#define CQ_EXCEPTION (1 << 6) + +/* task error info */ +#define CQTERRI 0x54 + +/* CQTERRI bit fields */ +#define CQ_RMECI 0x1F +#define CQ_RMETI (0x1F << 8) +#define CQ_RMEFV (1 << 15) +#define CQ_DTECI (0x3F << 16) +#define CQ_DTETI (0x1F << 24) +#define CQ_DTEFV (1 << 31) + +#define GET_CMD_ERR_TAG(__r__) ((__r__ & CQ_RMETI) >> 8) +#define GET_DAT_ERR_TAG(__r__) ((__r__ & CQ_DTETI) >> 24) + +/* command response index */ +#define CQCRI 0x58 + +/* command response argument */ +#define CQCRA 0x5C + +#define CQ_INT_ALL 0x3F +#define CQIC_DEFAULT_ICCTH 31 +#define CQIC_DEFAULT_ICTOVAL 1 + +/* attribute fields */ +#define VALID(x) ((x & 1) << 0) +#define END(x) ((x & 1) << 1) +#define INT(x) ((x & 1) << 2) +#define ACT(x) ((x & 0x7) << 3) + +/* data command task descriptor fields */ +#define FORCED_PROG(x) ((x & 1) << 6) +#define CONTEXT(x) ((x & 0xF) << 7) +#define DATA_TAG(x) ((x & 1) << 11) +#define DATA_DIR(x) ((x & 1) << 12) +#define PRIORITY(x) ((x & 1) << 13) +#define QBAR(x) ((x & 1) << 14) +#define REL_WRITE(x) ((x & 1) << 15) +#define BLK_COUNT(x) ((x & 0xFFFF) << 16) +#define BLK_ADDR(x) ((x & 0xFFFFFFFF) << 32) + +/* direct command task descriptor fields */ +#define CMD_INDEX(x) ((x & 0x3F) << 16) +#define CMD_TIMING(x) ((x & 1) << 22) +#define RESP_TYPE(x) ((x & 0x3) << 23) + +/* transfer descriptor fields */ +#define DAT_LENGTH(x) ((x & 0xFFFF) << 16) +#define DAT_ADDR_LO(x) ((x & 0xFFFFFFFF) << 32) +#define DAT_ADDR_HI(x) ((x & 0xFFFFFFFF) << 0) + +/* + * Add new macro for updated CQ vendor specific + * register address for SDHC v5.0 onwards. + */ +#define CQ_V5_VENDOR_CFG 0x900 +#define CQ_VENDOR_CFG 0x100 +#define CMDQ_SEND_STATUS_TRIGGER (1 << 31) + +#define CQ_TASK_DESC_TASK_PARAMS_SIZE 8 +#define CQ_TASK_DESC_ICE_PARAMS_SIZE 8 + +struct task_history { + u64 task; + bool is_dcmd; +}; + +struct cmdq_host { + const struct cmdq_host_ops *ops; + void __iomem *mmio; + struct mmc_host *mmc; + + /* 64 bit DMA */ + bool dma64; + int num_slots; + + u32 dcmd_slot; + u32 caps; +#define CMDQ_TASK_DESC_SZ_128 0x1 +#define CMDQ_CAP_CRYPTO_SUPPORT 0x2 + + u32 quirks; +#define CMDQ_QUIRK_SHORT_TXFR_DESC_SZ 0x1 +#define CMDQ_QUIRK_NO_DCMD 0x2 + + bool enabled; + bool halted; + bool init_done; + bool offset_changed; + + u8 *desc_base; + + /* total descriptor size */ + u8 slot_sz; + + /* 64/128 bit depends on CQCFG */ + u8 task_desc_len; + + /* 64 bit on 32-bit arch, 128 bit on 64-bit */ + u8 link_desc_len; + + u8 *trans_desc_base; + /* same length as transfer descriptor */ + u8 trans_desc_len; + + dma_addr_t desc_dma_base; + dma_addr_t trans_desc_dma_base; + + struct task_history *thist; + u8 thist_idx; + + struct completion halt_comp; + struct mmc_request **mrq_slot; + void *private; +}; + +struct cmdq_host_ops { + void (*set_transfer_params)(struct mmc_host *mmc); + void (*set_data_timeout)(struct mmc_host *mmc, u32 val); + void (*clear_set_irqs)(struct mmc_host *mmc, bool clear); + void (*set_block_size)(struct mmc_host *mmc); + void (*dump_vendor_regs)(struct mmc_host *mmc); + void (*write_l)(struct cmdq_host *host, u32 val, int reg); + u32 (*read_l)(struct cmdq_host *host, int reg); + void (*clear_set_dumpregs)(struct mmc_host *mmc, bool set); + void (*enhanced_strobe_mask)(struct mmc_host *mmc, bool set); + int (*reset)(struct mmc_host *mmc); + int (*crypto_cfg)(struct mmc_host *mmc, struct mmc_request *mrq, + u32 slot, u64 *ice_ctx); + int (*crypto_cfg_end)(struct mmc_host *mmc, struct mmc_request *mrq); + void (*crypto_cfg_reset)(struct mmc_host *mmc, unsigned int slot); + void (*post_cqe_halt)(struct mmc_host *mmc); +}; + +static inline void cmdq_writel(struct cmdq_host *host, u32 val, int reg) +{ + if (unlikely(host->ops && host->ops->write_l)) + host->ops->write_l(host, val, reg); + else + writel_relaxed(val, host->mmio + reg); +} + +static inline u32 cmdq_readl(struct cmdq_host *host, int reg) +{ + if (unlikely(host->ops && host->ops->read_l)) + return host->ops->read_l(host, reg); + else + return readl_relaxed(host->mmio + reg); +} + +extern irqreturn_t cmdq_irq(struct mmc_host *mmc, int err); +extern int cmdq_init(struct cmdq_host *cq_host, struct mmc_host *mmc, + bool dma64); +extern struct cmdq_host *cmdq_pltfm_init(struct platform_device *pdev); +#endif diff --git a/drivers/mmc/host/sdhci-msm-ice.c b/drivers/mmc/host/sdhci-msm-ice.c new file mode 100644 index 000000000000..e73bdfd424cc --- /dev/null +++ b/drivers/mmc/host/sdhci-msm-ice.c @@ -0,0 +1,565 @@ +/* + * Copyright (c) 2015, 2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "sdhci-msm-ice.h" + +static void sdhci_msm_ice_error_cb(void *host_ctrl, u32 error) +{ + struct sdhci_msm_host *msm_host = (struct sdhci_msm_host *)host_ctrl; + + dev_err(&msm_host->pdev->dev, "%s: Error in ice operation 0x%x", + __func__, error); + + if (msm_host->ice.state == SDHCI_MSM_ICE_STATE_ACTIVE) + msm_host->ice.state = SDHCI_MSM_ICE_STATE_DISABLED; +} + +static struct platform_device *sdhci_msm_ice_get_pdevice(struct device *dev) +{ + struct device_node *node; + struct platform_device *ice_pdev = NULL; + + node = of_parse_phandle(dev->of_node, SDHC_MSM_CRYPTO_LABEL, 0); + if (!node) { + dev_dbg(dev, "%s: sdhc-msm-crypto property not specified\n", + __func__); + goto out; + } + ice_pdev = qcom_ice_get_pdevice(node); +out: + return ice_pdev; +} + +static +struct qcom_ice_variant_ops *sdhci_msm_ice_get_vops(struct device *dev) +{ + struct qcom_ice_variant_ops *ice_vops = NULL; + struct device_node *node; + + node = of_parse_phandle(dev->of_node, SDHC_MSM_CRYPTO_LABEL, 0); + if (!node) { + dev_dbg(dev, "%s: sdhc-msm-crypto property not specified\n", + __func__); + goto out; + } + ice_vops = qcom_ice_get_variant_ops(node); + of_node_put(node); +out: + return ice_vops; +} + +static +void sdhci_msm_enable_ice_hci(struct sdhci_host *host, bool enable) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + u32 config = 0; + u32 ice_cap = 0; + + /* + * Enable the cryptographic support inside SDHC. + * This is a global config which needs to be enabled + * all the time. + * Only when it it is enabled, the ICE_HCI capability + * will get reflected in CQCAP register. + */ + config = readl_relaxed(host->ioaddr + HC_VENDOR_SPECIFIC_FUNC4); + + if (enable) + config &= ~DISABLE_CRYPTO; + else + config |= DISABLE_CRYPTO; + writel_relaxed(config, host->ioaddr + HC_VENDOR_SPECIFIC_FUNC4); + + /* + * CQCAP register is in different register space from above + * ice global enable register. So a mb() is required to ensure + * above write gets completed before reading the CQCAP register. + */ + mb(); + + /* + * Check if ICE HCI capability support is present + * If present, enable it. + */ + ice_cap = readl_relaxed(msm_host->cryptoio + ICE_CQ_CAPABILITIES); + if (ice_cap & ICE_HCI_SUPPORT) { + config = readl_relaxed(msm_host->cryptoio + ICE_CQ_CONFIG); + + if (enable) + config |= CRYPTO_GENERAL_ENABLE; + else + config &= ~CRYPTO_GENERAL_ENABLE; + writel_relaxed(config, msm_host->cryptoio + ICE_CQ_CONFIG); + } +} + +int sdhci_msm_ice_get_dev(struct sdhci_host *host) +{ + struct device *sdhc_dev; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + if (!msm_host || !msm_host->pdev) { + pr_err("%s: invalid msm_host %p or msm_host->pdev\n", + __func__, msm_host); + return -EINVAL; + } + + sdhc_dev = &msm_host->pdev->dev; + msm_host->ice.vops = sdhci_msm_ice_get_vops(sdhc_dev); + msm_host->ice.pdev = sdhci_msm_ice_get_pdevice(sdhc_dev); + + if (msm_host->ice.pdev == ERR_PTR(-EPROBE_DEFER)) { + dev_err(sdhc_dev, "%s: ICE device not probed yet\n", + __func__); + msm_host->ice.pdev = NULL; + msm_host->ice.vops = NULL; + return -EPROBE_DEFER; + } + + if (!msm_host->ice.pdev) { + dev_dbg(sdhc_dev, "%s: invalid platform device\n", __func__); + msm_host->ice.vops = NULL; + return -ENODEV; + } + if (!msm_host->ice.vops) { + dev_dbg(sdhc_dev, "%s: invalid ice vops\n", __func__); + msm_host->ice.pdev = NULL; + return -ENODEV; + } + msm_host->ice.state = SDHCI_MSM_ICE_STATE_DISABLED; + return 0; +} + +static +int sdhci_msm_ice_pltfm_init(struct sdhci_msm_host *msm_host) +{ + struct resource *ice_memres = NULL; + struct platform_device *pdev = msm_host->pdev; + int err = 0; + + if (!msm_host->ice_hci_support) + goto out; + /* + * ICE HCI registers are present in cmdq register space. + * So map the cmdq mem for accessing ICE HCI registers. + */ + ice_memres = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "cmdq_mem"); + if (!ice_memres) { + dev_err(&pdev->dev, "Failed to get iomem resource for ice\n"); + err = -EINVAL; + goto out; + } + msm_host->cryptoio = devm_ioremap(&pdev->dev, + ice_memres->start, + resource_size(ice_memres)); + if (!msm_host->cryptoio) { + dev_err(&pdev->dev, "Failed to remap registers\n"); + err = -ENOMEM; + } +out: + return err; +} + +int sdhci_msm_ice_init(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + + if (msm_host->ice.vops->init) { + err = sdhci_msm_ice_pltfm_init(msm_host); + if (err) + goto out; + + if (msm_host->ice_hci_support) + sdhci_msm_enable_ice_hci(host, true); + + err = msm_host->ice.vops->init(msm_host->ice.pdev, + msm_host, + sdhci_msm_ice_error_cb); + if (err) { + pr_err("%s: ice init err %d\n", + mmc_hostname(host->mmc), err); + sdhci_msm_ice_print_regs(host); + if (msm_host->ice_hci_support) + sdhci_msm_enable_ice_hci(host, false); + goto out; + } + msm_host->ice.state = SDHCI_MSM_ICE_STATE_ACTIVE; + } + +out: + return err; +} + +void sdhci_msm_ice_cfg_reset(struct sdhci_host *host, u32 slot) +{ + writel_relaxed(SDHCI_MSM_ICE_ENABLE_BYPASS, + host->ioaddr + CORE_VENDOR_SPEC_ICE_CTRL_INFO_3_n + 16 * slot); +} + +static +int sdhci_msm_ice_get_cfg(struct sdhci_msm_host *msm_host, struct request *req, + unsigned int *bypass, short *key_index) +{ + int err = 0; + struct ice_data_setting ice_set; + + memset(&ice_set, 0, sizeof(struct ice_data_setting)); + if (msm_host->ice.vops->config_start) { + err = msm_host->ice.vops->config_start( + msm_host->ice.pdev, + req, &ice_set, false); + if (err) { + pr_err("%s: ice config failed %d\n", + mmc_hostname(msm_host->mmc), err); + return err; + } + } + /* if writing data command */ + if (rq_data_dir(req) == WRITE) + *bypass = ice_set.encr_bypass ? + SDHCI_MSM_ICE_ENABLE_BYPASS : + SDHCI_MSM_ICE_DISABLE_BYPASS; + /* if reading data command */ + else if (rq_data_dir(req) == READ) + *bypass = ice_set.decr_bypass ? + SDHCI_MSM_ICE_ENABLE_BYPASS : + SDHCI_MSM_ICE_DISABLE_BYPASS; + *key_index = ice_set.crypto_data.key_index; + return err; +} + +static +void sdhci_msm_ice_update_cfg(struct sdhci_host *host, u64 lba, + u32 slot, unsigned int bypass, short key_index) +{ + unsigned int ctrl_info_val = 0; + + /* Configure ICE index */ + ctrl_info_val = + (key_index & + MASK_SDHCI_MSM_ICE_CTRL_INFO_KEY_INDEX) + << OFFSET_SDHCI_MSM_ICE_CTRL_INFO_KEY_INDEX; + + /* Configure data unit size of transfer request */ + ctrl_info_val |= + (SDHCI_MSM_ICE_TR_DATA_UNIT_512_B & + MASK_SDHCI_MSM_ICE_CTRL_INFO_CDU) + << OFFSET_SDHCI_MSM_ICE_CTRL_INFO_CDU; + + /* Configure ICE bypass mode */ + ctrl_info_val |= + (bypass & MASK_SDHCI_MSM_ICE_CTRL_INFO_BYPASS) + << OFFSET_SDHCI_MSM_ICE_CTRL_INFO_BYPASS; + + writel_relaxed((lba & 0xFFFFFFFF), + host->ioaddr + CORE_VENDOR_SPEC_ICE_CTRL_INFO_1_n + 16 * slot); + writel_relaxed(((lba >> 32) & 0xFFFFFFFF), + host->ioaddr + CORE_VENDOR_SPEC_ICE_CTRL_INFO_2_n + 16 * slot); + writel_relaxed(ctrl_info_val, + host->ioaddr + CORE_VENDOR_SPEC_ICE_CTRL_INFO_3_n + 16 * slot); + /* Ensure ICE registers are configured before issuing SDHCI request */ + mb(); +} + +static inline +void sdhci_msm_ice_hci_update_cmdq_cfg(u64 dun, unsigned int bypass, + short key_index, u64 *ice_ctx) +{ + /* + * The naming convention got changed between ICE2.0 and ICE3.0 + * registers fields. Below is the equivalent names for + * ICE3.0 Vs ICE2.0: + * Data Unit Number(DUN) == Logical Base address(LBA) + * Crypto Configuration index (CCI) == Key Index + * Crypto Enable (CE) == !BYPASS + */ + if (ice_ctx) + *ice_ctx = DATA_UNIT_NUM(dun) | + CRYPTO_CONFIG_INDEX(key_index) | + CRYPTO_ENABLE(!bypass); +} + +static +void sdhci_msm_ice_hci_update_noncq_cfg(struct sdhci_host *host, + u64 dun, unsigned int bypass, short key_index) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + unsigned int crypto_params = 0; + /* + * The naming convention got changed between ICE2.0 and ICE3.0 + * registers fields. Below is the equivalent names for + * ICE3.0 Vs ICE2.0: + * Data Unit Number(DUN) == Logical Base address(LBA) + * Crypto Configuration index (CCI) == Key Index + * Crypto Enable (CE) == !BYPASS + */ + /* Configure ICE bypass mode */ + crypto_params |= + (!bypass & MASK_SDHCI_MSM_ICE_HCI_PARAM_CE) + << OFFSET_SDHCI_MSM_ICE_HCI_PARAM_CE; + /* Configure Crypto Configure Index (CCI) */ + crypto_params |= (key_index & + MASK_SDHCI_MSM_ICE_HCI_PARAM_CCI) + << OFFSET_SDHCI_MSM_ICE_HCI_PARAM_CCI; + + writel_relaxed((crypto_params & 0xFFFFFFFF), + msm_host->cryptoio + ICE_NONCQ_CRYPTO_PARAMS); + + /* Update DUN */ + writel_relaxed((dun & 0xFFFFFFFF), + msm_host->cryptoio + ICE_NONCQ_CRYPTO_DUN); + /* Ensure ICE registers are configured before issuing SDHCI request */ + mb(); +} + +int sdhci_msm_ice_cfg(struct sdhci_host *host, struct mmc_request *mrq, + u32 slot) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + short key_index = 0; + sector_t lba = 0; + unsigned int bypass = SDHCI_MSM_ICE_ENABLE_BYPASS; + struct request *req; + + if (msm_host->ice.state != SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + WARN_ON(!mrq); + if (!mrq) + return -EINVAL; + req = mrq->req; + if (req) { + lba = req->__sector; + err = sdhci_msm_ice_get_cfg(msm_host, req, &bypass, &key_index); + if (err) + return err; + pr_debug("%s: %s: slot %d bypass %d key_index %d\n", + mmc_hostname(host->mmc), + (rq_data_dir(req) == WRITE) ? "WRITE" : "READ", + slot, bypass, key_index); + } + + if (msm_host->ice_hci_support) { + /* For ICE HCI / ICE3.0 */ + sdhci_msm_ice_hci_update_noncq_cfg(host, lba, bypass, + key_index); + } else { + /* For ICE versions earlier to ICE3.0 */ + sdhci_msm_ice_update_cfg(host, lba, slot, bypass, key_index); + } + return 0; +} + +int sdhci_msm_ice_cmdq_cfg(struct sdhci_host *host, + struct mmc_request *mrq, u32 slot, u64 *ice_ctx) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + short key_index = 0; + sector_t lba = 0; + unsigned int bypass = SDHCI_MSM_ICE_ENABLE_BYPASS; + struct request *req; + + if (msm_host->ice.state != SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + WARN_ON(!mrq); + if (!mrq) + return -EINVAL; + req = mrq->req; + if (req) { + lba = req->__sector; + err = sdhci_msm_ice_get_cfg(msm_host, req, &bypass, &key_index); + if (err) + return err; + pr_debug("%s: %s: slot %d bypass %d key_index %d\n", + mmc_hostname(host->mmc), + (rq_data_dir(req) == WRITE) ? "WRITE" : "READ", + slot, bypass, key_index); + } + + if (msm_host->ice_hci_support) { + /* For ICE HCI / ICE3.0 */ + sdhci_msm_ice_hci_update_cmdq_cfg(lba, bypass, key_index, + ice_ctx); + } else { + /* For ICE versions earlier to ICE3.0 */ + sdhci_msm_ice_update_cfg(host, lba, slot, bypass, key_index); + } + return 0; +} + +int sdhci_msm_ice_cfg_end(struct sdhci_host *host, struct mmc_request *mrq) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + struct request *req; + + if (!host->is_crypto_en) + return 0; + + if (msm_host->ice.state != SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + req = mrq->req; + if (req) { + if (msm_host->ice.vops->config_end) { + err = msm_host->ice.vops->config_end(req); + if (err) { + pr_err("%s: ice config end failed %d\n", + mmc_hostname(host->mmc), err); + return err; + } + } + } + + return 0; +} + +int sdhci_msm_ice_reset(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + + if (msm_host->ice.state != SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state before reset %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + if (msm_host->ice.vops->reset) { + err = msm_host->ice.vops->reset(msm_host->ice.pdev); + if (err) { + pr_err("%s: ice reset failed %d\n", + mmc_hostname(host->mmc), err); + sdhci_msm_ice_print_regs(host); + return err; + } + } + + /* If ICE HCI support is present then re-enable it */ + if (msm_host->ice_hci_support) + sdhci_msm_enable_ice_hci(host, true); + + if (msm_host->ice.state != SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state after reset %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + return 0; +} + +int sdhci_msm_ice_resume(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + + if (msm_host->ice.state != + SDHCI_MSM_ICE_STATE_SUSPENDED) { + pr_err("%s: ice is in invalid state before resume %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + if (msm_host->ice.vops->resume) { + err = msm_host->ice.vops->resume(msm_host->ice.pdev); + if (err) { + pr_err("%s: ice resume failed %d\n", + mmc_hostname(host->mmc), err); + return err; + } + } + + msm_host->ice.state = SDHCI_MSM_ICE_STATE_ACTIVE; + return 0; +} + +int sdhci_msm_ice_suspend(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int err = 0; + + if (msm_host->ice.state != + SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state before resume %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + if (msm_host->ice.vops->suspend) { + err = msm_host->ice.vops->suspend(msm_host->ice.pdev); + if (err) { + pr_err("%s: ice suspend failed %d\n", + mmc_hostname(host->mmc), err); + return -EINVAL; + } + } + msm_host->ice.state = SDHCI_MSM_ICE_STATE_SUSPENDED; + return 0; +} + +int sdhci_msm_ice_get_status(struct sdhci_host *host, int *ice_status) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int stat = -EINVAL; + + if (msm_host->ice.state != SDHCI_MSM_ICE_STATE_ACTIVE) { + pr_err("%s: ice is in invalid state %d\n", + mmc_hostname(host->mmc), msm_host->ice.state); + return -EINVAL; + } + + if (msm_host->ice.vops->status) { + *ice_status = 0; + stat = msm_host->ice.vops->status(msm_host->ice.pdev); + if (stat < 0) { + pr_err("%s: ice get sts failed %d\n", + mmc_hostname(host->mmc), stat); + return -EINVAL; + } + *ice_status = stat; + } + return 0; +} + +void sdhci_msm_ice_print_regs(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + if (msm_host->ice.vops->debug) + msm_host->ice.vops->debug(msm_host->ice.pdev); +} diff --git a/drivers/mmc/host/sdhci-msm-ice.h b/drivers/mmc/host/sdhci-msm-ice.h new file mode 100644 index 000000000000..7699464cf71e --- /dev/null +++ b/drivers/mmc/host/sdhci-msm-ice.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2015, 2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef __SDHCI_MSM_ICE_H__ +#define __SDHCI_MSM_ICE_H__ + +#include <linux/io.h> +#include <linux/of.h> +#include <linux/blkdev.h> +#include <crypto/ice.h> + +#include "sdhci-msm.h" + +#define SDHC_MSM_CRYPTO_LABEL "sdhc-msm-crypto" +/* Timeout waiting for ICE initialization, that requires TZ access */ +#define SDHCI_MSM_ICE_COMPLETION_TIMEOUT_MS 500 + +/* + * SDHCI host controller ICE registers. There are n [0..31] + * of each of these registers + */ +#define NUM_SDHCI_MSM_ICE_CTRL_INFO_n_REGS 32 + +#define CORE_VENDOR_SPEC_ICE_CTRL 0x300 +#define CORE_VENDOR_SPEC_ICE_CTRL_INFO_1_n 0x304 +#define CORE_VENDOR_SPEC_ICE_CTRL_INFO_2_n 0x308 +#define CORE_VENDOR_SPEC_ICE_CTRL_INFO_3_n 0x30C + +/* ICE3.0 register which got added cmdq reg space */ +#define ICE_CQ_CAPABILITIES 0x04 +#define ICE_HCI_SUPPORT (1 << 28) +#define ICE_CQ_CONFIG 0x08 +#define CRYPTO_GENERAL_ENABLE (1 << 1) +#define ICE_NONCQ_CRYPTO_PARAMS 0x70 +#define ICE_NONCQ_CRYPTO_DUN 0x74 + +/* ICE3.0 register which got added hc reg space */ +#define HC_VENDOR_SPECIFIC_FUNC4 0x260 +#define DISABLE_CRYPTO (1 << 15) +#define HC_VENDOR_SPECIFIC_ICE_CTRL 0x800 +#define ICE_SW_RST_EN (1 << 0) + +/* SDHCI MSM ICE CTRL Info register offset */ +enum { + OFFSET_SDHCI_MSM_ICE_CTRL_INFO_BYPASS = 0, + OFFSET_SDHCI_MSM_ICE_CTRL_INFO_KEY_INDEX = 1, + OFFSET_SDHCI_MSM_ICE_CTRL_INFO_CDU = 6, + OFFSET_SDHCI_MSM_ICE_HCI_PARAM_CCI = 0, + OFFSET_SDHCI_MSM_ICE_HCI_PARAM_CE = 8, +}; + +/* SDHCI MSM ICE CTRL Info register masks */ +enum { + MASK_SDHCI_MSM_ICE_CTRL_INFO_BYPASS = 0x1, + MASK_SDHCI_MSM_ICE_CTRL_INFO_KEY_INDEX = 0x1F, + MASK_SDHCI_MSM_ICE_CTRL_INFO_CDU = 0x7, + MASK_SDHCI_MSM_ICE_HCI_PARAM_CE = 0x1, + MASK_SDHCI_MSM_ICE_HCI_PARAM_CCI = 0xff +}; + +/* SDHCI MSM ICE encryption/decryption bypass state */ +enum { + SDHCI_MSM_ICE_DISABLE_BYPASS = 0, + SDHCI_MSM_ICE_ENABLE_BYPASS = 1, +}; + +/* SDHCI MSM ICE Crypto Data Unit of target DUN of Transfer Request */ +enum { + SDHCI_MSM_ICE_TR_DATA_UNIT_512_B = 0, + SDHCI_MSM_ICE_TR_DATA_UNIT_1_KB = 1, + SDHCI_MSM_ICE_TR_DATA_UNIT_2_KB = 2, + SDHCI_MSM_ICE_TR_DATA_UNIT_4_KB = 3, + SDHCI_MSM_ICE_TR_DATA_UNIT_8_KB = 4, + SDHCI_MSM_ICE_TR_DATA_UNIT_16_KB = 5, + SDHCI_MSM_ICE_TR_DATA_UNIT_32_KB = 6, + SDHCI_MSM_ICE_TR_DATA_UNIT_64_KB = 7, +}; + +/* SDHCI MSM ICE internal state */ +enum { + SDHCI_MSM_ICE_STATE_DISABLED = 0, + SDHCI_MSM_ICE_STATE_ACTIVE = 1, + SDHCI_MSM_ICE_STATE_SUSPENDED = 2, +}; + +/* crypto context fields in cmdq data command task descriptor */ +#define DATA_UNIT_NUM(x) (((u64)(x) & 0xFFFFFFFF) << 0) +#define CRYPTO_CONFIG_INDEX(x) (((u64)(x) & 0xFF) << 32) +#define CRYPTO_ENABLE(x) (((u64)(x) & 0x1) << 47) + +#ifdef CONFIG_MMC_SDHCI_MSM_ICE +int sdhci_msm_ice_get_dev(struct sdhci_host *host); +int sdhci_msm_ice_init(struct sdhci_host *host); +void sdhci_msm_ice_cfg_reset(struct sdhci_host *host, u32 slot); +int sdhci_msm_ice_cfg(struct sdhci_host *host, struct mmc_request *mrq, + u32 slot); +int sdhci_msm_ice_cmdq_cfg(struct sdhci_host *host, + struct mmc_request *mrq, u32 slot, u64 *ice_ctx); +int sdhci_msm_ice_cfg_end(struct sdhci_host *host, struct mmc_request *mrq); +int sdhci_msm_ice_reset(struct sdhci_host *host); +int sdhci_msm_ice_resume(struct sdhci_host *host); +int sdhci_msm_ice_suspend(struct sdhci_host *host); +int sdhci_msm_ice_get_status(struct sdhci_host *host, int *ice_status); +void sdhci_msm_ice_print_regs(struct sdhci_host *host); +#else +inline int sdhci_msm_ice_get_dev(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + if (msm_host) { + msm_host->ice.pdev = NULL; + msm_host->ice.vops = NULL; + } + return -ENODEV; +} +inline int sdhci_msm_ice_init(struct sdhci_host *host) +{ + return 0; +} + +inline void sdhci_msm_ice_cfg_reset(struct sdhci_host *host, u32 slot) +{ +} + +inline int sdhci_msm_ice_cfg(struct sdhci_host *host, + struct mmc_request *mrq, u32 slot) +{ + return 0; +} +inline int sdhci_msm_ice_cmdq_cfg(struct sdhci_host *host, + struct mmc_request *mrq, u32 slot, u64 *ice_ctx) +{ + return 0; +} +inline int sdhci_msm_ice_cfg_end(struct sdhci_host *host, + struct mmc_request *mrq) +{ + return 0; +} +inline int sdhci_msm_ice_reset(struct sdhci_host *host) +{ + return 0; +} +inline int sdhci_msm_ice_resume(struct sdhci_host *host) +{ + return 0; +} +inline int sdhci_msm_ice_suspend(struct sdhci_host *host) +{ + return 0; +} +inline int sdhci_msm_ice_get_status(struct sdhci_host *host, + int *ice_status) +{ + return 0; +} +inline void sdhci_msm_ice_print_regs(struct sdhci_host *host) +{ + return; +} +#endif /* CONFIG_MMC_SDHCI_MSM_ICE */ +#endif /* __SDHCI_MSM_ICE_H__ */ diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index 4695bee203ea..907763ddf234 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -1,7 +1,8 @@ /* - * drivers/mmc/host/sdhci-msm.c - Qualcomm SDHCI Platform driver + * drivers/mmc/host/sdhci-msm.c - Qualcomm Technologies, Inc. MSM SDHCI Platform + * driver source file * - * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. + * Copyright (c) 2012-2017, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -15,97 +16,497 @@ */ #include <linux/module.h> +#include <linux/mmc/host.h> +#include <linux/mmc/card.h> +#include <linux/mmc/sdio_func.h> +#include <linux/gfp.h> +#include <linux/of.h> #include <linux/of_device.h> +#include <linux/of_gpio.h> +#include <linux/regulator/consumer.h> +#include <linux/types.h> +#include <linux/input.h> +#include <linux/platform_device.h> +#include <linux/wait.h> +#include <linux/io.h> #include <linux/delay.h> -#include <linux/mmc/mmc.h> +#include <linux/scatterlist.h> #include <linux/slab.h> +#include <linux/mmc/slot-gpio.h> +#include <linux/dma-mapping.h> +#include <linux/iopoll.h> +#include <linux/pinctrl/consumer.h> +#include <linux/msm-bus.h> +#include <linux/pm_runtime.h> +#include <trace/events/mmc.h> -#include "sdhci-pltfm.h" +#include "sdhci-msm.h" +#include "sdhci-msm-ice.h" +#include "cmdq_hci.h" -#define CORE_MCI_VERSION 0x50 +#define QOS_REMOVE_DELAY_MS 10 +#define CORE_POWER 0x0 +#define CORE_SW_RST (1 << 7) + +#define SDHCI_VER_100 0x2B + +#define CORE_VERSION_STEP_MASK 0x0000FFFF +#define CORE_VERSION_MINOR_MASK 0x0FFF0000 +#define CORE_VERSION_MINOR_SHIFT 16 +#define CORE_VERSION_MAJOR_MASK 0xF0000000 #define CORE_VERSION_MAJOR_SHIFT 28 -#define CORE_VERSION_MAJOR_MASK (0xf << CORE_VERSION_MAJOR_SHIFT) -#define CORE_VERSION_MINOR_MASK 0xff +#define CORE_VERSION_TARGET_MASK 0x000000FF +#define SDHCI_MSM_VER_420 0x49 + +#define SWITCHABLE_SIGNALLING_VOL (1 << 29) #define CORE_HC_MODE 0x78 #define HC_MODE_EN 0x1 -#define CORE_POWER 0x0 -#define CORE_SW_RST BIT(7) +#define FF_CLK_SW_RST_DIS (1 << 13) + +#define CORE_PWRCTL_BUS_OFF 0x01 +#define CORE_PWRCTL_BUS_ON (1 << 1) +#define CORE_PWRCTL_IO_LOW (1 << 2) +#define CORE_PWRCTL_IO_HIGH (1 << 3) +#define CORE_PWRCTL_BUS_SUCCESS 0x01 +#define CORE_PWRCTL_BUS_FAIL (1 << 1) +#define CORE_PWRCTL_IO_SUCCESS (1 << 2) +#define CORE_PWRCTL_IO_FAIL (1 << 3) + +#define INT_MASK 0xF #define MAX_PHASES 16 -#define CORE_DLL_LOCK BIT(7) -#define CORE_DLL_EN BIT(16) -#define CORE_CDR_EN BIT(17) -#define CORE_CK_OUT_EN BIT(18) -#define CORE_CDR_EXT_EN BIT(19) -#define CORE_DLL_PDN BIT(29) -#define CORE_DLL_RST BIT(30) -#define CORE_DLL_CONFIG 0x100 -#define CORE_DLL_STATUS 0x108 - -#define CORE_VENDOR_SPEC 0x10c -#define CORE_CLK_PWRSAVE BIT(1) - -#define CORE_VENDOR_SPEC_CAPABILITIES0 0x11c - -#define CDR_SELEXT_SHIFT 20 -#define CDR_SELEXT_MASK (0xf << CDR_SELEXT_SHIFT) -#define CMUX_SHIFT_PHASE_SHIFT 24 -#define CMUX_SHIFT_PHASE_MASK (7 << CMUX_SHIFT_PHASE_SHIFT) - -struct sdhci_msm_host { - struct platform_device *pdev; - void __iomem *core_mem; /* MSM SDCC mapped address */ - struct clk *clk; /* main SD/MMC bus clock */ - struct clk *pclk; /* SDHC peripheral bus clock */ - struct clk *bus_clk; /* SDHC bus voter clock */ - struct mmc_host *mmc; - struct sdhci_pltfm_data sdhci_msm_pdata; + +#define CORE_CMD_DAT_TRACK_SEL (1 << 0) +#define CORE_DLL_EN (1 << 16) +#define CORE_CDR_EN (1 << 17) +#define CORE_CK_OUT_EN (1 << 18) +#define CORE_CDR_EXT_EN (1 << 19) +#define CORE_DLL_PDN (1 << 29) +#define CORE_DLL_RST (1 << 30) + +#define CORE_DLL_LOCK (1 << 7) +#define CORE_DDR_DLL_LOCK (1 << 11) + +#define CORE_CLK_PWRSAVE (1 << 1) +#define CORE_HC_MCLK_SEL_DFLT (2 << 8) +#define CORE_HC_MCLK_SEL_HS400 (3 << 8) +#define CORE_HC_MCLK_SEL_MASK (3 << 8) +#define CORE_HC_AUTO_CMD21_EN (1 << 6) +#define CORE_IO_PAD_PWR_SWITCH_EN (1 << 15) +#define CORE_IO_PAD_PWR_SWITCH (1 << 16) +#define CORE_HC_SELECT_IN_EN (1 << 18) +#define CORE_HC_SELECT_IN_HS400 (6 << 19) +#define CORE_HC_SELECT_IN_MASK (7 << 19) +#define CORE_VENDOR_SPEC_POR_VAL 0xA1C + +#define HC_SW_RST_WAIT_IDLE_DIS (1 << 20) +#define HC_SW_RST_REQ (1 << 21) +#define CORE_ONE_MID_EN (1 << 25) + +#define CORE_8_BIT_SUPPORT (1 << 18) +#define CORE_3_3V_SUPPORT (1 << 24) +#define CORE_3_0V_SUPPORT (1 << 25) +#define CORE_1_8V_SUPPORT (1 << 26) +#define CORE_SYS_BUS_SUPPORT_64_BIT BIT(28) + +#define CORE_CSR_CDC_CTLR_CFG0 0x130 +#define CORE_SW_TRIG_FULL_CALIB (1 << 16) +#define CORE_HW_AUTOCAL_ENA (1 << 17) + +#define CORE_CSR_CDC_CTLR_CFG1 0x134 +#define CORE_CSR_CDC_CAL_TIMER_CFG0 0x138 +#define CORE_TIMER_ENA (1 << 16) + +#define CORE_CSR_CDC_CAL_TIMER_CFG1 0x13C +#define CORE_CSR_CDC_REFCOUNT_CFG 0x140 +#define CORE_CSR_CDC_COARSE_CAL_CFG 0x144 +#define CORE_CDC_OFFSET_CFG 0x14C +#define CORE_CSR_CDC_DELAY_CFG 0x150 +#define CORE_CDC_SLAVE_DDA_CFG 0x160 +#define CORE_CSR_CDC_STATUS0 0x164 +#define CORE_CALIBRATION_DONE (1 << 0) + +#define CORE_CDC_ERROR_CODE_MASK 0x7000000 + +#define CQ_CMD_DBG_RAM 0x110 +#define CQ_CMD_DBG_RAM_WA 0x150 +#define CQ_CMD_DBG_RAM_OL 0x154 + +#define CORE_CSR_CDC_GEN_CFG 0x178 +#define CORE_CDC_SWITCH_BYPASS_OFF (1 << 0) +#define CORE_CDC_SWITCH_RC_EN (1 << 1) + +#define CORE_CDC_T4_DLY_SEL (1 << 0) +#define CORE_CMDIN_RCLK_EN (1 << 1) +#define CORE_START_CDC_TRAFFIC (1 << 6) + +#define CORE_PWRSAVE_DLL (1 << 3) +#define CORE_CMDEN_HS400_INPUT_MASK_CNT (1 << 13) + +#define CORE_DDR_CAL_EN (1 << 0) +#define CORE_FLL_CYCLE_CNT (1 << 18) +#define CORE_DLL_CLOCK_DISABLE (1 << 21) + +#define DDR_CONFIG_POR_VAL 0x80040853 +#define DDR_CONFIG_PRG_RCLK_DLY_MASK 0x1FF +#define DDR_CONFIG_PRG_RCLK_DLY 115 +#define DDR_CONFIG_2_POR_VAL 0x80040873 + +/* 512 descriptors */ +#define SDHCI_MSM_MAX_SEGMENTS (1 << 9) +#define SDHCI_MSM_MMC_CLK_GATE_DELAY 200 /* msecs */ + +#define CORE_FREQ_100MHZ (100 * 1000 * 1000) +#define TCXO_FREQ 19200000 + +#define INVALID_TUNING_PHASE -1 +#define sdhci_is_valid_gpio_wakeup_int(_h) ((_h)->pdata->sdiowakeup_irq >= 0) + +#define NUM_TUNING_PHASES 16 +#define MAX_DRV_TYPES_SUPPORTED_HS200 4 +#define MSM_AUTOSUSPEND_DELAY_MS 100 + +struct sdhci_msm_offset { + u32 CORE_MCI_DATA_CNT; + u32 CORE_MCI_STATUS; + u32 CORE_MCI_FIFO_CNT; + u32 CORE_MCI_VERSION; + u32 CORE_GENERICS; + u32 CORE_TESTBUS_CONFIG; + u32 CORE_TESTBUS_SEL2_BIT; + u32 CORE_TESTBUS_ENA; + u32 CORE_TESTBUS_SEL2; + u32 CORE_PWRCTL_STATUS; + u32 CORE_PWRCTL_MASK; + u32 CORE_PWRCTL_CLEAR; + u32 CORE_PWRCTL_CTL; + u32 CORE_SDCC_DEBUG_REG; + u32 CORE_DLL_CONFIG; + u32 CORE_DLL_STATUS; + u32 CORE_VENDOR_SPEC; + u32 CORE_VENDOR_SPEC_ADMA_ERR_ADDR0; + u32 CORE_VENDOR_SPEC_ADMA_ERR_ADDR1; + u32 CORE_VENDOR_SPEC_FUNC2; + u32 CORE_VENDOR_SPEC_CAPABILITIES0; + u32 CORE_DDR_200_CFG; + u32 CORE_VENDOR_SPEC3; + u32 CORE_DLL_CONFIG_2; + u32 CORE_DDR_CONFIG; + u32 CORE_DDR_CONFIG_2; +}; + +struct sdhci_msm_offset sdhci_msm_offset_mci_removed = { + .CORE_MCI_DATA_CNT = 0x35C, + .CORE_MCI_STATUS = 0x324, + .CORE_MCI_FIFO_CNT = 0x308, + .CORE_MCI_VERSION = 0x318, + .CORE_GENERICS = 0x320, + .CORE_TESTBUS_CONFIG = 0x32C, + .CORE_TESTBUS_SEL2_BIT = 3, + .CORE_TESTBUS_ENA = (1 << 31), + .CORE_TESTBUS_SEL2 = (1 << 3), + .CORE_PWRCTL_STATUS = 0x240, + .CORE_PWRCTL_MASK = 0x244, + .CORE_PWRCTL_CLEAR = 0x248, + .CORE_PWRCTL_CTL = 0x24C, + .CORE_SDCC_DEBUG_REG = 0x358, + .CORE_DLL_CONFIG = 0x200, + .CORE_DLL_STATUS = 0x208, + .CORE_VENDOR_SPEC = 0x20C, + .CORE_VENDOR_SPEC_ADMA_ERR_ADDR0 = 0x214, + .CORE_VENDOR_SPEC_ADMA_ERR_ADDR1 = 0x218, + .CORE_VENDOR_SPEC_FUNC2 = 0x210, + .CORE_VENDOR_SPEC_CAPABILITIES0 = 0x21C, + .CORE_DDR_200_CFG = 0x224, + .CORE_VENDOR_SPEC3 = 0x250, + .CORE_DLL_CONFIG_2 = 0x254, + .CORE_DDR_CONFIG = 0x258, + .CORE_DDR_CONFIG_2 = 0x25C, }; -/* Platform specific tuning */ -static inline int msm_dll_poll_ck_out_en(struct sdhci_host *host, u8 poll) +struct sdhci_msm_offset sdhci_msm_offset_mci_present = { + .CORE_MCI_DATA_CNT = 0x30, + .CORE_MCI_STATUS = 0x34, + .CORE_MCI_FIFO_CNT = 0x44, + .CORE_MCI_VERSION = 0x050, + .CORE_GENERICS = 0x70, + .CORE_TESTBUS_CONFIG = 0x0CC, + .CORE_TESTBUS_SEL2_BIT = 4, + .CORE_TESTBUS_ENA = (1 << 3), + .CORE_TESTBUS_SEL2 = (1 << 4), + .CORE_PWRCTL_STATUS = 0xDC, + .CORE_PWRCTL_MASK = 0xE0, + .CORE_PWRCTL_CLEAR = 0xE4, + .CORE_PWRCTL_CTL = 0xE8, + .CORE_SDCC_DEBUG_REG = 0x124, + .CORE_DLL_CONFIG = 0x100, + .CORE_DLL_STATUS = 0x108, + .CORE_VENDOR_SPEC = 0x10C, + .CORE_VENDOR_SPEC_ADMA_ERR_ADDR0 = 0x114, + .CORE_VENDOR_SPEC_ADMA_ERR_ADDR1 = 0x118, + .CORE_VENDOR_SPEC_FUNC2 = 0x110, + .CORE_VENDOR_SPEC_CAPABILITIES0 = 0x11C, + .CORE_DDR_200_CFG = 0x184, + .CORE_VENDOR_SPEC3 = 0x1B0, + .CORE_DLL_CONFIG_2 = 0x1B4, + .CORE_DDR_CONFIG = 0x1B8, + .CORE_DDR_CONFIG_2 = 0x1BC, +}; + +u8 sdhci_msm_readb_relaxed(struct sdhci_host *host, u32 offset) { + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + void __iomem *base_addr; + + if (msm_host->mci_removed) + base_addr = host->ioaddr; + else + base_addr = msm_host->core_mem; + + return readb_relaxed(base_addr + offset); +} + +u32 sdhci_msm_readl_relaxed(struct sdhci_host *host, u32 offset) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + void __iomem *base_addr; + + if (msm_host->mci_removed) + base_addr = host->ioaddr; + else + base_addr = msm_host->core_mem; + + return readl_relaxed(base_addr + offset); +} + +void sdhci_msm_writeb_relaxed(u8 val, struct sdhci_host *host, u32 offset) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + void __iomem *base_addr; + + if (msm_host->mci_removed) + base_addr = host->ioaddr; + else + base_addr = msm_host->core_mem; + + writeb_relaxed(val, base_addr + offset); +} + +void sdhci_msm_writel_relaxed(u32 val, struct sdhci_host *host, u32 offset) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + void __iomem *base_addr; + + if (msm_host->mci_removed) + base_addr = host->ioaddr; + else + base_addr = msm_host->core_mem; + + writel_relaxed(val, base_addr + offset); +} + +/* Timeout value to avoid infinite waiting for pwr_irq */ +#define MSM_PWR_IRQ_TIMEOUT_MS 5000 + +static const u32 tuning_block_64[] = { + 0x00FF0FFF, 0xCCC3CCFF, 0xFFCC3CC3, 0xEFFEFFFE, + 0xDDFFDFFF, 0xFBFFFBFF, 0xFF7FFFBF, 0xEFBDF777, + 0xF0FFF0FF, 0x3CCCFC0F, 0xCFCC33CC, 0xEEFFEFFF, + 0xFDFFFDFF, 0xFFBFFFDF, 0xFFF7FFBB, 0xDE7B7FF7 +}; + +static const u32 tuning_block_128[] = { + 0xFF00FFFF, 0x0000FFFF, 0xCCCCFFFF, 0xCCCC33CC, + 0xCC3333CC, 0xFFFFCCCC, 0xFFFFEEFF, 0xFFEEEEFF, + 0xFFDDFFFF, 0xDDDDFFFF, 0xBBFFFFFF, 0xBBFFFFFF, + 0xFFFFFFBB, 0xFFFFFF77, 0x77FF7777, 0xFFEEDDBB, + 0x00FFFFFF, 0x00FFFFFF, 0xCCFFFF00, 0xCC33CCCC, + 0x3333CCCC, 0xFFCCCCCC, 0xFFEEFFFF, 0xEEEEFFFF, + 0xDDFFFFFF, 0xDDFFFFFF, 0xFFFFFFDD, 0xFFFFFFBB, + 0xFFFFBBBB, 0xFFFF77FF, 0xFF7777FF, 0xEEDDBB77 +}; + +/* global to hold each slot instance for debug */ +static struct sdhci_msm_host *sdhci_slot[2]; + +static int disable_slots; +/* root can write, others read */ +module_param(disable_slots, int, S_IRUGO|S_IWUSR); + +static bool nocmdq; +module_param(nocmdq, bool, S_IRUGO|S_IWUSR); + +enum vdd_io_level { + /* set vdd_io_data->low_vol_level */ + VDD_IO_LOW, + /* set vdd_io_data->high_vol_level */ + VDD_IO_HIGH, + /* + * set whatever there in voltage_level (third argument) of + * sdhci_msm_set_vdd_io_vol() function. + */ + VDD_IO_SET_LEVEL, +}; + +/* MSM platform specific tuning */ +static inline int msm_dll_poll_ck_out_en(struct sdhci_host *host, + u8 poll) +{ + int rc = 0; u32 wait_cnt = 50; - u8 ck_out_en; + u8 ck_out_en = 0; struct mmc_host *mmc = host->mmc; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; - /* Poll for CK_OUT_EN bit. max. poll time = 50us */ - ck_out_en = !!(readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) & - CORE_CK_OUT_EN); + /* poll for CK_OUT_EN bit. max. poll time = 50us */ + ck_out_en = !!(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) & CORE_CK_OUT_EN); while (ck_out_en != poll) { if (--wait_cnt == 0) { - dev_err(mmc_dev(mmc), "%s: CK_OUT_EN bit is not %d\n", - mmc_hostname(mmc), poll); - return -ETIMEDOUT; + pr_err("%s: %s: CK_OUT_EN bit is not %d\n", + mmc_hostname(mmc), __func__, poll); + rc = -ETIMEDOUT; + goto out; } udelay(1); - ck_out_en = !!(readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) & - CORE_CK_OUT_EN); + ck_out_en = !!(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) & CORE_CK_OUT_EN); } +out: + return rc; +} - return 0; +/* + * Enable CDR to track changes of DAT lines and adjust sampling + * point according to voltage/temperature variations + */ +static int msm_enable_cdr_cm_sdc4_dll(struct sdhci_host *host) +{ + int rc = 0; + u32 config; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + config = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + config |= CORE_CDR_EN; + config &= ~(CORE_CDR_EXT_EN | CORE_CK_OUT_EN); + writel_relaxed(config, host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + + rc = msm_dll_poll_ck_out_en(host, 0); + if (rc) + goto err; + + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) | CORE_CK_OUT_EN), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); + + rc = msm_dll_poll_ck_out_en(host, 1); + if (rc) + goto err; + goto out; +err: + pr_err("%s: %s: failed\n", mmc_hostname(host->mmc), __func__); +out: + return rc; +} + +static ssize_t store_auto_cmd21(struct device *dev, struct device_attribute + *attr, const char *buf, size_t count) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + u32 tmp; + unsigned long flags; + + if (!kstrtou32(buf, 0, &tmp)) { + spin_lock_irqsave(&host->lock, flags); + msm_host->en_auto_cmd21 = !!tmp; + spin_unlock_irqrestore(&host->lock, flags); + } + return count; +} + +static ssize_t show_auto_cmd21(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + return snprintf(buf, PAGE_SIZE, "%d\n", msm_host->en_auto_cmd21); +} + +/* MSM auto-tuning handler */ +static int sdhci_msm_config_auto_tuning_cmd(struct sdhci_host *host, + bool enable, + u32 type) +{ + int rc = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + u32 val = 0; + + if (!msm_host->en_auto_cmd21) + return 0; + + if (type == MMC_SEND_TUNING_BLOCK_HS200) + val = CORE_HC_AUTO_CMD21_EN; + else + return 0; + + if (enable) { + rc = msm_enable_cdr_cm_sdc4_dll(host); + writel_relaxed(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) | val, + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC); + } else { + writel_relaxed(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) & ~val, + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC); + } + return rc; } static int msm_config_cm_dll_phase(struct sdhci_host *host, u8 phase) { - int rc; - static const u8 grey_coded_phase_table[] = { - 0x0, 0x1, 0x3, 0x2, 0x6, 0x7, 0x5, 0x4, - 0xc, 0xd, 0xf, 0xe, 0xa, 0xb, 0x9, 0x8 - }; + int rc = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + u8 grey_coded_phase_table[] = {0x0, 0x1, 0x3, 0x2, 0x6, 0x7, 0x5, 0x4, + 0xC, 0xD, 0xF, 0xE, 0xA, 0xB, 0x9, + 0x8}; unsigned long flags; u32 config; struct mmc_host *mmc = host->mmc; + pr_debug("%s: Enter %s\n", mmc_hostname(mmc), __func__); spin_lock_irqsave(&host->lock, flags); - config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG); + config = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); config &= ~(CORE_CDR_EN | CORE_CK_OUT_EN); config |= (CORE_CDR_EXT_EN | CORE_DLL_EN); - writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed(config, host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); /* Wait until CK_OUT_EN bit of DLL_CONFIG register becomes '0' */ rc = msm_dll_poll_ck_out_en(host, 0); @@ -116,31 +517,36 @@ static int msm_config_cm_dll_phase(struct sdhci_host *host, u8 phase) * Write the selected DLL clock output phase (0 ... 15) * to CDR_SELEXT bit field of DLL_CONFIG register. */ - config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG); - config &= ~CDR_SELEXT_MASK; - config |= grey_coded_phase_table[phase] << CDR_SELEXT_SHIFT; - writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed(((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + & ~(0xF << 20)) + | (grey_coded_phase_table[phase] << 20)), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); /* Set CK_OUT_EN bit of DLL_CONFIG register to 1. */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - | CORE_CK_OUT_EN), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) | CORE_CK_OUT_EN), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); /* Wait until CK_OUT_EN bit of DLL_CONFIG register becomes '1' */ rc = msm_dll_poll_ck_out_en(host, 1); if (rc) goto err_out; - config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG); + config = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); config |= CORE_CDR_EN; config &= ~CORE_CDR_EXT_EN; - writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed(config, host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); goto out; err_out: - dev_err(mmc_dev(mmc), "%s: Failed to set DLL phase: %d\n", - mmc_hostname(mmc), phase); + pr_err("%s: %s: Failed to set DLL phase: %d\n", + mmc_hostname(mmc), __func__, phase); out: spin_unlock_irqrestore(&host->lock, flags); + pr_debug("%s: Exit %s\n", mmc_hostname(mmc), __func__); return rc; } @@ -148,26 +554,27 @@ out: * Find out the greatest range of consecuitive selected * DLL clock output phases that can be used as sampling * setting for SD3.0 UHS-I card read operation (in SDR104 - * timing mode) or for eMMC4.5 card read operation (in HS200 - * timing mode). + * timing mode) or for eMMC4.5 card read operation (in + * HS400/HS200 timing mode). * Select the 3/4 of the range and configure the DLL with the * selected DLL clock output phase. */ static int msm_find_most_appropriate_phase(struct sdhci_host *host, - u8 *phase_table, u8 total_phases) + u8 *phase_table, u8 total_phases) { int ret; u8 ranges[MAX_PHASES][MAX_PHASES] = { {0}, {0} }; - u8 phases_per_row[MAX_PHASES] = { 0 }; + u8 phases_per_row[MAX_PHASES] = {0}; int row_index = 0, col_index = 0, selected_row_index = 0, curr_max = 0; int i, cnt, phase_0_raw_index = 0, phase_15_raw_index = 0; bool phase_0_found = false, phase_15_found = false; struct mmc_host *mmc = host->mmc; + pr_debug("%s: Enter %s\n", mmc_hostname(mmc), __func__); if (!total_phases || (total_phases > MAX_PHASES)) { - dev_err(mmc_dev(mmc), "%s: Invalid argument: total_phases=%d\n", - mmc_hostname(mmc), total_phases); + pr_err("%s: %s: invalid argument: total_phases=%d\n", + mmc_hostname(mmc), __func__, total_phases); return -EINVAL; } @@ -225,7 +632,7 @@ static int msm_find_most_appropriate_phase(struct sdhci_host *host, i = phases_15; for (cnt = 0; cnt < phases_0; cnt++) { ranges[phase_15_raw_index][i] = - ranges[phase_0_raw_index][cnt]; + ranges[phase_0_raw_index][cnt]; if (++i >= MAX_PHASES) break; } @@ -241,24 +648,29 @@ static int msm_find_most_appropriate_phase(struct sdhci_host *host, } } - i = (curr_max * 3) / 4; + i = ((curr_max * 3) / 4); if (i) i--; - ret = ranges[selected_row_index][i]; + ret = (int)ranges[selected_row_index][i]; if (ret >= MAX_PHASES) { ret = -EINVAL; - dev_err(mmc_dev(mmc), "%s: Invalid phase selected=%d\n", - mmc_hostname(mmc), ret); + pr_err("%s: %s: invalid phase selected=%d\n", + mmc_hostname(mmc), __func__, ret); } + pr_debug("%s: Exit %s\n", mmc_hostname(mmc), __func__); return ret; } static inline void msm_cm_dll_set_freq(struct sdhci_host *host) { - u32 mclk_freq = 0, config; + u32 mclk_freq = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; /* Program the MCLK value to MCLK_FREQ bit field */ if (host->clock <= 112000000) @@ -278,117 +690,639 @@ static inline void msm_cm_dll_set_freq(struct sdhci_host *host) else if (host->clock <= 200000000) mclk_freq = 7; - config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG); - config &= ~CMUX_SHIFT_PHASE_MASK; - config |= mclk_freq << CMUX_SHIFT_PHASE_SHIFT; - writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed(((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + & ~(7 << 24)) | (mclk_freq << 24)), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); } -/* Initialize the DLL (Programmable Delay Line) */ +/* Initialize the DLL (Programmable Delay Line ) */ static int msm_init_cm_dll(struct sdhci_host *host) { + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; struct mmc_host *mmc = host->mmc; - int wait_cnt = 50; + int rc = 0; unsigned long flags; + u32 wait_cnt; + bool prev_pwrsave, curr_pwrsave; + pr_debug("%s: Enter %s\n", mmc_hostname(mmc), __func__); spin_lock_irqsave(&host->lock, flags); - + prev_pwrsave = !!(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) & CORE_CLK_PWRSAVE); + curr_pwrsave = prev_pwrsave; /* * Make sure that clock is always enabled when DLL * tuning is in progress. Keeping PWRSAVE ON may - * turn off the clock. + * turn off the clock. So let's disable the PWRSAVE + * here and re-enable it once tuning is completed. */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC) - & ~CORE_CLK_PWRSAVE), host->ioaddr + CORE_VENDOR_SPEC); + if (prev_pwrsave) { + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) + & ~CORE_CLK_PWRSAVE), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + curr_pwrsave = false; + } + + if (msm_host->use_updated_dll_reset) { + /* Disable the DLL clock */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + & ~CORE_CK_OUT_EN), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2) + | CORE_DLL_CLOCK_DISABLE), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2); + } /* Write 1 to DLL_RST bit of DLL_CONFIG register */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - | CORE_DLL_RST), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) | CORE_DLL_RST), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); /* Write 1 to DLL_PDN bit of DLL_CONFIG register */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - | CORE_DLL_PDN), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) | CORE_DLL_PDN), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); msm_cm_dll_set_freq(host); + if (msm_host->use_updated_dll_reset) { + u32 mclk_freq = 0; + + if ((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2) + & CORE_FLL_CYCLE_CNT)) + mclk_freq = (u32) ((host->clock / TCXO_FREQ) * 8); + else + mclk_freq = (u32) ((host->clock / TCXO_FREQ) * 4); + + writel_relaxed(((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2) + & ~(0xFF << 10)) | (mclk_freq << 10)), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG_2); + /* wait for 5us before enabling DLL clock */ + udelay(5); + } + /* Write 0 to DLL_RST bit of DLL_CONFIG register */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - & ~CORE_DLL_RST), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) & ~CORE_DLL_RST), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); /* Write 0 to DLL_PDN bit of DLL_CONFIG register */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - & ~CORE_DLL_PDN), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) & ~CORE_DLL_PDN), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); + + if (msm_host->use_updated_dll_reset) { + msm_cm_dll_set_freq(host); + /* Enable the DLL clock */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2) + & ~CORE_DLL_CLOCK_DISABLE), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2); + } /* Set DLL_EN bit to 1. */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - | CORE_DLL_EN), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) | CORE_DLL_EN), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG); /* Set CK_OUT_EN bit to 1. */ - writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG) - | CORE_CK_OUT_EN), host->ioaddr + CORE_DLL_CONFIG); + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + | CORE_CK_OUT_EN), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + wait_cnt = 50; /* Wait until DLL_LOCK bit of DLL_STATUS register becomes '1' */ - while (!(readl_relaxed(host->ioaddr + CORE_DLL_STATUS) & - CORE_DLL_LOCK)) { + while (!(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_STATUS) & CORE_DLL_LOCK)) { /* max. wait for 50us sec for LOCK bit to be set */ if (--wait_cnt == 0) { - dev_err(mmc_dev(mmc), "%s: DLL failed to LOCK\n", - mmc_hostname(mmc)); - spin_unlock_irqrestore(&host->lock, flags); - return -ETIMEDOUT; + pr_err("%s: %s: DLL failed to LOCK\n", + mmc_hostname(mmc), __func__); + rc = -ETIMEDOUT; + goto out; } + /* wait for 1us before polling again */ udelay(1); } +out: + /* Restore the correct PWRSAVE state */ + if (prev_pwrsave ^ curr_pwrsave) { + u32 reg = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + + if (prev_pwrsave) + reg |= CORE_CLK_PWRSAVE; + else + reg &= ~CORE_CLK_PWRSAVE; + + writel_relaxed(reg, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + } + spin_unlock_irqrestore(&host->lock, flags); - return 0; + pr_debug("%s: Exit %s\n", mmc_hostname(mmc), __func__); + return rc; } -static int sdhci_msm_execute_tuning(struct sdhci_host *host, u32 opcode) +static int sdhci_msm_cdclp533_calibration(struct sdhci_host *host) { + u32 calib_done; + int ret = 0; + int cdc_err = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + pr_debug("%s: Enter %s\n", mmc_hostname(host->mmc), __func__); + + /* Write 0 to CDC_T4_DLY_SEL field in VENDOR_SPEC_DDR200_CFG */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DDR_200_CFG) + & ~CORE_CDC_T4_DLY_SEL), + host->ioaddr + msm_host_offset->CORE_DDR_200_CFG); + + /* Write 0 to CDC_SWITCH_BYPASS_OFF field in CORE_CSR_CDC_GEN_CFG */ + writel_relaxed((readl_relaxed(host->ioaddr + CORE_CSR_CDC_GEN_CFG) + & ~CORE_CDC_SWITCH_BYPASS_OFF), + host->ioaddr + CORE_CSR_CDC_GEN_CFG); + + /* Write 1 to CDC_SWITCH_RC_EN field in CORE_CSR_CDC_GEN_CFG */ + writel_relaxed((readl_relaxed(host->ioaddr + CORE_CSR_CDC_GEN_CFG) + | CORE_CDC_SWITCH_RC_EN), + host->ioaddr + CORE_CSR_CDC_GEN_CFG); + + /* Write 0 to START_CDC_TRAFFIC field in CORE_DDR200_CFG */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DDR_200_CFG) + & ~CORE_START_CDC_TRAFFIC), + host->ioaddr + msm_host_offset->CORE_DDR_200_CFG); + + /* + * Perform CDC Register Initialization Sequence + * + * CORE_CSR_CDC_CTLR_CFG0 0x11800EC + * CORE_CSR_CDC_CTLR_CFG1 0x3011111 + * CORE_CSR_CDC_CAL_TIMER_CFG0 0x1201000 + * CORE_CSR_CDC_CAL_TIMER_CFG1 0x4 + * CORE_CSR_CDC_REFCOUNT_CFG 0xCB732020 + * CORE_CSR_CDC_COARSE_CAL_CFG 0xB19 + * CORE_CSR_CDC_DELAY_CFG 0x3AC + * CORE_CDC_OFFSET_CFG 0x0 + * CORE_CDC_SLAVE_DDA_CFG 0x16334 + */ + + writel_relaxed(0x11800EC, host->ioaddr + CORE_CSR_CDC_CTLR_CFG0); + writel_relaxed(0x3011111, host->ioaddr + CORE_CSR_CDC_CTLR_CFG1); + writel_relaxed(0x1201000, host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG0); + writel_relaxed(0x4, host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG1); + writel_relaxed(0xCB732020, host->ioaddr + CORE_CSR_CDC_REFCOUNT_CFG); + writel_relaxed(0xB19, host->ioaddr + CORE_CSR_CDC_COARSE_CAL_CFG); + writel_relaxed(0x4E2, host->ioaddr + CORE_CSR_CDC_DELAY_CFG); + writel_relaxed(0x0, host->ioaddr + CORE_CDC_OFFSET_CFG); + writel_relaxed(0x16334, host->ioaddr + CORE_CDC_SLAVE_DDA_CFG); + + /* CDC HW Calibration */ + + /* Write 1 to SW_TRIG_FULL_CALIB field in CORE_CSR_CDC_CTLR_CFG0 */ + writel_relaxed((readl_relaxed(host->ioaddr + CORE_CSR_CDC_CTLR_CFG0) + | CORE_SW_TRIG_FULL_CALIB), + host->ioaddr + CORE_CSR_CDC_CTLR_CFG0); + + /* Write 0 to SW_TRIG_FULL_CALIB field in CORE_CSR_CDC_CTLR_CFG0 */ + writel_relaxed((readl_relaxed(host->ioaddr + CORE_CSR_CDC_CTLR_CFG0) + & ~CORE_SW_TRIG_FULL_CALIB), + host->ioaddr + CORE_CSR_CDC_CTLR_CFG0); + + /* Write 1 to HW_AUTOCAL_ENA field in CORE_CSR_CDC_CTLR_CFG0 */ + writel_relaxed((readl_relaxed(host->ioaddr + CORE_CSR_CDC_CTLR_CFG0) + | CORE_HW_AUTOCAL_ENA), + host->ioaddr + CORE_CSR_CDC_CTLR_CFG0); + + /* Write 1 to TIMER_ENA field in CORE_CSR_CDC_CAL_TIMER_CFG0 */ + writel_relaxed((readl_relaxed(host->ioaddr + + CORE_CSR_CDC_CAL_TIMER_CFG0) | CORE_TIMER_ENA), + host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG0); + + mb(); + + /* Poll on CALIBRATION_DONE field in CORE_CSR_CDC_STATUS0 to be 1 */ + ret = readl_poll_timeout(host->ioaddr + CORE_CSR_CDC_STATUS0, + calib_done, (calib_done & CORE_CALIBRATION_DONE), 1, 50); + + if (ret == -ETIMEDOUT) { + pr_err("%s: %s: CDC Calibration was not completed\n", + mmc_hostname(host->mmc), __func__); + goto out; + } + + /* Verify CDC_ERROR_CODE field in CORE_CSR_CDC_STATUS0 is 0 */ + cdc_err = readl_relaxed(host->ioaddr + CORE_CSR_CDC_STATUS0) + & CORE_CDC_ERROR_CODE_MASK; + if (cdc_err) { + pr_err("%s: %s: CDC Error Code %d\n", + mmc_hostname(host->mmc), __func__, cdc_err); + ret = -EINVAL; + goto out; + } + + /* Write 1 to START_CDC_TRAFFIC field in CORE_DDR200_CFG */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DDR_200_CFG) + | CORE_START_CDC_TRAFFIC), + host->ioaddr + msm_host_offset->CORE_DDR_200_CFG); +out: + pr_debug("%s: Exit %s, ret:%d\n", mmc_hostname(host->mmc), + __func__, ret); + return ret; +} + +static int sdhci_msm_cm_dll_sdc4_calibration(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + u32 dll_status, ddr_config; + int ret = 0; + + pr_debug("%s: Enter %s\n", mmc_hostname(host->mmc), __func__); + + /* + * Reprogramming the value in case it might have been modified by + * bootloaders. + */ + if (msm_host->rclk_delay_fix) { + writel_relaxed(DDR_CONFIG_2_POR_VAL, host->ioaddr + + msm_host_offset->CORE_DDR_CONFIG_2); + } else { + ddr_config = DDR_CONFIG_POR_VAL & + ~DDR_CONFIG_PRG_RCLK_DLY_MASK; + ddr_config |= DDR_CONFIG_PRG_RCLK_DLY; + writel_relaxed(ddr_config, host->ioaddr + + msm_host_offset->CORE_DDR_CONFIG); + } + + if (msm_host->enhanced_strobe && mmc_card_strobe(msm_host->mmc->card)) + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DDR_200_CFG) + | CORE_CMDIN_RCLK_EN), host->ioaddr + + msm_host_offset->CORE_DDR_200_CFG); + + /* Write 1 to DDR_CAL_EN field in CORE_DLL_CONFIG_2 */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG_2) + | CORE_DDR_CAL_EN), + host->ioaddr + msm_host_offset->CORE_DLL_CONFIG_2); + + /* Poll on DDR_DLL_LOCK bit in CORE_DLL_STATUS to be set */ + ret = readl_poll_timeout(host->ioaddr + + msm_host_offset->CORE_DLL_STATUS, + dll_status, (dll_status & CORE_DDR_DLL_LOCK), 10, 1000); + + if (ret == -ETIMEDOUT) { + pr_err("%s: %s: CM_DLL_SDC4 Calibration was not completed\n", + mmc_hostname(host->mmc), __func__); + goto out; + } + + /* + * set CORE_PWRSAVE_DLL bit in CORE_VENDOR_SPEC3. + * when MCLK is gated OFF, it is not gated for less than 0.5us + * and MCLK must be switched on for at-least 1us before DATA + * starts coming. Controllers with 14lpp tech DLL cannot + * guarantee above requirement. So PWRSAVE_DLL should not be + * turned on for host controllers using this DLL. + */ + if (!msm_host->use_14lpp_dll) + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC3) + | CORE_PWRSAVE_DLL), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC3); + mb(); +out: + pr_debug("%s: Exit %s, ret:%d\n", mmc_hostname(host->mmc), + __func__, ret); + return ret; +} + +static int sdhci_msm_enhanced_strobe(struct sdhci_host *host) +{ + int ret = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct mmc_host *mmc = host->mmc; + + pr_debug("%s: Enter %s\n", mmc_hostname(host->mmc), __func__); + + if (!msm_host->enhanced_strobe || !mmc_card_strobe(mmc->card)) { + pr_debug("%s: host/card does not support hs400 enhanced strobe\n", + mmc_hostname(mmc)); + return -EINVAL; + } + + if (msm_host->calibration_done || + !(mmc->ios.timing == MMC_TIMING_MMC_HS400)) { + return 0; + } + + /* + * Reset the tuning block. + */ + ret = msm_init_cm_dll(host); + if (ret) + goto out; + + ret = sdhci_msm_cm_dll_sdc4_calibration(host); +out: + if (!ret) + msm_host->calibration_done = true; + pr_debug("%s: Exit %s, ret:%d\n", mmc_hostname(host->mmc), + __func__, ret); + return ret; +} + +static int sdhci_msm_hs400_dll_calibration(struct sdhci_host *host) +{ + int ret = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + pr_debug("%s: Enter %s\n", mmc_hostname(host->mmc), __func__); + + /* + * Retuning in HS400 (DDR mode) will fail, just reset the + * tuning block and restore the saved tuning phase. + */ + ret = msm_init_cm_dll(host); + if (ret) + goto out; + + /* Set the selected phase in delay line hw block */ + ret = msm_config_cm_dll_phase(host, msm_host->saved_tuning_phase); + if (ret) + goto out; + + /* Write 1 to CMD_DAT_TRACK_SEL field in DLL_CONFIG */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + | CORE_CMD_DAT_TRACK_SEL), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + + if (msm_host->use_cdclp533) + /* Calibrate CDCLP533 DLL HW */ + ret = sdhci_msm_cdclp533_calibration(host); + else + /* Calibrate CM_DLL_SDC4 HW */ + ret = sdhci_msm_cm_dll_sdc4_calibration(host); +out: + pr_debug("%s: Exit %s, ret:%d\n", mmc_hostname(host->mmc), + __func__, ret); + return ret; +} + +static void sdhci_msm_set_mmc_drv_type(struct sdhci_host *host, u32 opcode, + u8 drv_type) +{ + struct mmc_command cmd = {0}; + struct mmc_request mrq = {NULL}; + struct mmc_host *mmc = host->mmc; + u8 val = ((drv_type << 4) | 2); + + cmd.opcode = MMC_SWITCH; + cmd.arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) | + (EXT_CSD_HS_TIMING << 16) | + (val << 8) | + EXT_CSD_CMD_SET_NORMAL; + cmd.flags = MMC_CMD_AC | MMC_RSP_R1B; + /* 1 sec */ + cmd.busy_timeout = 1000 * 1000; + + memset(cmd.resp, 0, sizeof(cmd.resp)); + cmd.retries = 3; + + mrq.cmd = &cmd; + cmd.data = NULL; + + mmc_wait_for_req(mmc, &mrq); + pr_debug("%s: %s: set card drive type to %d\n", + mmc_hostname(mmc), __func__, + drv_type); +} + +int sdhci_msm_execute_tuning(struct sdhci_host *host, u32 opcode) +{ + unsigned long flags; int tuning_seq_cnt = 3; - u8 phase, tuned_phases[16], tuned_phase_cnt = 0; + u8 phase, *data_buf, tuned_phases[NUM_TUNING_PHASES], tuned_phase_cnt; + const u32 *tuning_block_pattern = tuning_block_64; + int size = sizeof(tuning_block_64); /* Tuning pattern size in bytes */ int rc; struct mmc_host *mmc = host->mmc; - struct mmc_ios ios = host->mmc->ios; + struct mmc_ios ios = host->mmc->ios; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + u8 drv_type = 0; + bool drv_type_changed = false; + struct mmc_card *card = host->mmc->card; + int sts_retry; + u8 last_good_phase = 0; /* * Tuning is required for SDR104, HS200 and HS400 cards and * if clock frequency is greater than 100MHz in these modes. */ - if (host->clock <= 100 * 1000 * 1000 || - !((ios.timing == MMC_TIMING_MMC_HS200) || - (ios.timing == MMC_TIMING_UHS_SDR104))) + if (host->clock <= CORE_FREQ_100MHZ || + !((ios.timing == MMC_TIMING_MMC_HS400) || + (ios.timing == MMC_TIMING_MMC_HS200) || + (ios.timing == MMC_TIMING_UHS_SDR104))) + return 0; + + /* + * Don't allow re-tuning for CRC errors observed for any commands + * that are sent during tuning sequence itself. + */ + if (msm_host->tuning_in_progress) return 0; + msm_host->tuning_in_progress = true; + pr_debug("%s: Enter %s\n", mmc_hostname(mmc), __func__); + + /* CDC/SDC4 DLL HW calibration is only required for HS400 mode*/ + if (msm_host->tuning_done && !msm_host->calibration_done && + (mmc->ios.timing == MMC_TIMING_MMC_HS400)) { + rc = sdhci_msm_hs400_dll_calibration(host); + spin_lock_irqsave(&host->lock, flags); + if (!rc) + msm_host->calibration_done = true; + spin_unlock_irqrestore(&host->lock, flags); + goto out; + } + + spin_lock_irqsave(&host->lock, flags); + + if ((opcode == MMC_SEND_TUNING_BLOCK_HS200) && + (mmc->ios.bus_width == MMC_BUS_WIDTH_8)) { + tuning_block_pattern = tuning_block_128; + size = sizeof(tuning_block_128); + } + spin_unlock_irqrestore(&host->lock, flags); + + data_buf = kmalloc(size, GFP_KERNEL); + if (!data_buf) { + rc = -ENOMEM; + goto out; + } retry: - /* First of all reset the tuning block */ + tuned_phase_cnt = 0; + + /* first of all reset the tuning block */ rc = msm_init_cm_dll(host); if (rc) - return rc; + goto kfree; phase = 0; do { - /* Set the phase in delay line hw block */ + struct mmc_command cmd = {0}; + struct mmc_data data = {0}; + struct mmc_request mrq = { + .cmd = &cmd, + .data = &data + }; + struct scatterlist sg; + struct mmc_command sts_cmd = {0}; + + /* set the phase in delay line hw block */ rc = msm_config_cm_dll_phase(host, phase); if (rc) - return rc; + goto kfree; - rc = mmc_send_tuning(mmc, opcode, NULL); - if (!rc) { - /* Tuning is successful at this tuning point */ + cmd.opcode = opcode; + cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; + + data.blksz = size; + data.blocks = 1; + data.flags = MMC_DATA_READ; + data.timeout_ns = 1000 * 1000 * 1000; /* 1 sec */ + + data.sg = &sg; + data.sg_len = 1; + sg_init_one(&sg, data_buf, size); + memset(data_buf, 0, size); + mmc_wait_for_req(mmc, &mrq); + + if (card && (cmd.error || data.error)) { + /* + * Set the dll to last known good phase while sending + * status command to ensure that status command won't + * fail due to bad phase. + */ + if (tuned_phase_cnt) + last_good_phase = + tuned_phases[tuned_phase_cnt-1]; + else if (msm_host->saved_tuning_phase != + INVALID_TUNING_PHASE) + last_good_phase = msm_host->saved_tuning_phase; + + rc = msm_config_cm_dll_phase(host, last_good_phase); + if (rc) + goto kfree; + + sts_cmd.opcode = MMC_SEND_STATUS; + sts_cmd.arg = card->rca << 16; + sts_cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; + sts_retry = 5; + while (sts_retry) { + mmc_wait_for_cmd(mmc, &sts_cmd, 0); + + if (sts_cmd.error || + (R1_CURRENT_STATE(sts_cmd.resp[0]) + != R1_STATE_TRAN)) { + sts_retry--; + /* + * wait for at least 146 MCLK cycles for + * the card to move to TRANS state. As + * the MCLK would be min 200MHz for + * tuning, we need max 0.73us delay. To + * be on safer side 1ms delay is given. + */ + usleep_range(1000, 1200); + pr_debug("%s: phase %d sts cmd err %d resp 0x%x\n", + mmc_hostname(mmc), phase, + sts_cmd.error, sts_cmd.resp[0]); + continue; + } + break; + }; + } + + if (!cmd.error && !data.error && + !memcmp(data_buf, tuning_block_pattern, size)) { + /* tuning is successful at this tuning point */ tuned_phases[tuned_phase_cnt++] = phase; - dev_dbg(mmc_dev(mmc), "%s: Found good phase = %d\n", - mmc_hostname(mmc), phase); + pr_debug("%s: %s: found *** good *** phase = %d\n", + mmc_hostname(mmc), __func__, phase); + } else { + pr_debug("%s: %s: found ## bad ## phase = %d\n", + mmc_hostname(mmc), __func__, phase); + } + } while (++phase < 16); + + if ((tuned_phase_cnt == NUM_TUNING_PHASES) && + card && mmc_card_mmc(card)) { + /* + * If all phases pass then its a problem. So change the card's + * drive type to a different value, if supported and repeat + * tuning until at least one phase fails. Then set the original + * drive type back. + * + * If all the phases still pass after trying all possible + * drive types, then one of those 16 phases will be picked. + * This is no different from what was going on before the + * modification to change drive type and retune. + */ + pr_debug("%s: tuned phases count: %d\n", mmc_hostname(mmc), + tuned_phase_cnt); + + /* set drive type to other value . default setting is 0x0 */ + while (++drv_type <= MAX_DRV_TYPES_SUPPORTED_HS200) { + pr_debug("%s: trying different drive strength (%d)\n", + mmc_hostname(mmc), drv_type); + if (card->ext_csd.raw_driver_strength & + (1 << drv_type)) { + sdhci_msm_set_mmc_drv_type(host, opcode, + drv_type); + if (!drv_type_changed) + drv_type_changed = true; + goto retry; + } } - } while (++phase < ARRAY_SIZE(tuned_phases)); + } + + /* reset drive type to default (50 ohm) if changed */ + if (drv_type_changed) + sdhci_msm_set_mmc_drv_type(host, opcode, 0); if (tuned_phase_cnt) { rc = msm_find_most_appropriate_phase(host, tuned_phases, - tuned_phase_cnt); + tuned_phase_cnt); if (rc < 0) - return rc; + goto kfree; else - phase = rc; + phase = (u8)rc; /* * Finally set the selected phase in delay @@ -396,70 +1330,3020 @@ retry: */ rc = msm_config_cm_dll_phase(host, phase); if (rc) - return rc; - dev_dbg(mmc_dev(mmc), "%s: Setting the tuning phase to %d\n", - mmc_hostname(mmc), phase); + goto kfree; + msm_host->saved_tuning_phase = phase; + pr_debug("%s: %s: finally setting the tuning phase to %d\n", + mmc_hostname(mmc), __func__, phase); } else { if (--tuning_seq_cnt) goto retry; - /* Tuning failed */ - dev_dbg(mmc_dev(mmc), "%s: No tuning point found\n", - mmc_hostname(mmc)); + /* tuning failed */ + pr_err("%s: %s: no tuning point found\n", + mmc_hostname(mmc), __func__); rc = -EIO; } +kfree: + kfree(data_buf); +out: + spin_lock_irqsave(&host->lock, flags); + if (!rc) + msm_host->tuning_done = true; + spin_unlock_irqrestore(&host->lock, flags); + msm_host->tuning_in_progress = false; + pr_debug("%s: Exit %s, err(%d)\n", mmc_hostname(mmc), __func__, rc); return rc; } -static const struct of_device_id sdhci_msm_dt_match[] = { - { .compatible = "qcom,sdhci-msm-v4" }, - {}, -}; +static int sdhci_msm_setup_gpio(struct sdhci_msm_pltfm_data *pdata, bool enable) +{ + struct sdhci_msm_gpio_data *curr; + int i, ret = 0; -MODULE_DEVICE_TABLE(of, sdhci_msm_dt_match); + curr = pdata->pin_data->gpio_data; + for (i = 0; i < curr->size; i++) { + if (!gpio_is_valid(curr->gpio[i].no)) { + ret = -EINVAL; + pr_err("%s: Invalid gpio = %d\n", __func__, + curr->gpio[i].no); + goto free_gpios; + } + if (enable) { + ret = gpio_request(curr->gpio[i].no, + curr->gpio[i].name); + if (ret) { + pr_err("%s: gpio_request(%d, %s) failed %d\n", + __func__, curr->gpio[i].no, + curr->gpio[i].name, ret); + goto free_gpios; + } + curr->gpio[i].is_enabled = true; + } else { + gpio_free(curr->gpio[i].no); + curr->gpio[i].is_enabled = false; + } + } + return ret; + +free_gpios: + for (i--; i >= 0; i--) { + gpio_free(curr->gpio[i].no); + curr->gpio[i].is_enabled = false; + } + return ret; +} + +static int sdhci_msm_setup_pinctrl(struct sdhci_msm_pltfm_data *pdata, + bool enable) +{ + int ret = 0; + + if (enable) + ret = pinctrl_select_state(pdata->pctrl_data->pctrl, + pdata->pctrl_data->pins_active); + else + ret = pinctrl_select_state(pdata->pctrl_data->pctrl, + pdata->pctrl_data->pins_sleep); + + if (ret < 0) + pr_err("%s state for pinctrl failed with %d\n", + enable ? "Enabling" : "Disabling", ret); + + return ret; +} + +static int sdhci_msm_setup_pins(struct sdhci_msm_pltfm_data *pdata, bool enable) +{ + int ret = 0; + + if (pdata->pin_cfg_sts == enable) { + return 0; + } else if (pdata->pctrl_data) { + ret = sdhci_msm_setup_pinctrl(pdata, enable); + goto out; + } else if (!pdata->pin_data) { + return 0; + } + if (pdata->pin_data->is_gpio) + ret = sdhci_msm_setup_gpio(pdata, enable); +out: + if (!ret) + pdata->pin_cfg_sts = enable; + + return ret; +} + +static int sdhci_msm_dt_get_array(struct device *dev, const char *prop_name, + u32 **out, int *len, u32 size) +{ + int ret = 0; + struct device_node *np = dev->of_node; + size_t sz; + u32 *arr = NULL; + + if (!of_get_property(np, prop_name, len)) { + ret = -EINVAL; + goto out; + } + sz = *len = *len / sizeof(*arr); + if (sz <= 0 || (size > 0 && (sz > size))) { + dev_err(dev, "%s invalid size\n", prop_name); + ret = -EINVAL; + goto out; + } + + arr = devm_kzalloc(dev, sz * sizeof(*arr), GFP_KERNEL); + if (!arr) { + dev_err(dev, "%s failed allocating memory\n", prop_name); + ret = -ENOMEM; + goto out; + } + + ret = of_property_read_u32_array(np, prop_name, arr, sz); + if (ret < 0) { + dev_err(dev, "%s failed reading array %d\n", prop_name, ret); + goto out; + } + *out = arr; +out: + if (ret) + *len = 0; + return ret; +} + +#define MAX_PROP_SIZE 32 +static int sdhci_msm_dt_parse_vreg_info(struct device *dev, + struct sdhci_msm_reg_data **vreg_data, const char *vreg_name) +{ + int len, ret = 0; + const __be32 *prop; + char prop_name[MAX_PROP_SIZE]; + struct sdhci_msm_reg_data *vreg; + struct device_node *np = dev->of_node; + + snprintf(prop_name, MAX_PROP_SIZE, "%s-supply", vreg_name); + if (!of_parse_phandle(np, prop_name, 0)) { + dev_info(dev, "No vreg data found for %s\n", vreg_name); + return ret; + } + + vreg = devm_kzalloc(dev, sizeof(*vreg), GFP_KERNEL); + if (!vreg) { + dev_err(dev, "No memory for vreg: %s\n", vreg_name); + ret = -ENOMEM; + return ret; + } + + vreg->name = vreg_name; + + snprintf(prop_name, MAX_PROP_SIZE, + "qcom,%s-always-on", vreg_name); + if (of_get_property(np, prop_name, NULL)) + vreg->is_always_on = true; + + snprintf(prop_name, MAX_PROP_SIZE, + "qcom,%s-lpm-sup", vreg_name); + if (of_get_property(np, prop_name, NULL)) + vreg->lpm_sup = true; + + snprintf(prop_name, MAX_PROP_SIZE, + "qcom,%s-voltage-level", vreg_name); + prop = of_get_property(np, prop_name, &len); + if (!prop || (len != (2 * sizeof(__be32)))) { + dev_warn(dev, "%s %s property\n", + prop ? "invalid format" : "no", prop_name); + } else { + vreg->low_vol_level = be32_to_cpup(&prop[0]); + vreg->high_vol_level = be32_to_cpup(&prop[1]); + } + + snprintf(prop_name, MAX_PROP_SIZE, + "qcom,%s-current-level", vreg_name); + prop = of_get_property(np, prop_name, &len); + if (!prop || (len != (2 * sizeof(__be32)))) { + dev_warn(dev, "%s %s property\n", + prop ? "invalid format" : "no", prop_name); + } else { + vreg->lpm_uA = be32_to_cpup(&prop[0]); + vreg->hpm_uA = be32_to_cpup(&prop[1]); + } + + *vreg_data = vreg; + dev_dbg(dev, "%s: %s %s vol=[%d %d]uV, curr=[%d %d]uA\n", + vreg->name, vreg->is_always_on ? "always_on," : "", + vreg->lpm_sup ? "lpm_sup," : "", vreg->low_vol_level, + vreg->high_vol_level, vreg->lpm_uA, vreg->hpm_uA); + + return ret; +} + +/* GPIO/Pad data extraction */ +static int sdhci_msm_parse_pinctrl_info(struct device *dev, + struct sdhci_msm_pltfm_data *pdata) +{ + struct sdhci_pinctrl_data *pctrl_data; + struct pinctrl *pctrl; + int ret = 0; + + /* Try to obtain pinctrl handle */ + pctrl = devm_pinctrl_get(dev); + if (IS_ERR(pctrl)) { + ret = PTR_ERR(pctrl); + goto out; + } + pctrl_data = devm_kzalloc(dev, sizeof(*pctrl_data), GFP_KERNEL); + if (!pctrl_data) { + dev_err(dev, "No memory for sdhci_pinctrl_data\n"); + ret = -ENOMEM; + goto out; + } + pctrl_data->pctrl = pctrl; + /* Look-up and keep the states handy to be used later */ + pctrl_data->pins_active = pinctrl_lookup_state( + pctrl_data->pctrl, "active"); + if (IS_ERR(pctrl_data->pins_active)) { + ret = PTR_ERR(pctrl_data->pins_active); + dev_err(dev, "Could not get active pinstates, err:%d\n", ret); + goto out; + } + pctrl_data->pins_sleep = pinctrl_lookup_state( + pctrl_data->pctrl, "sleep"); + if (IS_ERR(pctrl_data->pins_sleep)) { + ret = PTR_ERR(pctrl_data->pins_sleep); + dev_err(dev, "Could not get sleep pinstates, err:%d\n", ret); + goto out; + } + pdata->pctrl_data = pctrl_data; +out: + return ret; +} + +#define GPIO_NAME_MAX_LEN 32 +static int sdhci_msm_dt_parse_gpio_info(struct device *dev, + struct sdhci_msm_pltfm_data *pdata) +{ + int ret = 0, cnt, i; + struct sdhci_msm_pin_data *pin_data; + struct device_node *np = dev->of_node; + + ret = sdhci_msm_parse_pinctrl_info(dev, pdata); + if (!ret) { + goto out; + } else if (ret == -EPROBE_DEFER) { + dev_err(dev, "Pinctrl framework not registered, err:%d\n", ret); + goto out; + } else { + dev_err(dev, "Parsing Pinctrl failed with %d, falling back on GPIO lib\n", + ret); + ret = 0; + } + pin_data = devm_kzalloc(dev, sizeof(*pin_data), GFP_KERNEL); + if (!pin_data) { + dev_err(dev, "No memory for pin_data\n"); + ret = -ENOMEM; + goto out; + } + + cnt = of_gpio_count(np); + if (cnt > 0) { + pin_data->is_gpio = true; + pin_data->gpio_data = devm_kzalloc(dev, + sizeof(struct sdhci_msm_gpio_data), GFP_KERNEL); + if (!pin_data->gpio_data) { + dev_err(dev, "No memory for gpio_data\n"); + ret = -ENOMEM; + goto out; + } + pin_data->gpio_data->size = cnt; + pin_data->gpio_data->gpio = devm_kzalloc(dev, cnt * + sizeof(struct sdhci_msm_gpio), GFP_KERNEL); + + if (!pin_data->gpio_data->gpio) { + dev_err(dev, "No memory for gpio\n"); + ret = -ENOMEM; + goto out; + } + for (i = 0; i < cnt; i++) { + const char *name = NULL; + char result[GPIO_NAME_MAX_LEN]; + pin_data->gpio_data->gpio[i].no = of_get_gpio(np, i); + of_property_read_string_index(np, + "qcom,gpio-names", i, &name); + + snprintf(result, GPIO_NAME_MAX_LEN, "%s-%s", + dev_name(dev), name ? name : "?"); + pin_data->gpio_data->gpio[i].name = result; + dev_dbg(dev, "%s: gpio[%s] = %d\n", __func__, + pin_data->gpio_data->gpio[i].name, + pin_data->gpio_data->gpio[i].no); + } + } + pdata->pin_data = pin_data; +out: + if (ret) + dev_err(dev, "%s failed with err %d\n", __func__, ret); + return ret; +} + +#ifdef CONFIG_SMP +static inline void parse_affine_irq(struct sdhci_msm_pltfm_data *pdata) +{ + pdata->pm_qos_data.irq_req_type = PM_QOS_REQ_AFFINE_IRQ; +} +#else +static inline void parse_affine_irq(struct sdhci_msm_pltfm_data *pdata) { } +#endif + +static int sdhci_msm_pm_qos_parse_irq(struct device *dev, + struct sdhci_msm_pltfm_data *pdata) +{ + struct device_node *np = dev->of_node; + const char *str; + u32 cpu; + int ret = 0; + int i; + + pdata->pm_qos_data.irq_valid = false; + pdata->pm_qos_data.irq_req_type = PM_QOS_REQ_AFFINE_CORES; + if (!of_property_read_string(np, "qcom,pm-qos-irq-type", &str) && + !strcmp(str, "affine_irq")) { + parse_affine_irq(pdata); + } + + /* must specify cpu for "affine_cores" type */ + if (pdata->pm_qos_data.irq_req_type == PM_QOS_REQ_AFFINE_CORES) { + pdata->pm_qos_data.irq_cpu = -1; + ret = of_property_read_u32(np, "qcom,pm-qos-irq-cpu", &cpu); + if (ret) { + dev_err(dev, "%s: error %d reading irq cpu\n", __func__, + ret); + goto out; + } + if (cpu < 0 || cpu >= num_possible_cpus()) { + dev_err(dev, "%s: invalid irq cpu %d (NR_CPUS=%d)\n", + __func__, cpu, num_possible_cpus()); + ret = -EINVAL; + goto out; + } + pdata->pm_qos_data.irq_cpu = cpu; + } + + if (of_property_count_u32_elems(np, "qcom,pm-qos-irq-latency") != + SDHCI_POWER_POLICY_NUM) { + dev_err(dev, "%s: could not read %d values for 'qcom,pm-qos-irq-latency'\n", + __func__, SDHCI_POWER_POLICY_NUM); + ret = -EINVAL; + goto out; + } + + for (i = 0; i < SDHCI_POWER_POLICY_NUM; i++) + of_property_read_u32_index(np, "qcom,pm-qos-irq-latency", i, + &pdata->pm_qos_data.irq_latency.latency[i]); + + pdata->pm_qos_data.irq_valid = true; +out: + return ret; +} + +static int sdhci_msm_pm_qos_parse_cpu_groups(struct device *dev, + struct sdhci_msm_pltfm_data *pdata) +{ + struct device_node *np = dev->of_node; + u32 mask; + int nr_groups; + int ret; + int i; + + /* Read cpu group mapping */ + nr_groups = of_property_count_u32_elems(np, "qcom,pm-qos-cpu-groups"); + if (nr_groups <= 0) { + ret = -EINVAL; + goto out; + } + pdata->pm_qos_data.cpu_group_map.nr_groups = nr_groups; + pdata->pm_qos_data.cpu_group_map.mask = + kcalloc(nr_groups, sizeof(cpumask_t), GFP_KERNEL); + if (!pdata->pm_qos_data.cpu_group_map.mask) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < nr_groups; i++) { + of_property_read_u32_index(np, "qcom,pm-qos-cpu-groups", + i, &mask); + + pdata->pm_qos_data.cpu_group_map.mask[i].bits[0] = mask; + if (!cpumask_subset(&pdata->pm_qos_data.cpu_group_map.mask[i], + cpu_possible_mask)) { + dev_err(dev, "%s: invalid mask 0x%x of cpu group #%d\n", + __func__, mask, i); + ret = -EINVAL; + goto free_res; + } + } + return 0; + +free_res: + kfree(pdata->pm_qos_data.cpu_group_map.mask); +out: + return ret; +} + +static int sdhci_msm_pm_qos_parse_latency(struct device *dev, const char *name, + int nr_groups, struct sdhci_msm_pm_qos_latency **latency) +{ + struct device_node *np = dev->of_node; + struct sdhci_msm_pm_qos_latency *values; + int ret; + int i; + int group; + int cfg; + + ret = of_property_count_u32_elems(np, name); + if (ret > 0 && ret != SDHCI_POWER_POLICY_NUM * nr_groups) { + dev_err(dev, "%s: invalid number of values for property %s: expected=%d actual=%d\n", + __func__, name, SDHCI_POWER_POLICY_NUM * nr_groups, + ret); + return -EINVAL; + } else if (ret < 0) { + return ret; + } + + values = kcalloc(nr_groups, sizeof(struct sdhci_msm_pm_qos_latency), + GFP_KERNEL); + if (!values) + return -ENOMEM; + + for (i = 0; i < SDHCI_POWER_POLICY_NUM * nr_groups; i++) { + group = i / SDHCI_POWER_POLICY_NUM; + cfg = i % SDHCI_POWER_POLICY_NUM; + of_property_read_u32_index(np, name, i, + &(values[group].latency[cfg])); + } + + *latency = values; + return 0; +} + +static void sdhci_msm_pm_qos_parse(struct device *dev, + struct sdhci_msm_pltfm_data *pdata) +{ + if (sdhci_msm_pm_qos_parse_irq(dev, pdata)) + dev_notice(dev, "%s: PM QoS voting for IRQ will be disabled\n", + __func__); + + if (!sdhci_msm_pm_qos_parse_cpu_groups(dev, pdata)) { + pdata->pm_qos_data.cmdq_valid = + !sdhci_msm_pm_qos_parse_latency(dev, + "qcom,pm-qos-cmdq-latency-us", + pdata->pm_qos_data.cpu_group_map.nr_groups, + &pdata->pm_qos_data.cmdq_latency); + pdata->pm_qos_data.legacy_valid = + !sdhci_msm_pm_qos_parse_latency(dev, + "qcom,pm-qos-legacy-latency-us", + pdata->pm_qos_data.cpu_group_map.nr_groups, + &pdata->pm_qos_data.latency); + if (!pdata->pm_qos_data.cmdq_valid && + !pdata->pm_qos_data.legacy_valid) { + /* clean-up previously allocated arrays */ + kfree(pdata->pm_qos_data.latency); + kfree(pdata->pm_qos_data.cmdq_latency); + dev_err(dev, "%s: invalid PM QoS latency values. Voting for cpu group will be disabled\n", + __func__); + } + } else { + dev_notice(dev, "%s: PM QoS voting for cpu group will be disabled\n", + __func__); + } +} + +/* Parse platform data */ +static +struct sdhci_msm_pltfm_data *sdhci_msm_populate_pdata(struct device *dev, + struct sdhci_msm_host *msm_host) +{ + struct sdhci_msm_pltfm_data *pdata = NULL; + struct device_node *np = dev->of_node; + u32 bus_width = 0; + int len, i; + int clk_table_len; + u32 *clk_table = NULL; + int ice_clk_table_len; + u32 *ice_clk_table = NULL; + enum of_gpio_flags flags = OF_GPIO_ACTIVE_LOW; + const char *lower_bus_speed = NULL; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(dev, "failed to allocate memory for platform data\n"); + goto out; + } + + pdata->status_gpio = of_get_named_gpio_flags(np, "cd-gpios", 0, &flags); + if (gpio_is_valid(pdata->status_gpio) && !(flags & OF_GPIO_ACTIVE_LOW)) + pdata->caps2 |= MMC_CAP2_CD_ACTIVE_HIGH; + + of_property_read_u32(np, "qcom,bus-width", &bus_width); + if (bus_width == 8) + pdata->mmc_bus_width = MMC_CAP_8_BIT_DATA; + else if (bus_width == 4) + pdata->mmc_bus_width = MMC_CAP_4_BIT_DATA; + else { + dev_notice(dev, "invalid bus-width, default to 1-bit mode\n"); + pdata->mmc_bus_width = 0; + } + + if (sdhci_msm_dt_get_array(dev, "qcom,devfreq,freq-table", + &msm_host->mmc->clk_scaling.pltfm_freq_table, + &msm_host->mmc->clk_scaling.pltfm_freq_table_sz, 0)) + pr_debug("%s: no clock scaling frequencies were supplied\n", + dev_name(dev)); + else if (!msm_host->mmc->clk_scaling.pltfm_freq_table || + !msm_host->mmc->clk_scaling.pltfm_freq_table_sz) + dev_err(dev, "bad dts clock scaling frequencies\n"); + + /* + * Few hosts can support DDR52 mode at the same lower + * system voltage corner as high-speed mode. In such cases, + * it is always better to put it in DDR mode which will + * improve the performance without any power impact. + */ + if (!of_property_read_string(np, "qcom,scaling-lower-bus-speed-mode", + &lower_bus_speed)) { + if (!strcmp(lower_bus_speed, "DDR52")) + msm_host->mmc->clk_scaling.lower_bus_speed_mode |= + MMC_SCALING_LOWER_DDR52_MODE; + } + + if (sdhci_msm_dt_get_array(dev, "qcom,clk-rates", + &clk_table, &clk_table_len, 0)) { + dev_err(dev, "failed parsing supported clock rates\n"); + goto out; + } + if (!clk_table || !clk_table_len) { + dev_err(dev, "Invalid clock table\n"); + goto out; + } + pdata->sup_clk_table = clk_table; + pdata->sup_clk_cnt = clk_table_len; + + if (msm_host->ice.pdev) { + if (sdhci_msm_dt_get_array(dev, "qcom,ice-clk-rates", + &ice_clk_table, &ice_clk_table_len, 0)) { + dev_err(dev, "failed parsing supported ice clock rates\n"); + goto out; + } + if (!ice_clk_table || !ice_clk_table_len) { + dev_err(dev, "Invalid clock table\n"); + goto out; + } + if (ice_clk_table_len != 2) { + dev_err(dev, "Need max and min frequencies in the table\n"); + goto out; + } + pdata->sup_ice_clk_table = ice_clk_table; + pdata->sup_ice_clk_cnt = ice_clk_table_len; + pdata->ice_clk_max = pdata->sup_ice_clk_table[0]; + pdata->ice_clk_min = pdata->sup_ice_clk_table[1]; + dev_dbg(dev, "supported ICE clock rates (Hz): max: %u min: %u\n", + pdata->ice_clk_max, pdata->ice_clk_min); + } + + pdata->vreg_data = devm_kzalloc(dev, sizeof(struct + sdhci_msm_slot_reg_data), + GFP_KERNEL); + if (!pdata->vreg_data) { + dev_err(dev, "failed to allocate memory for vreg data\n"); + goto out; + } + + if (sdhci_msm_dt_parse_vreg_info(dev, &pdata->vreg_data->vdd_data, + "vdd")) { + dev_err(dev, "failed parsing vdd data\n"); + goto out; + } + if (sdhci_msm_dt_parse_vreg_info(dev, + &pdata->vreg_data->vdd_io_data, + "vdd-io")) { + dev_err(dev, "failed parsing vdd-io data\n"); + goto out; + } + + if (sdhci_msm_dt_parse_gpio_info(dev, pdata)) { + dev_err(dev, "failed parsing gpio data\n"); + goto out; + } + + len = of_property_count_strings(np, "qcom,bus-speed-mode"); + + for (i = 0; i < len; i++) { + const char *name = NULL; + + of_property_read_string_index(np, + "qcom,bus-speed-mode", i, &name); + if (!name) + continue; + + if (!strncmp(name, "HS400_1p8v", sizeof("HS400_1p8v"))) + pdata->caps2 |= MMC_CAP2_HS400_1_8V; + else if (!strncmp(name, "HS400_1p2v", sizeof("HS400_1p2v"))) + pdata->caps2 |= MMC_CAP2_HS400_1_2V; + else if (!strncmp(name, "HS200_1p8v", sizeof("HS200_1p8v"))) + pdata->caps2 |= MMC_CAP2_HS200_1_8V_SDR; + else if (!strncmp(name, "HS200_1p2v", sizeof("HS200_1p2v"))) + pdata->caps2 |= MMC_CAP2_HS200_1_2V_SDR; + else if (!strncmp(name, "DDR_1p8v", sizeof("DDR_1p8v"))) + pdata->caps |= MMC_CAP_1_8V_DDR + | MMC_CAP_UHS_DDR50; + else if (!strncmp(name, "DDR_1p2v", sizeof("DDR_1p2v"))) + pdata->caps |= MMC_CAP_1_2V_DDR + | MMC_CAP_UHS_DDR50; + } + + if (of_get_property(np, "qcom,nonremovable", NULL)) + pdata->nonremovable = true; + + if (of_get_property(np, "qcom,nonhotplug", NULL)) + pdata->nonhotplug = true; + + pdata->largeaddressbus = + of_property_read_bool(np, "qcom,large-address-bus"); + + if (of_property_read_bool(np, "qcom,wakeup-on-idle")) + msm_host->mmc->wakeup_on_idle = true; + + sdhci_msm_pm_qos_parse(dev, pdata); + + if (of_get_property(np, "qcom,core_3_0v_support", NULL)) + msm_host->core_3_0v_support = true; + + pdata->sdr104_wa = of_property_read_bool(np, "qcom,sdr104-wa"); + + return pdata; +out: + return NULL; +} + +/* Returns required bandwidth in Bytes per Sec */ +static unsigned int sdhci_get_bw_required(struct sdhci_host *host, + struct mmc_ios *ios) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + unsigned int bw; + + bw = msm_host->clk_rate; + /* + * For DDR mode, SDCC controller clock will be at + * the double rate than the actual clock that goes to card. + */ + if (ios->bus_width == MMC_BUS_WIDTH_4) + bw /= 2; + else if (ios->bus_width == MMC_BUS_WIDTH_1) + bw /= 8; + + return bw; +} + +static int sdhci_msm_bus_get_vote_for_bw(struct sdhci_msm_host *host, + unsigned int bw) +{ + unsigned int *table = host->pdata->voting_data->bw_vecs; + unsigned int size = host->pdata->voting_data->bw_vecs_size; + int i; + + if (host->msm_bus_vote.is_max_bw_needed && bw) + return host->msm_bus_vote.max_bw_vote; + + for (i = 0; i < size; i++) { + if (bw <= table[i]) + break; + } + + if (i && (i == size)) + i--; + + return i; +} + +/* + * This function must be called with host lock acquired. + * Caller of this function should also ensure that msm bus client + * handle is not null. + */ +static inline int sdhci_msm_bus_set_vote(struct sdhci_msm_host *msm_host, + int vote, + unsigned long *flags) +{ + struct sdhci_host *host = platform_get_drvdata(msm_host->pdev); + int rc = 0; + + BUG_ON(!flags); + + if (vote != msm_host->msm_bus_vote.curr_vote) { + spin_unlock_irqrestore(&host->lock, *flags); + rc = msm_bus_scale_client_update_request( + msm_host->msm_bus_vote.client_handle, vote); + spin_lock_irqsave(&host->lock, *flags); + if (rc) { + pr_err("%s: msm_bus_scale_client_update_request() failed: bus_client_handle=0x%x, vote=%d, err=%d\n", + mmc_hostname(host->mmc), + msm_host->msm_bus_vote.client_handle, vote, rc); + goto out; + } + msm_host->msm_bus_vote.curr_vote = vote; + } +out: + return rc; +} + +/* + * Internal work. Work to set 0 bandwidth for msm bus. + */ +static void sdhci_msm_bus_work(struct work_struct *work) +{ + struct sdhci_msm_host *msm_host; + struct sdhci_host *host; + unsigned long flags; + + msm_host = container_of(work, struct sdhci_msm_host, + msm_bus_vote.vote_work.work); + host = platform_get_drvdata(msm_host->pdev); + + if (!msm_host->msm_bus_vote.client_handle) + return; + + spin_lock_irqsave(&host->lock, flags); + /* don't vote for 0 bandwidth if any request is in progress */ + if (!host->mrq) { + sdhci_msm_bus_set_vote(msm_host, + msm_host->msm_bus_vote.min_bw_vote, &flags); + } else + pr_warning("%s: %s: Transfer in progress. skipping bus voting to 0 bandwidth\n", + mmc_hostname(host->mmc), __func__); + spin_unlock_irqrestore(&host->lock, flags); +} + +/* + * This function cancels any scheduled delayed work and sets the bus + * vote based on bw (bandwidth) argument. + */ +static void sdhci_msm_bus_cancel_work_and_set_vote(struct sdhci_host *host, + unsigned int bw) +{ + int vote; + unsigned long flags; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + cancel_delayed_work_sync(&msm_host->msm_bus_vote.vote_work); + spin_lock_irqsave(&host->lock, flags); + vote = sdhci_msm_bus_get_vote_for_bw(msm_host, bw); + sdhci_msm_bus_set_vote(msm_host, vote, &flags); + spin_unlock_irqrestore(&host->lock, flags); +} + +#define MSM_MMC_BUS_VOTING_DELAY 200 /* msecs */ + +/* This function queues a work which will set the bandwidth requiement to 0 */ +static void sdhci_msm_bus_queue_work(struct sdhci_host *host) +{ + unsigned long flags; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + spin_lock_irqsave(&host->lock, flags); + if (msm_host->msm_bus_vote.min_bw_vote != + msm_host->msm_bus_vote.curr_vote) + queue_delayed_work(system_wq, + &msm_host->msm_bus_vote.vote_work, + msecs_to_jiffies(MSM_MMC_BUS_VOTING_DELAY)); + spin_unlock_irqrestore(&host->lock, flags); +} + +static int sdhci_msm_bus_register(struct sdhci_msm_host *host, + struct platform_device *pdev) +{ + int rc = 0; + struct msm_bus_scale_pdata *bus_pdata; + + struct sdhci_msm_bus_voting_data *data; + struct device *dev = &pdev->dev; + + data = devm_kzalloc(dev, + sizeof(struct sdhci_msm_bus_voting_data), GFP_KERNEL); + if (!data) { + dev_err(&pdev->dev, + "%s: failed to allocate memory\n", __func__); + rc = -ENOMEM; + goto out; + } + data->bus_pdata = msm_bus_cl_get_pdata(pdev); + if (data->bus_pdata) { + rc = sdhci_msm_dt_get_array(dev, "qcom,bus-bw-vectors-bps", + &data->bw_vecs, &data->bw_vecs_size, 0); + if (rc) { + dev_err(&pdev->dev, + "%s: Failed to get bus-bw-vectors-bps\n", + __func__); + goto out; + } + host->pdata->voting_data = data; + } + if (host->pdata->voting_data && + host->pdata->voting_data->bus_pdata && + host->pdata->voting_data->bw_vecs && + host->pdata->voting_data->bw_vecs_size) { + + bus_pdata = host->pdata->voting_data->bus_pdata; + host->msm_bus_vote.client_handle = + msm_bus_scale_register_client(bus_pdata); + if (!host->msm_bus_vote.client_handle) { + dev_err(&pdev->dev, "msm_bus_scale_register_client()\n"); + rc = -EFAULT; + goto out; + } + /* cache the vote index for minimum and maximum bandwidth */ + host->msm_bus_vote.min_bw_vote = + sdhci_msm_bus_get_vote_for_bw(host, 0); + host->msm_bus_vote.max_bw_vote = + sdhci_msm_bus_get_vote_for_bw(host, UINT_MAX); + } else { + devm_kfree(dev, data); + } + +out: + return rc; +} + +static void sdhci_msm_bus_unregister(struct sdhci_msm_host *host) +{ + if (host->msm_bus_vote.client_handle) + msm_bus_scale_unregister_client( + host->msm_bus_vote.client_handle); +} + +static void sdhci_msm_bus_voting(struct sdhci_host *host, u32 enable) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct mmc_ios *ios = &host->mmc->ios; + unsigned int bw; + + if (!msm_host->msm_bus_vote.client_handle) + return; + + bw = sdhci_get_bw_required(host, ios); + if (enable) { + sdhci_msm_bus_cancel_work_and_set_vote(host, bw); + } else { + /* + * If clock gating is enabled, then remove the vote + * immediately because clocks will be disabled only + * after SDHCI_MSM_MMC_CLK_GATE_DELAY and thus no + * additional delay is required to remove the bus vote. + */ +#ifdef CONFIG_MMC_CLKGATE + if (host->mmc->clkgate_delay) + sdhci_msm_bus_cancel_work_and_set_vote(host, 0); + else +#endif + sdhci_msm_bus_queue_work(host); + } +} + +/* Regulator utility functions */ +static int sdhci_msm_vreg_init_reg(struct device *dev, + struct sdhci_msm_reg_data *vreg) +{ + int ret = 0; + + /* check if regulator is already initialized? */ + if (vreg->reg) + goto out; + + /* Get the regulator handle */ + vreg->reg = devm_regulator_get(dev, vreg->name); + if (IS_ERR(vreg->reg)) { + ret = PTR_ERR(vreg->reg); + pr_err("%s: devm_regulator_get(%s) failed. ret=%d\n", + __func__, vreg->name, ret); + goto out; + } + + if (regulator_count_voltages(vreg->reg) > 0) { + vreg->set_voltage_sup = true; + /* sanity check */ + if (!vreg->high_vol_level || !vreg->hpm_uA) { + pr_err("%s: %s invalid constraints specified\n", + __func__, vreg->name); + ret = -EINVAL; + } + } + +out: + return ret; +} + +static void sdhci_msm_vreg_deinit_reg(struct sdhci_msm_reg_data *vreg) +{ + if (vreg->reg) + devm_regulator_put(vreg->reg); +} + +static int sdhci_msm_vreg_set_optimum_mode(struct sdhci_msm_reg_data + *vreg, int uA_load) +{ + int ret = 0; + + /* + * regulators that do not support regulator_set_voltage also + * do not support regulator_set_optimum_mode + */ + if (vreg->set_voltage_sup) { + ret = regulator_set_load(vreg->reg, uA_load); + if (ret < 0) + pr_err("%s: regulator_set_load(reg=%s,uA_load=%d) failed. ret=%d\n", + __func__, vreg->name, uA_load, ret); + else + /* + * regulator_set_load() can return non zero + * value even for success case. + */ + ret = 0; + } + return ret; +} + +static int sdhci_msm_vreg_set_voltage(struct sdhci_msm_reg_data *vreg, + int min_uV, int max_uV) +{ + int ret = 0; + if (vreg->set_voltage_sup) { + ret = regulator_set_voltage(vreg->reg, min_uV, max_uV); + if (ret) { + pr_err("%s: regulator_set_voltage(%s)failed. min_uV=%d,max_uV=%d,ret=%d\n", + __func__, vreg->name, min_uV, max_uV, ret); + } + } + + return ret; +} + +static int sdhci_msm_vreg_enable(struct sdhci_msm_reg_data *vreg) +{ + int ret = 0; + + /* Put regulator in HPM (high power mode) */ + ret = sdhci_msm_vreg_set_optimum_mode(vreg, vreg->hpm_uA); + if (ret < 0) + return ret; + + if (!vreg->is_enabled) { + /* Set voltage level */ + ret = sdhci_msm_vreg_set_voltage(vreg, vreg->high_vol_level, + vreg->high_vol_level); + if (ret) + return ret; + } + ret = regulator_enable(vreg->reg); + if (ret) { + pr_err("%s: regulator_enable(%s) failed. ret=%d\n", + __func__, vreg->name, ret); + return ret; + } + vreg->is_enabled = true; + return ret; +} + +static int sdhci_msm_vreg_disable(struct sdhci_msm_reg_data *vreg) +{ + int ret = 0; + + /* Never disable regulator marked as always_on */ + if (vreg->is_enabled && !vreg->is_always_on) { + ret = regulator_disable(vreg->reg); + if (ret) { + pr_err("%s: regulator_disable(%s) failed. ret=%d\n", + __func__, vreg->name, ret); + goto out; + } + vreg->is_enabled = false; + + ret = sdhci_msm_vreg_set_optimum_mode(vreg, 0); + if (ret < 0) + goto out; + + /* Set min. voltage level to 0 */ + ret = sdhci_msm_vreg_set_voltage(vreg, 0, vreg->high_vol_level); + if (ret) + goto out; + } else if (vreg->is_enabled && vreg->is_always_on) { + if (vreg->lpm_sup) { + /* Put always_on regulator in LPM (low power mode) */ + ret = sdhci_msm_vreg_set_optimum_mode(vreg, + vreg->lpm_uA); + if (ret < 0) + goto out; + } + } +out: + return ret; +} + +static int sdhci_msm_setup_vreg(struct sdhci_msm_pltfm_data *pdata, + bool enable, bool is_init) +{ + int ret = 0, i; + struct sdhci_msm_slot_reg_data *curr_slot; + struct sdhci_msm_reg_data *vreg_table[2]; + + curr_slot = pdata->vreg_data; + if (!curr_slot) { + pr_debug("%s: vreg info unavailable,assuming the slot is powered by always on domain\n", + __func__); + goto out; + } + + vreg_table[0] = curr_slot->vdd_data; + vreg_table[1] = curr_slot->vdd_io_data; + + for (i = 0; i < ARRAY_SIZE(vreg_table); i++) { + if (vreg_table[i]) { + if (enable) + ret = sdhci_msm_vreg_enable(vreg_table[i]); + else + ret = sdhci_msm_vreg_disable(vreg_table[i]); + if (ret) + goto out; + } + } +out: + return ret; +} + +/* This init function should be called only once for each SDHC slot */ +static int sdhci_msm_vreg_init(struct device *dev, + struct sdhci_msm_pltfm_data *pdata, + bool is_init) +{ + int ret = 0; + struct sdhci_msm_slot_reg_data *curr_slot; + struct sdhci_msm_reg_data *curr_vdd_reg, *curr_vdd_io_reg; + + curr_slot = pdata->vreg_data; + if (!curr_slot) + goto out; + + curr_vdd_reg = curr_slot->vdd_data; + curr_vdd_io_reg = curr_slot->vdd_io_data; + + if (!is_init) + /* Deregister all regulators from regulator framework */ + goto vdd_io_reg_deinit; + + /* + * Get the regulator handle from voltage regulator framework + * and then try to set the voltage level for the regulator + */ + if (curr_vdd_reg) { + ret = sdhci_msm_vreg_init_reg(dev, curr_vdd_reg); + if (ret) + goto out; + } + if (curr_vdd_io_reg) { + ret = sdhci_msm_vreg_init_reg(dev, curr_vdd_io_reg); + if (ret) + goto vdd_reg_deinit; + } + + if (ret) + dev_err(dev, "vreg reset failed (%d)\n", ret); + goto out; + +vdd_io_reg_deinit: + if (curr_vdd_io_reg) + sdhci_msm_vreg_deinit_reg(curr_vdd_io_reg); +vdd_reg_deinit: + if (curr_vdd_reg) + sdhci_msm_vreg_deinit_reg(curr_vdd_reg); +out: + return ret; +} + + +static int sdhci_msm_set_vdd_io_vol(struct sdhci_msm_pltfm_data *pdata, + enum vdd_io_level level, + unsigned int voltage_level) +{ + int ret = 0; + int set_level; + struct sdhci_msm_reg_data *vdd_io_reg; + + if (!pdata->vreg_data) + return ret; + + vdd_io_reg = pdata->vreg_data->vdd_io_data; + if (vdd_io_reg && vdd_io_reg->is_enabled) { + switch (level) { + case VDD_IO_LOW: + set_level = vdd_io_reg->low_vol_level; + break; + case VDD_IO_HIGH: + set_level = vdd_io_reg->high_vol_level; + break; + case VDD_IO_SET_LEVEL: + set_level = voltage_level; + break; + default: + pr_err("%s: invalid argument level = %d", + __func__, level); + ret = -EINVAL; + return ret; + } + ret = sdhci_msm_vreg_set_voltage(vdd_io_reg, set_level, + set_level); + } + return ret; +} + +/* + * Acquire spin-lock host->lock before calling this function + */ +static void sdhci_msm_cfg_sdiowakeup_gpio_irq(struct sdhci_host *host, + bool enable) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + if (enable && !msm_host->is_sdiowakeup_enabled) + enable_irq(msm_host->pdata->sdiowakeup_irq); + else if (!enable && msm_host->is_sdiowakeup_enabled) + disable_irq_nosync(msm_host->pdata->sdiowakeup_irq); + else + dev_warn(&msm_host->pdev->dev, "%s: wakeup to config: %d curr: %d\n", + __func__, enable, msm_host->is_sdiowakeup_enabled); + msm_host->is_sdiowakeup_enabled = enable; +} + +static irqreturn_t sdhci_msm_sdiowakeup_irq(int irq, void *data) +{ + struct sdhci_host *host = (struct sdhci_host *)data; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + unsigned long flags; + + pr_debug("%s: irq (%d) received\n", __func__, irq); + + spin_lock_irqsave(&host->lock, flags); + sdhci_msm_cfg_sdiowakeup_gpio_irq(host, false); + spin_unlock_irqrestore(&host->lock, flags); + msm_host->sdio_pending_processing = true; + + return IRQ_HANDLED; +} + +void sdhci_msm_dump_pwr_ctrl_regs(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + unsigned int irq_flags = 0; + struct irq_desc *pwr_irq_desc = irq_to_desc(msm_host->pwr_irq); + + if (pwr_irq_desc) + irq_flags = pwr_irq_desc->irq_data.common->state_use_accessors; + + pr_err("%s: PWRCTL_STATUS: 0x%08x | PWRCTL_MASK: 0x%08x | PWRCTL_CTL: 0x%08x, pwr isr state=0x%x\n", + mmc_hostname(host->mmc), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_PWRCTL_STATUS), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_PWRCTL_MASK), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_PWRCTL_CTL), irq_flags); + + MMC_TRACE(host->mmc, + "%s: Sts: 0x%08x | Mask: 0x%08x | Ctrl: 0x%08x, pwr isr state=0x%x\n", + __func__, + sdhci_msm_readb_relaxed(host, + msm_host_offset->CORE_PWRCTL_STATUS), + sdhci_msm_readb_relaxed(host, + msm_host_offset->CORE_PWRCTL_MASK), + sdhci_msm_readb_relaxed(host, + msm_host_offset->CORE_PWRCTL_CTL), irq_flags); +} + +static irqreturn_t sdhci_msm_pwr_irq(int irq, void *data) +{ + struct sdhci_host *host = (struct sdhci_host *)data; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + u8 irq_status = 0; + u8 irq_ack = 0; + int ret = 0; + int pwr_state = 0, io_level = 0; + unsigned long flags; + int retry = 10; + + irq_status = sdhci_msm_readb_relaxed(host, + msm_host_offset->CORE_PWRCTL_STATUS); + + pr_debug("%s: Received IRQ(%d), status=0x%x\n", + mmc_hostname(msm_host->mmc), irq, irq_status); + + /* Clear the interrupt */ + sdhci_msm_writeb_relaxed(irq_status, host, + msm_host_offset->CORE_PWRCTL_CLEAR); + + /* + * SDHC has core_mem and hc_mem device memory and these memory + * addresses do not fall within 1KB region. Hence, any update to + * core_mem address space would require an mb() to ensure this gets + * completed before its next update to registers within hc_mem. + */ + mb(); + /* + * There is a rare HW scenario where the first clear pulse could be + * lost when actual reset and clear/read of status register is + * happening at a time. Hence, retry for at least 10 times to make + * sure status register is cleared. Otherwise, this will result in + * a spurious power IRQ resulting in system instability. + */ + while (irq_status & sdhci_msm_readb_relaxed(host, + msm_host_offset->CORE_PWRCTL_STATUS)) { + if (retry == 0) { + pr_err("%s: Timedout clearing (0x%x) pwrctl status register\n", + mmc_hostname(host->mmc), irq_status); + sdhci_msm_dump_pwr_ctrl_regs(host); + BUG_ON(1); + } + sdhci_msm_writeb_relaxed(irq_status, host, + msm_host_offset->CORE_PWRCTL_CLEAR); + retry--; + udelay(10); + } + if (likely(retry < 10)) + pr_debug("%s: success clearing (0x%x) pwrctl status register, retries left %d\n", + mmc_hostname(host->mmc), irq_status, retry); + + /* Handle BUS ON/OFF*/ + if (irq_status & CORE_PWRCTL_BUS_ON) { + ret = sdhci_msm_setup_vreg(msm_host->pdata, true, false); + if (!ret) { + ret = sdhci_msm_setup_pins(msm_host->pdata, true); + ret |= sdhci_msm_set_vdd_io_vol(msm_host->pdata, + VDD_IO_HIGH, 0); + } + if (ret) + irq_ack |= CORE_PWRCTL_BUS_FAIL; + else + irq_ack |= CORE_PWRCTL_BUS_SUCCESS; + + pwr_state = REQ_BUS_ON; + io_level = REQ_IO_HIGH; + } + if (irq_status & CORE_PWRCTL_BUS_OFF) { + if (msm_host->pltfm_init_done) + ret = sdhci_msm_setup_vreg(msm_host->pdata, + false, false); + if (!ret) { + ret = sdhci_msm_setup_pins(msm_host->pdata, false); + ret |= sdhci_msm_set_vdd_io_vol(msm_host->pdata, + VDD_IO_LOW, 0); + } + if (ret) + irq_ack |= CORE_PWRCTL_BUS_FAIL; + else + irq_ack |= CORE_PWRCTL_BUS_SUCCESS; + + pwr_state = REQ_BUS_OFF; + io_level = REQ_IO_LOW; + } + /* Handle IO LOW/HIGH */ + if (irq_status & CORE_PWRCTL_IO_LOW) { + /* Switch voltage Low */ + ret = sdhci_msm_set_vdd_io_vol(msm_host->pdata, VDD_IO_LOW, 0); + if (ret) + irq_ack |= CORE_PWRCTL_IO_FAIL; + else + irq_ack |= CORE_PWRCTL_IO_SUCCESS; + + io_level = REQ_IO_LOW; + } + if (irq_status & CORE_PWRCTL_IO_HIGH) { + /* Switch voltage High */ + ret = sdhci_msm_set_vdd_io_vol(msm_host->pdata, VDD_IO_HIGH, 0); + if (ret) + irq_ack |= CORE_PWRCTL_IO_FAIL; + else + irq_ack |= CORE_PWRCTL_IO_SUCCESS; + + io_level = REQ_IO_HIGH; + } + + /* ACK status to the core */ + sdhci_msm_writeb_relaxed(irq_ack, host, + msm_host_offset->CORE_PWRCTL_CTL); + /* + * SDHC has core_mem and hc_mem device memory and these memory + * addresses do not fall within 1KB region. Hence, any update to + * core_mem address space would require an mb() to ensure this gets + * completed before its next update to registers within hc_mem. + */ + mb(); + + if ((io_level & REQ_IO_HIGH) && + (msm_host->caps_0 & CORE_3_0V_SUPPORT) && + !msm_host->core_3_0v_support) + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) & + ~CORE_IO_PAD_PWR_SWITCH), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + else if ((io_level & REQ_IO_LOW) || + (msm_host->caps_0 & CORE_1_8V_SUPPORT)) + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) | + CORE_IO_PAD_PWR_SWITCH), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + mb(); + + pr_debug("%s: Handled IRQ(%d), ret=%d, ack=0x%x\n", + mmc_hostname(msm_host->mmc), irq, ret, irq_ack); + spin_lock_irqsave(&host->lock, flags); + if (pwr_state) + msm_host->curr_pwr_state = pwr_state; + if (io_level) + msm_host->curr_io_level = io_level; + complete(&msm_host->pwr_irq_completion); + spin_unlock_irqrestore(&host->lock, flags); + + return IRQ_HANDLED; +} + +static ssize_t +show_polling(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + int poll; + unsigned long flags; + + spin_lock_irqsave(&host->lock, flags); + poll = !!(host->mmc->caps & MMC_CAP_NEEDS_POLL); + spin_unlock_irqrestore(&host->lock, flags); + + return snprintf(buf, PAGE_SIZE, "%d\n", poll); +} + +static ssize_t +store_polling(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + int value; + unsigned long flags; + + if (!kstrtou32(buf, 0, &value)) { + spin_lock_irqsave(&host->lock, flags); + if (value) { + host->mmc->caps |= MMC_CAP_NEEDS_POLL; + mmc_detect_change(host->mmc, 0); + } else { + host->mmc->caps &= ~MMC_CAP_NEEDS_POLL; + } + spin_unlock_irqrestore(&host->lock, flags); + } + return count; +} + +static ssize_t +show_sdhci_max_bus_bw(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + return snprintf(buf, PAGE_SIZE, "%u\n", + msm_host->msm_bus_vote.is_max_bw_needed); +} + +static ssize_t +store_sdhci_max_bus_bw(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + uint32_t value; + unsigned long flags; + + if (!kstrtou32(buf, 0, &value)) { + spin_lock_irqsave(&host->lock, flags); + msm_host->msm_bus_vote.is_max_bw_needed = !!value; + spin_unlock_irqrestore(&host->lock, flags); + } + return count; +} + +static void sdhci_msm_check_power_status(struct sdhci_host *host, u32 req_type) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + unsigned long flags; + bool done = false; + u32 io_sig_sts = SWITCHABLE_SIGNALLING_VOL; + + spin_lock_irqsave(&host->lock, flags); + pr_debug("%s: %s: request %d curr_pwr_state %x curr_io_level %x\n", + mmc_hostname(host->mmc), __func__, req_type, + msm_host->curr_pwr_state, msm_host->curr_io_level); + if (!msm_host->mci_removed) + io_sig_sts = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_GENERICS); + + /* + * The IRQ for request type IO High/Low will be generated when - + * 1. SWITCHABLE_SIGNALLING_VOL is enabled in HW. + * 2. If 1 is true and when there is a state change in 1.8V enable + * bit (bit 3) of SDHCI_HOST_CONTROL2 register. The reset state of + * that bit is 0 which indicates 3.3V IO voltage. So, when MMC core + * layer tries to set it to 3.3V before card detection happens, the + * IRQ doesn't get triggered as there is no state change in this bit. + * The driver already handles this case by changing the IO voltage + * level to high as part of controller power up sequence. Hence, check + * for host->pwr to handle a case where IO voltage high request is + * issued even before controller power up. + */ + if (req_type & (REQ_IO_HIGH | REQ_IO_LOW)) { + if (!(io_sig_sts & SWITCHABLE_SIGNALLING_VOL) || + ((req_type & REQ_IO_HIGH) && !host->pwr)) { + pr_debug("%s: do not wait for power IRQ that never comes\n", + mmc_hostname(host->mmc)); + spin_unlock_irqrestore(&host->lock, flags); + return; + } + } + + if ((req_type & msm_host->curr_pwr_state) || + (req_type & msm_host->curr_io_level)) + done = true; + spin_unlock_irqrestore(&host->lock, flags); + + /* + * This is needed here to hanlde a case where IRQ gets + * triggered even before this function is called so that + * x->done counter of completion gets reset. Otherwise, + * next call to wait_for_completion returns immediately + * without actually waiting for the IRQ to be handled. + */ + if (done) + init_completion(&msm_host->pwr_irq_completion); + else if (!wait_for_completion_timeout(&msm_host->pwr_irq_completion, + msecs_to_jiffies(MSM_PWR_IRQ_TIMEOUT_MS))) { + __WARN_printf("%s: request(%d) timed out waiting for pwr_irq\n", + mmc_hostname(host->mmc), req_type); + MMC_TRACE(host->mmc, + "%s: request(%d) timed out waiting for pwr_irq\n", + __func__, req_type); + sdhci_msm_dump_pwr_ctrl_regs(host); + } + pr_debug("%s: %s: request %d done\n", mmc_hostname(host->mmc), + __func__, req_type); +} + +static void sdhci_msm_toggle_cdr(struct sdhci_host *host, bool enable) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + u32 config = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + + if (enable) { + config |= CORE_CDR_EN; + config &= ~CORE_CDR_EXT_EN; + writel_relaxed(config, host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + } else { + config &= ~CORE_CDR_EN; + config |= CORE_CDR_EXT_EN; + writel_relaxed(config, host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + } +} + +static unsigned int sdhci_msm_max_segs(void) +{ + return SDHCI_MSM_MAX_SEGMENTS; +} + +static unsigned int sdhci_msm_get_min_clock(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + return msm_host->pdata->sup_clk_table[0]; +} + +static unsigned int sdhci_msm_get_max_clock(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int max_clk_index = msm_host->pdata->sup_clk_cnt; + + return msm_host->pdata->sup_clk_table[max_clk_index - 1]; +} + +static unsigned int sdhci_msm_get_sup_clk_rate(struct sdhci_host *host, + u32 req_clk) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + unsigned int sel_clk = -1; + unsigned char cnt; + + if (req_clk < sdhci_msm_get_min_clock(host)) { + sel_clk = sdhci_msm_get_min_clock(host); + return sel_clk; + } + + for (cnt = 0; cnt < msm_host->pdata->sup_clk_cnt; cnt++) { + if (msm_host->pdata->sup_clk_table[cnt] > req_clk) { + break; + } else if (msm_host->pdata->sup_clk_table[cnt] == req_clk) { + sel_clk = msm_host->pdata->sup_clk_table[cnt]; + break; + } else { + sel_clk = msm_host->pdata->sup_clk_table[cnt]; + } + } + return sel_clk; +} + +static int sdhci_msm_enable_controller_clock(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int rc = 0; + + if (atomic_read(&msm_host->controller_clock)) + return 0; + + sdhci_msm_bus_voting(host, 1); + + if (!IS_ERR(msm_host->pclk)) { + rc = clk_prepare_enable(msm_host->pclk); + if (rc) { + pr_err("%s: %s: failed to enable the pclk with error %d\n", + mmc_hostname(host->mmc), __func__, rc); + goto remove_vote; + } + } + + rc = clk_prepare_enable(msm_host->clk); + if (rc) { + pr_err("%s: %s: failed to enable the host-clk with error %d\n", + mmc_hostname(host->mmc), __func__, rc); + goto disable_pclk; + } + + if (!IS_ERR(msm_host->ice_clk)) { + rc = clk_prepare_enable(msm_host->ice_clk); + if (rc) { + pr_err("%s: %s: failed to enable the ice-clk with error %d\n", + mmc_hostname(host->mmc), __func__, rc); + goto disable_host_clk; + } + } + atomic_set(&msm_host->controller_clock, 1); + pr_debug("%s: %s: enabled controller clock\n", + mmc_hostname(host->mmc), __func__); + goto out; + +disable_host_clk: + if (!IS_ERR(msm_host->clk)) + clk_disable_unprepare(msm_host->clk); +disable_pclk: + if (!IS_ERR(msm_host->pclk)) + clk_disable_unprepare(msm_host->pclk); +remove_vote: + if (msm_host->msm_bus_vote.client_handle) + sdhci_msm_bus_cancel_work_and_set_vote(host, 0); +out: + return rc; +} + +static void sdhci_msm_disable_controller_clock(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + if (atomic_read(&msm_host->controller_clock)) { + if (!IS_ERR(msm_host->clk)) + clk_disable_unprepare(msm_host->clk); + if (!IS_ERR(msm_host->pclk)) + clk_disable_unprepare(msm_host->pclk); + if (!IS_ERR(msm_host->ice_clk)) + clk_disable_unprepare(msm_host->ice_clk); + sdhci_msm_bus_voting(host, 0); + atomic_set(&msm_host->controller_clock, 0); + pr_debug("%s: %s: disabled controller clock\n", + mmc_hostname(host->mmc), __func__); + } +} + +static int sdhci_msm_prepare_clocks(struct sdhci_host *host, bool enable) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int rc = 0; + + if (enable && !atomic_read(&msm_host->clks_on)) { + pr_debug("%s: request to enable clocks\n", + mmc_hostname(host->mmc)); + + /* + * The bus-width or the clock rate might have changed + * after controller clocks are enbaled, update bus vote + * in such case. + */ + if (atomic_read(&msm_host->controller_clock)) + sdhci_msm_bus_voting(host, 1); + + rc = sdhci_msm_enable_controller_clock(host); + if (rc) + goto remove_vote; + + if (!IS_ERR_OR_NULL(msm_host->bus_clk)) { + rc = clk_prepare_enable(msm_host->bus_clk); + if (rc) { + pr_err("%s: %s: failed to enable the bus-clock with error %d\n", + mmc_hostname(host->mmc), __func__, rc); + goto disable_controller_clk; + } + } + if (!IS_ERR(msm_host->ff_clk)) { + rc = clk_prepare_enable(msm_host->ff_clk); + if (rc) { + pr_err("%s: %s: failed to enable the ff_clk with error %d\n", + mmc_hostname(host->mmc), __func__, rc); + goto disable_bus_clk; + } + } + if (!IS_ERR(msm_host->sleep_clk)) { + rc = clk_prepare_enable(msm_host->sleep_clk); + if (rc) { + pr_err("%s: %s: failed to enable the sleep_clk with error %d\n", + mmc_hostname(host->mmc), __func__, rc); + goto disable_ff_clk; + } + } + mb(); + + } else if (!enable && atomic_read(&msm_host->clks_on)) { + sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL); + mb(); + /* + * During 1.8V signal switching the clock source must + * still be ON as it requires accessing SDHC + * registers (SDHCi host control2 register bit 3 must + * be written and polled after stopping the SDCLK). + */ + if (host->mmc->card_clock_off) + return 0; + pr_debug("%s: request to disable clocks\n", + mmc_hostname(host->mmc)); + if (!IS_ERR_OR_NULL(msm_host->sleep_clk)) + clk_disable_unprepare(msm_host->sleep_clk); + if (!IS_ERR_OR_NULL(msm_host->ff_clk)) + clk_disable_unprepare(msm_host->ff_clk); + clk_disable_unprepare(msm_host->clk); + if (!IS_ERR(msm_host->ice_clk)) + clk_disable_unprepare(msm_host->ice_clk); + if (!IS_ERR(msm_host->pclk)) + clk_disable_unprepare(msm_host->pclk); + if (!IS_ERR_OR_NULL(msm_host->bus_clk)) + clk_disable_unprepare(msm_host->bus_clk); + + atomic_set(&msm_host->controller_clock, 0); + sdhci_msm_bus_voting(host, 0); + } + atomic_set(&msm_host->clks_on, enable); + goto out; +disable_ff_clk: + if (!IS_ERR_OR_NULL(msm_host->ff_clk)) + clk_disable_unprepare(msm_host->ff_clk); +disable_bus_clk: + if (!IS_ERR_OR_NULL(msm_host->bus_clk)) + clk_disable_unprepare(msm_host->bus_clk); +disable_controller_clk: + if (!IS_ERR_OR_NULL(msm_host->clk)) + clk_disable_unprepare(msm_host->clk); + if (!IS_ERR(msm_host->ice_clk)) + clk_disable_unprepare(msm_host->ice_clk); + if (!IS_ERR_OR_NULL(msm_host->pclk)) + clk_disable_unprepare(msm_host->pclk); + atomic_set(&msm_host->controller_clock, 0); +remove_vote: + if (msm_host->msm_bus_vote.client_handle) + sdhci_msm_bus_cancel_work_and_set_vote(host, 0); +out: + return rc; +} + +static void sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock) +{ + int rc; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + struct mmc_card *card = host->mmc->card; + struct mmc_ios curr_ios = host->mmc->ios; + u32 sup_clock, ddr_clock, dll_lock; + bool curr_pwrsave; + + if (!clock) { + /* + * disable pwrsave to ensure clock is not auto-gated until + * the rate is >400KHz (initialization complete). + */ + writel_relaxed(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) & + ~CORE_CLK_PWRSAVE, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + sdhci_msm_prepare_clocks(host, false); + host->clock = clock; + goto out; + } + + rc = sdhci_msm_prepare_clocks(host, true); + if (rc) + goto out; + + curr_pwrsave = !!(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) & CORE_CLK_PWRSAVE); + if ((clock > 400000) && + !curr_pwrsave && card && mmc_host_may_gate_card(card)) + writel_relaxed(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) + | CORE_CLK_PWRSAVE, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + /* + * Disable pwrsave for a newly added card if doesn't allow clock + * gating. + */ + else if (curr_pwrsave && card && !mmc_host_may_gate_card(card)) + writel_relaxed(readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) + & ~CORE_CLK_PWRSAVE, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + + sup_clock = sdhci_msm_get_sup_clk_rate(host, clock); + if ((curr_ios.timing == MMC_TIMING_UHS_DDR50) || + (curr_ios.timing == MMC_TIMING_MMC_DDR52) || + (curr_ios.timing == MMC_TIMING_MMC_HS400)) { + /* + * The SDHC requires internal clock frequency to be double the + * actual clock that will be set for DDR mode. The controller + * uses the faster clock(100/400MHz) for some of its parts and + * send the actual required clock (50/200MHz) to the card. + */ + ddr_clock = clock * 2; + sup_clock = sdhci_msm_get_sup_clk_rate(host, + ddr_clock); + } + + /* + * In general all timing modes are controlled via UHS mode select in + * Host Control2 register. eMMC specific HS200/HS400 doesn't have + * their respective modes defined here, hence we use these values. + * + * HS200 - SDR104 (Since they both are equivalent in functionality) + * HS400 - This involves multiple configurations + * Initially SDR104 - when tuning is required as HS200 + * Then when switching to DDR @ 400MHz (HS400) we use + * the vendor specific HC_SELECT_IN to control the mode. + * + * In addition to controlling the modes we also need to select the + * correct input clock for DLL depending on the mode. + * + * HS400 - divided clock (free running MCLK/2) + * All other modes - default (free running MCLK) + */ + if (curr_ios.timing == MMC_TIMING_MMC_HS400) { + /* Select the divided clock (free running MCLK/2) */ + writel_relaxed(((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) + & ~CORE_HC_MCLK_SEL_MASK) + | CORE_HC_MCLK_SEL_HS400), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + /* + * Select HS400 mode using the HC_SELECT_IN from VENDOR SPEC + * register + */ + if ((msm_host->tuning_done || + (card && mmc_card_strobe(card) && + msm_host->enhanced_strobe)) && + !msm_host->calibration_done) { + /* + * Write 0x6 to HC_SELECT_IN and 1 to HC_SELECT_IN_EN + * field in VENDOR_SPEC_FUNC + */ + writel_relaxed((readl_relaxed(host->ioaddr + \ + msm_host_offset->CORE_VENDOR_SPEC) + | CORE_HC_SELECT_IN_HS400 + | CORE_HC_SELECT_IN_EN), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + } + if (!host->mmc->ios.old_rate && !msm_host->use_cdclp533) { + /* + * Poll on DLL_LOCK and DDR_DLL_LOCK bits in + * CORE_DLL_STATUS to be set. This should get set + * with in 15 us at 200 MHz. + */ + rc = readl_poll_timeout(host->ioaddr + + msm_host_offset->CORE_DLL_STATUS, + dll_lock, (dll_lock & (CORE_DLL_LOCK | + CORE_DDR_DLL_LOCK)), 10, 1000); + if (rc == -ETIMEDOUT) + pr_err("%s: Unable to get DLL_LOCK/DDR_DLL_LOCK, dll_status: 0x%08x\n", + mmc_hostname(host->mmc), + dll_lock); + } + } else { + if (!msm_host->use_cdclp533) + /* set CORE_PWRSAVE_DLL bit in CORE_VENDOR_SPEC3 */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC3) + & ~CORE_PWRSAVE_DLL), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC3); + + /* Select the default clock (free running MCLK) */ + writel_relaxed(((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) + & ~CORE_HC_MCLK_SEL_MASK) + | CORE_HC_MCLK_SEL_DFLT), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + + /* + * Disable HC_SELECT_IN to be able to use the UHS mode select + * configuration from Host Control2 register for all other + * modes. + * + * Write 0 to HC_SELECT_IN and HC_SELECT_IN_EN field + * in VENDOR_SPEC_FUNC + */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) + & ~CORE_HC_SELECT_IN_EN + & ~CORE_HC_SELECT_IN_MASK), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + } + mb(); + + if (sup_clock != msm_host->clk_rate) { + pr_debug("%s: %s: setting clk rate to %u\n", + mmc_hostname(host->mmc), __func__, sup_clock); + rc = clk_set_rate(msm_host->clk, sup_clock); + if (rc) { + pr_err("%s: %s: Failed to set rate %u for host-clk : %d\n", + mmc_hostname(host->mmc), __func__, + sup_clock, rc); + goto out; + } + msm_host->clk_rate = sup_clock; + host->clock = clock; + /* + * Update the bus vote in case of frequency change due to + * clock scaling. + */ + sdhci_msm_bus_voting(host, 1); + } +out: + sdhci_set_clock(host, clock); +} + +static void sdhci_msm_set_uhs_signaling(struct sdhci_host *host, + unsigned int uhs) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + u16 ctrl_2; + + ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2); + /* Select Bus Speed Mode for host */ + ctrl_2 &= ~SDHCI_CTRL_UHS_MASK; + if ((uhs == MMC_TIMING_MMC_HS400) || + (uhs == MMC_TIMING_MMC_HS200) || + (uhs == MMC_TIMING_UHS_SDR104)) + ctrl_2 |= SDHCI_CTRL_UHS_SDR104; + else if (uhs == MMC_TIMING_UHS_SDR12) + ctrl_2 |= SDHCI_CTRL_UHS_SDR12; + else if (uhs == MMC_TIMING_UHS_SDR25) + ctrl_2 |= SDHCI_CTRL_UHS_SDR25; + else if (uhs == MMC_TIMING_UHS_SDR50) + ctrl_2 |= SDHCI_CTRL_UHS_SDR50; + else if ((uhs == MMC_TIMING_UHS_DDR50) || + (uhs == MMC_TIMING_MMC_DDR52)) + ctrl_2 |= SDHCI_CTRL_UHS_DDR50; + /* + * When clock frquency is less than 100MHz, the feedback clock must be + * provided and DLL must not be used so that tuning can be skipped. To + * provide feedback clock, the mode selection can be any value less + * than 3'b011 in bits [2:0] of HOST CONTROL2 register. + */ + if (host->clock <= CORE_FREQ_100MHZ) { + if ((uhs == MMC_TIMING_MMC_HS400) || + (uhs == MMC_TIMING_MMC_HS200) || + (uhs == MMC_TIMING_UHS_SDR104)) + ctrl_2 &= ~SDHCI_CTRL_UHS_MASK; + + /* + * Make sure DLL is disabled when not required + * + * Write 1 to DLL_RST bit of DLL_CONFIG register + */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + | CORE_DLL_RST), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + + /* Write 1 to DLL_PDN bit of DLL_CONFIG register */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG) + | CORE_DLL_PDN), host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG); + mb(); + + /* + * The DLL needs to be restored and CDCLP533 recalibrated + * when the clock frequency is set back to 400MHz. + */ + msm_host->calibration_done = false; + } + + pr_debug("%s: %s-clock:%u uhs mode:%u ctrl_2:0x%x\n", + mmc_hostname(host->mmc), __func__, host->clock, uhs, ctrl_2); + sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2); + +} + +#define MAX_TEST_BUS 60 +#define DRV_NAME "cmdq-host" +static void sdhci_msm_cmdq_dump_debug_ram(struct sdhci_host *host) +{ + int i = 0; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + struct cmdq_host *cq_host = host->cq_host; + + u32 version = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_VERSION); + u16 minor = version & CORE_VERSION_TARGET_MASK; + /* registers offset changed starting from 4.2.0 */ + int offset = minor >= SDHCI_MSM_VER_420 ? 0 : 0x48; + + if (cq_host->offset_changed) + offset += CQ_V5_VENDOR_CFG; + pr_err("---- Debug RAM dump ----\n"); + pr_err(DRV_NAME ": Debug RAM wrap-around: 0x%08x | Debug RAM overlap: 0x%08x\n", + cmdq_readl(cq_host, CQ_CMD_DBG_RAM_WA + offset), + cmdq_readl(cq_host, CQ_CMD_DBG_RAM_OL + offset)); + + while (i < 16) { + pr_err(DRV_NAME ": Debug RAM dump [%d]: 0x%08x\n", i, + cmdq_readl(cq_host, CQ_CMD_DBG_RAM + offset + (4 * i))); + i++; + } + pr_err("-------------------------\n"); +} + +static void sdhci_msm_cache_debug_data(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_debug_data *cached_data = &msm_host->cached_data; + + memcpy(&cached_data->copy_mmc, msm_host->mmc, + sizeof(struct mmc_host)); + if (msm_host->mmc->card) + memcpy(&cached_data->copy_card, msm_host->mmc->card, + sizeof(struct mmc_card)); + memcpy(&cached_data->copy_host, host, + sizeof(struct sdhci_host)); +} + +void sdhci_msm_dump_vendor_regs(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + int tbsel, tbsel2; + int i, index = 0; + u32 test_bus_val = 0; + u32 debug_reg[MAX_TEST_BUS] = {0}; + u32 sts = 0; + + sdhci_msm_cache_debug_data(host); + pr_info("----------- VENDOR REGISTER DUMP -----------\n"); + if (host->cq_host) + sdhci_msm_cmdq_dump_debug_ram(host); + + MMC_TRACE(host->mmc, "Data cnt: 0x%08x | Fifo cnt: 0x%08x\n", + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_DATA_CNT), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_FIFO_CNT)); + pr_info("Data cnt: 0x%08x | Fifo cnt: 0x%08x | Int sts: 0x%08x\n", + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_DATA_CNT), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_FIFO_CNT), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_STATUS)); + pr_info("DLL cfg: 0x%08x | DLL sts: 0x%08x | SDCC ver: 0x%08x\n", + readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_CONFIG), + readl_relaxed(host->ioaddr + + msm_host_offset->CORE_DLL_STATUS), + sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_VERSION)); + pr_info("Vndr func: 0x%08x | Vndr adma err : addr0: 0x%08x addr1: 0x%08x\n", + readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC), + readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_ADMA_ERR_ADDR0), + readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_ADMA_ERR_ADDR1)); + pr_info("Vndr func2: 0x%08x\n", + readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2)); + + /* + * tbsel indicates [2:0] bits and tbsel2 indicates [7:4] bits + * of CORE_TESTBUS_CONFIG register. + * + * To select test bus 0 to 7 use tbsel and to select any test bus + * above 7 use (tbsel2 | tbsel) to get the test bus number. For eg, + * to select test bus 14, write 0x1E to CORE_TESTBUS_CONFIG register + * i.e., tbsel2[7:4] = 0001, tbsel[2:0] = 110. + */ + for (tbsel2 = 0; tbsel2 < 7; tbsel2++) { + for (tbsel = 0; tbsel < 8; tbsel++) { + if (index >= MAX_TEST_BUS) + break; + test_bus_val = + (tbsel2 << msm_host_offset->CORE_TESTBUS_SEL2_BIT) | + tbsel | msm_host_offset->CORE_TESTBUS_ENA; + sdhci_msm_writel_relaxed(test_bus_val, host, + msm_host_offset->CORE_TESTBUS_CONFIG); + debug_reg[index++] = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_SDCC_DEBUG_REG); + } + } + for (i = 0; i < MAX_TEST_BUS; i = i + 4) + pr_info(" Test bus[%d to %d]: 0x%08x 0x%08x 0x%08x 0x%08x\n", + i, i + 3, debug_reg[i], debug_reg[i+1], + debug_reg[i+2], debug_reg[i+3]); + if (host->is_crypto_en) { + sdhci_msm_ice_get_status(host, &sts); + pr_info("%s: ICE status %x\n", mmc_hostname(host->mmc), sts); + sdhci_msm_ice_print_regs(host); + } +} + +void sdhci_msm_reset(struct sdhci_host *host, u8 mask) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + /* Set ICE core to be reset in sync with SDHC core */ + if (msm_host->ice.pdev) { + if (msm_host->ice_hci_support) + writel_relaxed(1, host->ioaddr + + HC_VENDOR_SPECIFIC_ICE_CTRL); + else + writel_relaxed(1, + host->ioaddr + CORE_VENDOR_SPEC_ICE_CTRL); + } + + sdhci_reset(host, mask); +} + +/* + * sdhci_msm_enhanced_strobe_mask :- + * Before running CMDQ transfers in HS400 Enhanced Strobe mode, + * SW should write 3 to + * HC_VENDOR_SPECIFIC_FUNC3.CMDEN_HS400_INPUT_MASK_CNT register. + * The default reset value of this register is 2. + */ +static void sdhci_msm_enhanced_strobe_mask(struct sdhci_host *host, bool set) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + if (!msm_host->enhanced_strobe || + !mmc_card_strobe(msm_host->mmc->card)) { + pr_debug("%s: host/card does not support hs400 enhanced strobe\n", + mmc_hostname(host->mmc)); + return; + } + + if (set) { + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC3) + | CORE_CMDEN_HS400_INPUT_MASK_CNT), + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC3); + } else { + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC3) + & ~CORE_CMDEN_HS400_INPUT_MASK_CNT), + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC3); + } +} + +static void sdhci_msm_clear_set_dumpregs(struct sdhci_host *host, bool set) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + if (set) { + sdhci_msm_writel_relaxed(msm_host_offset->CORE_TESTBUS_ENA, + host, msm_host_offset->CORE_TESTBUS_CONFIG); + } else { + u32 value; + value = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_TESTBUS_CONFIG); + value &= ~(msm_host_offset->CORE_TESTBUS_ENA); + sdhci_msm_writel_relaxed(value, host, + msm_host_offset->CORE_TESTBUS_CONFIG); + } +} + +int sdhci_msm_notify_load(struct sdhci_host *host, enum mmc_load state) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int ret = 0; + u32 clk_rate = 0; + + if (!IS_ERR(msm_host->ice_clk)) { + clk_rate = (state == MMC_LOAD_LOW) ? + msm_host->pdata->ice_clk_min : + msm_host->pdata->ice_clk_max; + if (msm_host->ice_clk_rate == clk_rate) + return 0; + pr_debug("%s: changing ICE clk rate to %u\n", + mmc_hostname(host->mmc), clk_rate); + ret = clk_set_rate(msm_host->ice_clk, clk_rate); + if (ret) { + pr_err("%s: ICE_CLK rate set failed (%d) for %u\n", + mmc_hostname(host->mmc), ret, clk_rate); + return ret; + } + msm_host->ice_clk_rate = clk_rate; + } + return 0; +} + +void sdhci_msm_reset_workaround(struct sdhci_host *host, u32 enable) +{ + u32 vendor_func2; + unsigned long timeout; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + vendor_func2 = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + + if (enable) { + writel_relaxed(vendor_func2 | HC_SW_RST_REQ, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + timeout = 10000; + while (readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2) & HC_SW_RST_REQ) { + if (timeout == 0) { + pr_info("%s: Applying wait idle disable workaround\n", + mmc_hostname(host->mmc)); + /* + * Apply the reset workaround to not wait for + * pending data transfers on AXI before + * resetting the controller. This could be + * risky if the transfers were stuck on the + * AXI bus. + */ + vendor_func2 = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + writel_relaxed(vendor_func2 | + HC_SW_RST_WAIT_IDLE_DIS, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + host->reset_wa_t = ktime_get(); + return; + } + timeout--; + udelay(10); + } + pr_info("%s: waiting for SW_RST_REQ is successful\n", + mmc_hostname(host->mmc)); + } else { + writel_relaxed(vendor_func2 & ~HC_SW_RST_WAIT_IDLE_DIS, + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + } +} + +static void sdhci_msm_pm_qos_irq_unvote_work(struct work_struct *work) +{ + struct sdhci_msm_pm_qos_irq *pm_qos_irq = + container_of(work, struct sdhci_msm_pm_qos_irq, + unvote_work.work); + + if (atomic_read(&pm_qos_irq->counter)) + return; + + pm_qos_irq->latency = PM_QOS_DEFAULT_VALUE; + pm_qos_update_request(&pm_qos_irq->req, pm_qos_irq->latency); +} + +void sdhci_msm_pm_qos_irq_vote(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_pm_qos_latency *latency = + &msm_host->pdata->pm_qos_data.irq_latency; + int counter; + + if (!msm_host->pm_qos_irq.enabled) + return; + + counter = atomic_inc_return(&msm_host->pm_qos_irq.counter); + /* Make sure to update the voting in case power policy has changed */ + if (msm_host->pm_qos_irq.latency == latency->latency[host->power_policy] + && counter > 1) + return; + + cancel_delayed_work_sync(&msm_host->pm_qos_irq.unvote_work); + msm_host->pm_qos_irq.latency = latency->latency[host->power_policy]; + pm_qos_update_request(&msm_host->pm_qos_irq.req, + msm_host->pm_qos_irq.latency); +} + +void sdhci_msm_pm_qos_irq_unvote(struct sdhci_host *host, bool async) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int counter; + + if (!msm_host->pm_qos_irq.enabled) + return; + + if (atomic_read(&msm_host->pm_qos_irq.counter)) { + counter = atomic_dec_return(&msm_host->pm_qos_irq.counter); + } else { + WARN(1, "attempt to decrement pm_qos_irq.counter when it's 0"); + return; + } + + if (counter) + return; + + if (async) { + schedule_delayed_work(&msm_host->pm_qos_irq.unvote_work, + msecs_to_jiffies(QOS_REMOVE_DELAY_MS)); + return; + } + + msm_host->pm_qos_irq.latency = PM_QOS_DEFAULT_VALUE; + pm_qos_update_request(&msm_host->pm_qos_irq.req, + msm_host->pm_qos_irq.latency); +} + +static ssize_t +sdhci_msm_pm_qos_irq_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_pm_qos_irq *irq = &msm_host->pm_qos_irq; + + return snprintf(buf, PAGE_SIZE, + "IRQ PM QoS: enabled=%d, counter=%d, latency=%d\n", + irq->enabled, atomic_read(&irq->counter), irq->latency); +} + +static ssize_t +sdhci_msm_pm_qos_irq_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + return snprintf(buf, PAGE_SIZE, "%u\n", msm_host->pm_qos_irq.enabled); +} + +static ssize_t +sdhci_msm_pm_qos_irq_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + uint32_t value; + bool enable; + int ret; + + ret = kstrtou32(buf, 0, &value); + if (ret) + goto out; + enable = !!value; + + if (enable == msm_host->pm_qos_irq.enabled) + goto out; + + msm_host->pm_qos_irq.enabled = enable; + if (!enable) { + cancel_delayed_work_sync(&msm_host->pm_qos_irq.unvote_work); + atomic_set(&msm_host->pm_qos_irq.counter, 0); + msm_host->pm_qos_irq.latency = PM_QOS_DEFAULT_VALUE; + pm_qos_update_request(&msm_host->pm_qos_irq.req, + msm_host->pm_qos_irq.latency); + } + +out: + return count; +} + +#ifdef CONFIG_SMP +static inline void set_affine_irq(struct sdhci_msm_host *msm_host, + struct sdhci_host *host) +{ + msm_host->pm_qos_irq.req.irq = host->irq; +} +#else +static inline void set_affine_irq(struct sdhci_msm_host *msm_host, + struct sdhci_host *host) { } +#endif + +void sdhci_msm_pm_qos_irq_init(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_pm_qos_latency *irq_latency; + int ret; + + if (!msm_host->pdata->pm_qos_data.irq_valid) + return; + + /* Initialize only once as this gets called per partition */ + if (msm_host->pm_qos_irq.enabled) + return; + + atomic_set(&msm_host->pm_qos_irq.counter, 0); + msm_host->pm_qos_irq.req.type = + msm_host->pdata->pm_qos_data.irq_req_type; + if ((msm_host->pm_qos_irq.req.type != PM_QOS_REQ_AFFINE_CORES) && + (msm_host->pm_qos_irq.req.type != PM_QOS_REQ_ALL_CORES)) + set_affine_irq(msm_host, host); + else + cpumask_copy(&msm_host->pm_qos_irq.req.cpus_affine, + cpumask_of(msm_host->pdata->pm_qos_data.irq_cpu)); + + INIT_DELAYED_WORK(&msm_host->pm_qos_irq.unvote_work, + sdhci_msm_pm_qos_irq_unvote_work); + /* For initialization phase, set the performance latency */ + irq_latency = &msm_host->pdata->pm_qos_data.irq_latency; + msm_host->pm_qos_irq.latency = + irq_latency->latency[SDHCI_PERFORMANCE_MODE]; + pm_qos_add_request(&msm_host->pm_qos_irq.req, PM_QOS_CPU_DMA_LATENCY, + msm_host->pm_qos_irq.latency); + msm_host->pm_qos_irq.enabled = true; + + /* sysfs */ + msm_host->pm_qos_irq.enable_attr.show = + sdhci_msm_pm_qos_irq_enable_show; + msm_host->pm_qos_irq.enable_attr.store = + sdhci_msm_pm_qos_irq_enable_store; + sysfs_attr_init(&msm_host->pm_qos_irq.enable_attr.attr); + msm_host->pm_qos_irq.enable_attr.attr.name = "pm_qos_irq_enable"; + msm_host->pm_qos_irq.enable_attr.attr.mode = S_IRUGO | S_IWUSR; + ret = device_create_file(&msm_host->pdev->dev, + &msm_host->pm_qos_irq.enable_attr); + if (ret) + pr_err("%s: fail to create pm_qos_irq_enable (%d)\n", + __func__, ret); + + msm_host->pm_qos_irq.status_attr.show = sdhci_msm_pm_qos_irq_show; + msm_host->pm_qos_irq.status_attr.store = NULL; + sysfs_attr_init(&msm_host->pm_qos_irq.status_attr.attr); + msm_host->pm_qos_irq.status_attr.attr.name = "pm_qos_irq_status"; + msm_host->pm_qos_irq.status_attr.attr.mode = S_IRUGO; + ret = device_create_file(&msm_host->pdev->dev, + &msm_host->pm_qos_irq.status_attr); + if (ret) + pr_err("%s: fail to create pm_qos_irq_status (%d)\n", + __func__, ret); +} + +static ssize_t sdhci_msm_pm_qos_group_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_pm_qos_group *group; + int i; + int nr_groups = msm_host->pdata->pm_qos_data.cpu_group_map.nr_groups; + int offset = 0; + + for (i = 0; i < nr_groups; i++) { + group = &msm_host->pm_qos[i]; + offset += snprintf(&buf[offset], PAGE_SIZE, + "Group #%d (mask=0x%lx) PM QoS: enabled=%d, counter=%d, latency=%d\n", + i, group->req.cpus_affine.bits[0], + msm_host->pm_qos_group_enable, + atomic_read(&group->counter), + group->latency); + } + + return offset; +} + +static ssize_t sdhci_msm_pm_qos_group_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + return snprintf(buf, PAGE_SIZE, "%s\n", + msm_host->pm_qos_group_enable ? "enabled" : "disabled"); +} + +static ssize_t sdhci_msm_pm_qos_group_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int nr_groups = msm_host->pdata->pm_qos_data.cpu_group_map.nr_groups; + uint32_t value; + bool enable; + int ret; + int i; + + ret = kstrtou32(buf, 0, &value); + if (ret) + goto out; + enable = !!value; + + if (enable == msm_host->pm_qos_group_enable) + goto out; + + msm_host->pm_qos_group_enable = enable; + if (!enable) { + for (i = 0; i < nr_groups; i++) { + cancel_delayed_work_sync( + &msm_host->pm_qos[i].unvote_work); + atomic_set(&msm_host->pm_qos[i].counter, 0); + msm_host->pm_qos[i].latency = PM_QOS_DEFAULT_VALUE; + pm_qos_update_request(&msm_host->pm_qos[i].req, + msm_host->pm_qos[i].latency); + } + } + +out: + return count; +} + +static int sdhci_msm_get_cpu_group(struct sdhci_msm_host *msm_host, int cpu) +{ + int i; + struct sdhci_msm_cpu_group_map *map = + &msm_host->pdata->pm_qos_data.cpu_group_map; + + if (cpu < 0) + goto not_found; + + for (i = 0; i < map->nr_groups; i++) + if (cpumask_test_cpu(cpu, &map->mask[i])) + return i; + +not_found: + return -EINVAL; +} + +void sdhci_msm_pm_qos_cpu_vote(struct sdhci_host *host, + struct sdhci_msm_pm_qos_latency *latency, int cpu) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int group = sdhci_msm_get_cpu_group(msm_host, cpu); + struct sdhci_msm_pm_qos_group *pm_qos_group; + int counter; + + if (!msm_host->pm_qos_group_enable || group < 0) + return; + + pm_qos_group = &msm_host->pm_qos[group]; + counter = atomic_inc_return(&pm_qos_group->counter); + + /* Make sure to update the voting in case power policy has changed */ + if (pm_qos_group->latency == latency->latency[host->power_policy] + && counter > 1) + return; + + cancel_delayed_work_sync(&pm_qos_group->unvote_work); + + pm_qos_group->latency = latency->latency[host->power_policy]; + pm_qos_update_request(&pm_qos_group->req, pm_qos_group->latency); +} + +static void sdhci_msm_pm_qos_cpu_unvote_work(struct work_struct *work) +{ + struct sdhci_msm_pm_qos_group *group = + container_of(work, struct sdhci_msm_pm_qos_group, + unvote_work.work); + + if (atomic_read(&group->counter)) + return; + + group->latency = PM_QOS_DEFAULT_VALUE; + pm_qos_update_request(&group->req, group->latency); +} + +bool sdhci_msm_pm_qos_cpu_unvote(struct sdhci_host *host, int cpu, bool async) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int group = sdhci_msm_get_cpu_group(msm_host, cpu); + + if (!msm_host->pm_qos_group_enable || group < 0 || + atomic_dec_return(&msm_host->pm_qos[group].counter)) + return false; + + if (async) { + schedule_delayed_work(&msm_host->pm_qos[group].unvote_work, + msecs_to_jiffies(QOS_REMOVE_DELAY_MS)); + return true; + } + + msm_host->pm_qos[group].latency = PM_QOS_DEFAULT_VALUE; + pm_qos_update_request(&msm_host->pm_qos[group].req, + msm_host->pm_qos[group].latency); + return true; +} + +void sdhci_msm_pm_qos_cpu_init(struct sdhci_host *host, + struct sdhci_msm_pm_qos_latency *latency) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int nr_groups = msm_host->pdata->pm_qos_data.cpu_group_map.nr_groups; + struct sdhci_msm_pm_qos_group *group; + int i; + int ret; + + if (msm_host->pm_qos_group_enable) + return; + + msm_host->pm_qos = kcalloc(nr_groups, sizeof(*msm_host->pm_qos), + GFP_KERNEL); + if (!msm_host->pm_qos) + return; + + for (i = 0; i < nr_groups; i++) { + group = &msm_host->pm_qos[i]; + INIT_DELAYED_WORK(&group->unvote_work, + sdhci_msm_pm_qos_cpu_unvote_work); + atomic_set(&group->counter, 0); + group->req.type = PM_QOS_REQ_AFFINE_CORES; + cpumask_copy(&group->req.cpus_affine, + &msm_host->pdata->pm_qos_data.cpu_group_map.mask[i]); + /* We set default latency here for all pm_qos cpu groups. */ + group->latency = PM_QOS_DEFAULT_VALUE; + pm_qos_add_request(&group->req, PM_QOS_CPU_DMA_LATENCY, + group->latency); + pr_info("%s (): voted for group #%d (mask=0x%lx) latency=%d (0x%p)\n", + __func__, i, + group->req.cpus_affine.bits[0], + group->latency, + &latency[i].latency[SDHCI_PERFORMANCE_MODE]); + } + msm_host->pm_qos_prev_cpu = -1; + msm_host->pm_qos_group_enable = true; + + /* sysfs */ + msm_host->pm_qos_group_status_attr.show = sdhci_msm_pm_qos_group_show; + msm_host->pm_qos_group_status_attr.store = NULL; + sysfs_attr_init(&msm_host->pm_qos_group_status_attr.attr); + msm_host->pm_qos_group_status_attr.attr.name = + "pm_qos_cpu_groups_status"; + msm_host->pm_qos_group_status_attr.attr.mode = S_IRUGO; + ret = device_create_file(&msm_host->pdev->dev, + &msm_host->pm_qos_group_status_attr); + if (ret) + dev_err(&msm_host->pdev->dev, "%s: fail to create pm_qos_group_status_attr (%d)\n", + __func__, ret); + msm_host->pm_qos_group_enable_attr.show = + sdhci_msm_pm_qos_group_enable_show; + msm_host->pm_qos_group_enable_attr.store = + sdhci_msm_pm_qos_group_enable_store; + sysfs_attr_init(&msm_host->pm_qos_group_enable_attr.attr); + msm_host->pm_qos_group_enable_attr.attr.name = + "pm_qos_cpu_groups_enable"; + msm_host->pm_qos_group_enable_attr.attr.mode = S_IRUGO; + ret = device_create_file(&msm_host->pdev->dev, + &msm_host->pm_qos_group_enable_attr); + if (ret) + dev_err(&msm_host->pdev->dev, "%s: fail to create pm_qos_group_enable_attr (%d)\n", + __func__, ret); +} + +static void sdhci_msm_pre_req(struct sdhci_host *host, + struct mmc_request *mmc_req) +{ + int cpu; + int group; + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int prev_group = sdhci_msm_get_cpu_group(msm_host, + msm_host->pm_qos_prev_cpu); + + sdhci_msm_pm_qos_irq_vote(host); + + cpu = get_cpu(); + put_cpu(); + group = sdhci_msm_get_cpu_group(msm_host, cpu); + if (group < 0) + return; + + if (group != prev_group && prev_group >= 0) { + sdhci_msm_pm_qos_cpu_unvote(host, + msm_host->pm_qos_prev_cpu, false); + prev_group = -1; /* make sure to vote for new group */ + } + + if (prev_group < 0) { + sdhci_msm_pm_qos_cpu_vote(host, + msm_host->pdata->pm_qos_data.latency, cpu); + msm_host->pm_qos_prev_cpu = cpu; + } +} + +static void sdhci_msm_post_req(struct sdhci_host *host, + struct mmc_request *mmc_req) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + sdhci_msm_pm_qos_irq_unvote(host, false); + + if (sdhci_msm_pm_qos_cpu_unvote(host, msm_host->pm_qos_prev_cpu, false)) + msm_host->pm_qos_prev_cpu = -1; +} + +static void sdhci_msm_init(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + sdhci_msm_pm_qos_irq_init(host); + + if (msm_host->pdata->pm_qos_data.legacy_valid) + sdhci_msm_pm_qos_cpu_init(host, + msm_host->pdata->pm_qos_data.latency); +} + +static unsigned int sdhci_msm_get_current_limit(struct sdhci_host *host) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_slot_reg_data *curr_slot = msm_host->pdata->vreg_data; + u32 max_curr = 0; + + if (curr_slot && curr_slot->vdd_data) + max_curr = curr_slot->vdd_data->hpm_uA; + + return max_curr; +} static struct sdhci_ops sdhci_msm_ops = { + .crypto_engine_cfg = sdhci_msm_ice_cfg, + .crypto_engine_cmdq_cfg = sdhci_msm_ice_cmdq_cfg, + .crypto_engine_cfg_end = sdhci_msm_ice_cfg_end, + .crypto_cfg_reset = sdhci_msm_ice_cfg_reset, + .crypto_engine_reset = sdhci_msm_ice_reset, + .set_uhs_signaling = sdhci_msm_set_uhs_signaling, + .check_power_status = sdhci_msm_check_power_status, .platform_execute_tuning = sdhci_msm_execute_tuning, - .reset = sdhci_reset, - .set_clock = sdhci_set_clock, + .enhanced_strobe = sdhci_msm_enhanced_strobe, + .toggle_cdr = sdhci_msm_toggle_cdr, + .get_max_segments = sdhci_msm_max_segs, + .set_clock = sdhci_msm_set_clock, + .get_min_clock = sdhci_msm_get_min_clock, + .get_max_clock = sdhci_msm_get_max_clock, + .dump_vendor_regs = sdhci_msm_dump_vendor_regs, + .config_auto_tuning_cmd = sdhci_msm_config_auto_tuning_cmd, + .enable_controller_clock = sdhci_msm_enable_controller_clock, .set_bus_width = sdhci_set_bus_width, - .set_uhs_signaling = sdhci_set_uhs_signaling, + .reset = sdhci_msm_reset, + .clear_set_dumpregs = sdhci_msm_clear_set_dumpregs, + .enhanced_strobe_mask = sdhci_msm_enhanced_strobe_mask, + .notify_load = sdhci_msm_notify_load, + .reset_workaround = sdhci_msm_reset_workaround, + .init = sdhci_msm_init, + .pre_req = sdhci_msm_pre_req, + .post_req = sdhci_msm_post_req, + .get_current_limit = sdhci_msm_get_current_limit, }; +static void sdhci_set_default_hw_caps(struct sdhci_msm_host *msm_host, + struct sdhci_host *host) +{ + u32 version, caps = 0; + u16 minor; + u8 major; + u32 val; + const struct sdhci_msm_offset *msm_host_offset = + msm_host->offset; + + version = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_MCI_VERSION); + major = (version & CORE_VERSION_MAJOR_MASK) >> + CORE_VERSION_MAJOR_SHIFT; + minor = version & CORE_VERSION_TARGET_MASK; + + caps = readl_relaxed(host->ioaddr + SDHCI_CAPABILITIES); + + /* + * Starting with SDCC 5 controller (core major version = 1) + * controller won't advertise 3.0v, 1.8v and 8-bit features + * except for some targets. + */ + if (major >= 1 && minor != 0x11 && minor != 0x12) { + struct sdhci_msm_reg_data *vdd_io_reg; + /* + * Enable 1.8V support capability on controllers that + * support dual voltage + */ + vdd_io_reg = msm_host->pdata->vreg_data->vdd_io_data; + if (vdd_io_reg && (vdd_io_reg->high_vol_level > 2700000)) + caps |= CORE_3_0V_SUPPORT; + if (vdd_io_reg && (vdd_io_reg->low_vol_level < 1950000)) + caps |= CORE_1_8V_SUPPORT; + if (msm_host->pdata->mmc_bus_width == MMC_CAP_8_BIT_DATA) + caps |= CORE_8_BIT_SUPPORT; + } + + /* + * Enable one MID mode for SDCC5 (major 1) on 8916/8939 (minor 0x2e) and + * on 8992 (minor 0x3e) as a workaround to reset for data stuck issue. + */ + if (major == 1 && (minor == 0x2e || minor == 0x3e)) { + host->quirks2 |= SDHCI_QUIRK2_USE_RESET_WORKAROUND; + val = readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + writel_relaxed((val | CORE_ONE_MID_EN), + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC_FUNC2); + } + /* + * SDCC 5 controller with major version 1, minor version 0x34 and later + * with HS 400 mode support will use CM DLL instead of CDC LP 533 DLL. + */ + if ((major == 1) && (minor < 0x34)) + msm_host->use_cdclp533 = true; + + /* + * SDCC 5 controller with major version 1, minor version 0x42 and later + * will require additional steps when resetting DLL. + * It also supports HS400 enhanced strobe mode. + */ + if ((major == 1) && (minor >= 0x42)) { + msm_host->use_updated_dll_reset = true; + msm_host->enhanced_strobe = true; + } + + /* + * SDCC 5 controller with major version 1 and minor version 0x42, + * 0x46 and 0x49 currently uses 14lpp tech DLL whose internal + * gating cannot guarantee MCLK timing requirement i.e. + * when MCLK is gated OFF, it is not gated for less than 0.5us + * and MCLK must be switched on for at-least 1us before DATA + * starts coming. + */ + if ((major == 1) && ((minor == 0x42) || (minor == 0x46) || + (minor == 0x49))) + msm_host->use_14lpp_dll = true; + + /* Fake 3.0V support for SDIO devices which requires such voltage */ + if (msm_host->core_3_0v_support) { + caps |= CORE_3_0V_SUPPORT; + writel_relaxed((readl_relaxed(host->ioaddr + + SDHCI_CAPABILITIES) | caps), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_CAPABILITIES0); + } + + if ((major == 1) && (minor >= 0x49)) + msm_host->rclk_delay_fix = true; + /* + * Mask 64-bit support for controller with 32-bit address bus so that + * smaller descriptor size will be used and improve memory consumption. + */ + if (!msm_host->pdata->largeaddressbus) + caps &= ~CORE_SYS_BUS_SUPPORT_64_BIT; + + writel_relaxed(caps, host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC_CAPABILITIES0); + /* keep track of the value in SDHCI_CAPABILITIES */ + msm_host->caps_0 = caps; + + if ((major == 1) && (minor >= 0x6b)) { + msm_host->ice_hci_support = true; + host->cdr_support = true; + } +} + +#ifdef CONFIG_MMC_CQ_HCI +static void sdhci_msm_cmdq_init(struct sdhci_host *host, + struct platform_device *pdev) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + + if (nocmdq) { + dev_dbg(&pdev->dev, "CMDQ disabled via cmdline\n"); + return; + } + + host->cq_host = cmdq_pltfm_init(pdev); + if (IS_ERR(host->cq_host)) { + dev_dbg(&pdev->dev, "cmdq-pltfm init: failed: %ld\n", + PTR_ERR(host->cq_host)); + host->cq_host = NULL; + } else { + msm_host->mmc->caps2 |= MMC_CAP2_CMD_QUEUE; + } +} +#else +static void sdhci_msm_cmdq_init(struct sdhci_host *host, + struct platform_device *pdev) +{ + +} +#endif + +static bool sdhci_msm_is_bootdevice(struct device *dev) +{ + if (strnstr(saved_command_line, "androidboot.bootdevice=", + strlen(saved_command_line))) { + char search_string[50]; + + snprintf(search_string, ARRAY_SIZE(search_string), + "androidboot.bootdevice=%s", dev_name(dev)); + if (strnstr(saved_command_line, search_string, + strlen(saved_command_line))) + return true; + else + return false; + } + + /* + * "androidboot.bootdevice=" argument is not present then + * return true as we don't know the boot device anyways. + */ + return true; +} + static int sdhci_msm_probe(struct platform_device *pdev) { + const struct sdhci_msm_offset *msm_host_offset; struct sdhci_host *host; struct sdhci_pltfm_host *pltfm_host; struct sdhci_msm_host *msm_host; - struct resource *core_memres; - int ret; - u16 host_version, core_minor; - u32 core_version, caps; - u8 core_major; + struct resource *core_memres = NULL; + int ret = 0, dead = 0; + u16 host_version; + u32 irq_status, irq_ctl; + struct resource *tlmm_memres = NULL; + void __iomem *tlmm_mem; + unsigned long flags; - msm_host = devm_kzalloc(&pdev->dev, sizeof(*msm_host), GFP_KERNEL); - if (!msm_host) - return -ENOMEM; + pr_debug("%s: Enter %s\n", dev_name(&pdev->dev), __func__); + msm_host = devm_kzalloc(&pdev->dev, sizeof(struct sdhci_msm_host), + GFP_KERNEL); + if (!msm_host) { + ret = -ENOMEM; + goto out; + } + if (of_find_compatible_node(NULL, NULL, "qcom,sdhci-msm-v5")) { + msm_host->mci_removed = true; + msm_host->offset = &sdhci_msm_offset_mci_removed; + } else { + msm_host->mci_removed = false; + msm_host->offset = &sdhci_msm_offset_mci_present; + } + msm_host_offset = msm_host->offset; msm_host->sdhci_msm_pdata.ops = &sdhci_msm_ops; host = sdhci_pltfm_init(pdev, &msm_host->sdhci_msm_pdata, 0); - if (IS_ERR(host)) - return PTR_ERR(host); + if (IS_ERR(host)) { + ret = PTR_ERR(host); + goto out_host_free; + } pltfm_host = sdhci_priv(host); pltfm_host->priv = msm_host; msm_host->mmc = host->mmc; msm_host->pdev = pdev; - ret = mmc_of_parse(host->mmc); - if (ret) + /* get the ice device vops if present */ + ret = sdhci_msm_ice_get_dev(host); + if (ret == -EPROBE_DEFER) { + /* + * SDHCI driver might be probed before ICE driver does. + * In that case we would like to return EPROBE_DEFER code + * in order to delay its probing. + */ + dev_err(&pdev->dev, "%s: required ICE device not probed yet err = %d\n", + __func__, ret); goto pltfm_free; - sdhci_get_of_property(pdev); + } else if (ret == -ENODEV) { + /* + * ICE device is not enabled in DTS file. No need for further + * initialization of ICE driver. + */ + dev_warn(&pdev->dev, "%s: ICE device is not enabled", + __func__); + } else if (ret) { + dev_err(&pdev->dev, "%s: sdhci_msm_ice_get_dev failed %d\n", + __func__, ret); + goto pltfm_free; + } + + /* Extract platform data */ + if (pdev->dev.of_node) { + ret = of_alias_get_id(pdev->dev.of_node, "sdhc"); + if (ret <= 0) { + dev_err(&pdev->dev, "Failed to get slot index %d\n", + ret); + goto pltfm_free; + } + + /* skip the probe if eMMC isn't a boot device */ + if ((ret == 1) && !sdhci_msm_is_bootdevice(&pdev->dev)) { + ret = -ENODEV; + goto pltfm_free; + } + + if (disable_slots & (1 << (ret - 1))) { + dev_info(&pdev->dev, "%s: Slot %d disabled\n", __func__, + ret); + ret = -ENODEV; + goto pltfm_free; + } + + if (ret <= 2) + sdhci_slot[ret-1] = msm_host; + + msm_host->pdata = sdhci_msm_populate_pdata(&pdev->dev, + msm_host); + if (!msm_host->pdata) { + dev_err(&pdev->dev, "DT parsing error\n"); + goto pltfm_free; + } + } else { + dev_err(&pdev->dev, "No device tree node\n"); + goto pltfm_free; + } + + /* Setup Clocks */ /* Setup SDCC bus voter clock. */ - msm_host->bus_clk = devm_clk_get(&pdev->dev, "bus"); - if (!IS_ERR(msm_host->bus_clk)) { + msm_host->bus_clk = devm_clk_get(&pdev->dev, "bus_clk"); + if (!IS_ERR_OR_NULL(msm_host->bus_clk)) { /* Vote for max. clk rate for max. performance */ ret = clk_set_rate(msm_host->bus_clk, INT_MAX); if (ret) @@ -470,99 +4354,420 @@ static int sdhci_msm_probe(struct platform_device *pdev) } /* Setup main peripheral bus clock */ - msm_host->pclk = devm_clk_get(&pdev->dev, "iface"); - if (IS_ERR(msm_host->pclk)) { - ret = PTR_ERR(msm_host->pclk); - dev_err(&pdev->dev, "Perpheral clk setup failed (%d)\n", ret); - goto bus_clk_disable; + msm_host->pclk = devm_clk_get(&pdev->dev, "iface_clk"); + if (!IS_ERR(msm_host->pclk)) { + ret = clk_prepare_enable(msm_host->pclk); + if (ret) + goto bus_clk_disable; } + atomic_set(&msm_host->controller_clock, 1); - ret = clk_prepare_enable(msm_host->pclk); - if (ret) - goto bus_clk_disable; + if (msm_host->ice.pdev) { + /* Setup SDC ICE clock */ + msm_host->ice_clk = devm_clk_get(&pdev->dev, "ice_core_clk"); + if (!IS_ERR(msm_host->ice_clk)) { + /* ICE core has only one clock frequency for now */ + ret = clk_set_rate(msm_host->ice_clk, + msm_host->pdata->ice_clk_max); + if (ret) { + dev_err(&pdev->dev, "ICE_CLK rate set failed (%d) for %u\n", + ret, + msm_host->pdata->ice_clk_max); + goto pclk_disable; + } + ret = clk_prepare_enable(msm_host->ice_clk); + if (ret) + goto pclk_disable; + + msm_host->ice_clk_rate = + msm_host->pdata->ice_clk_max; + } + } /* Setup SDC MMC clock */ - msm_host->clk = devm_clk_get(&pdev->dev, "core"); + msm_host->clk = devm_clk_get(&pdev->dev, "core_clk"); if (IS_ERR(msm_host->clk)) { ret = PTR_ERR(msm_host->clk); - dev_err(&pdev->dev, "SDC MMC clk setup failed (%d)\n", ret); goto pclk_disable; } - /* Vote for maximum clock rate for maximum performance */ - ret = clk_set_rate(msm_host->clk, INT_MAX); - if (ret) - dev_warn(&pdev->dev, "core clock boost failed\n"); - + /* Set to the minimum supported clock frequency */ + ret = clk_set_rate(msm_host->clk, sdhci_msm_get_min_clock(host)); + if (ret) { + dev_err(&pdev->dev, "MClk rate set failed (%d)\n", ret); + goto pclk_disable; + } ret = clk_prepare_enable(msm_host->clk); if (ret) goto pclk_disable; - core_memres = platform_get_resource(pdev, IORESOURCE_MEM, 1); - msm_host->core_mem = devm_ioremap_resource(&pdev->dev, core_memres); + msm_host->clk_rate = sdhci_msm_get_min_clock(host); + atomic_set(&msm_host->clks_on, 1); + + /* Setup CDC calibration fixed feedback clock */ + msm_host->ff_clk = devm_clk_get(&pdev->dev, "cal_clk"); + if (!IS_ERR(msm_host->ff_clk)) { + ret = clk_prepare_enable(msm_host->ff_clk); + if (ret) + goto clk_disable; + } - if (IS_ERR(msm_host->core_mem)) { - dev_err(&pdev->dev, "Failed to remap registers\n"); - ret = PTR_ERR(msm_host->core_mem); - goto clk_disable; + /* Setup CDC calibration sleep clock */ + msm_host->sleep_clk = devm_clk_get(&pdev->dev, "sleep_clk"); + if (!IS_ERR(msm_host->sleep_clk)) { + ret = clk_prepare_enable(msm_host->sleep_clk); + if (ret) + goto ff_clk_disable; + } + + msm_host->saved_tuning_phase = INVALID_TUNING_PHASE; + + ret = sdhci_msm_bus_register(msm_host, pdev); + if (ret) + goto sleep_clk_disable; + + if (msm_host->msm_bus_vote.client_handle) + INIT_DELAYED_WORK(&msm_host->msm_bus_vote.vote_work, + sdhci_msm_bus_work); + sdhci_msm_bus_voting(host, 1); + + /* Setup regulators */ + ret = sdhci_msm_vreg_init(&pdev->dev, msm_host->pdata, true); + if (ret) { + dev_err(&pdev->dev, "Regulator setup failed (%d)\n", ret); + goto bus_unregister; } /* Reset the core and Enable SDHC mode */ - writel_relaxed(readl_relaxed(msm_host->core_mem + CORE_POWER) | - CORE_SW_RST, msm_host->core_mem + CORE_POWER); + core_memres = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "core_mem"); + if (!msm_host->mci_removed) { + if (!core_memres) { + dev_err(&pdev->dev, "Failed to get iomem resource\n"); + goto vreg_deinit; + } + msm_host->core_mem = devm_ioremap(&pdev->dev, + core_memres->start, resource_size(core_memres)); + + if (!msm_host->core_mem) { + dev_err(&pdev->dev, "Failed to remap registers\n"); + ret = -ENOMEM; + goto vreg_deinit; + } + } + + tlmm_memres = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "tlmm_mem"); + if (tlmm_memres) { + tlmm_mem = devm_ioremap(&pdev->dev, tlmm_memres->start, + resource_size(tlmm_memres)); + + if (!tlmm_mem) { + dev_err(&pdev->dev, "Failed to remap tlmm registers\n"); + ret = -ENOMEM; + goto vreg_deinit; + } + writel_relaxed(readl_relaxed(tlmm_mem) | 0x2, tlmm_mem); + dev_dbg(&pdev->dev, "tlmm reg %pa value 0x%08x\n", + &tlmm_memres->start, readl_relaxed(tlmm_mem)); + } + + /* + * Reset the vendor spec register to power on reset state. + */ + writel_relaxed(CORE_VENDOR_SPEC_POR_VAL, + host->ioaddr + msm_host_offset->CORE_VENDOR_SPEC); + + if (!msm_host->mci_removed) { + /* Set HC_MODE_EN bit in HC_MODE register */ + writel_relaxed(HC_MODE_EN, (msm_host->core_mem + CORE_HC_MODE)); - /* SW reset can take upto 10HCLK + 15MCLK cycles. (min 40us) */ - usleep_range(1000, 5000); - if (readl(msm_host->core_mem + CORE_POWER) & CORE_SW_RST) { - dev_err(&pdev->dev, "Stuck in reset\n"); - ret = -ETIMEDOUT; - goto clk_disable; + /* Set FF_CLK_SW_RST_DIS bit in HC_MODE register */ + writel_relaxed(readl_relaxed(msm_host->core_mem + + CORE_HC_MODE) | FF_CLK_SW_RST_DIS, + msm_host->core_mem + CORE_HC_MODE); } + sdhci_set_default_hw_caps(msm_host, host); + + /* + * Set the PAD_PWR_SWTICH_EN bit so that the PAD_PWR_SWITCH bit can + * be used as required later on. + */ + writel_relaxed((readl_relaxed(host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC) | + CORE_IO_PAD_PWR_SWITCH_EN), host->ioaddr + + msm_host_offset->CORE_VENDOR_SPEC); + /* + * CORE_SW_RST above may trigger power irq if previous status of PWRCTL + * was either BUS_ON or IO_HIGH_V. So before we enable the power irq + * interrupt in GIC (by registering the interrupt handler), we need to + * ensure that any pending power irq interrupt status is acknowledged + * otherwise power irq interrupt handler would be fired prematurely. + */ + irq_status = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_PWRCTL_STATUS); + sdhci_msm_writel_relaxed(irq_status, host, + msm_host_offset->CORE_PWRCTL_CLEAR); + irq_ctl = sdhci_msm_readl_relaxed(host, + msm_host_offset->CORE_PWRCTL_CTL); + + if (irq_status & (CORE_PWRCTL_BUS_ON | CORE_PWRCTL_BUS_OFF)) + irq_ctl |= CORE_PWRCTL_BUS_SUCCESS; + if (irq_status & (CORE_PWRCTL_IO_HIGH | CORE_PWRCTL_IO_LOW)) + irq_ctl |= CORE_PWRCTL_IO_SUCCESS; + sdhci_msm_writel_relaxed(irq_ctl, host, + msm_host_offset->CORE_PWRCTL_CTL); - /* Set HC_MODE_EN bit in HC_MODE register */ - writel_relaxed(HC_MODE_EN, (msm_host->core_mem + CORE_HC_MODE)); + /* + * Ensure that above writes are propogated before interrupt enablement + * in GIC. + */ + mb(); + /* + * Following are the deviations from SDHC spec v3.0 - + * 1. Card detection is handled using separate GPIO. + * 2. Bus power control is handled by interacting with PMIC. + */ host->quirks |= SDHCI_QUIRK_BROKEN_CARD_DETECTION; host->quirks |= SDHCI_QUIRK_SINGLE_POWER_WRITE; + host->quirks |= SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN; + host->quirks |= SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC; + host->quirks2 |= SDHCI_QUIRK2_ALWAYS_USE_BASE_CLOCK; + host->quirks2 |= SDHCI_QUIRK2_IGNORE_DATATOUT_FOR_R1BCMD; + host->quirks2 |= SDHCI_QUIRK2_BROKEN_PRESET_VALUE; + host->quirks2 |= SDHCI_QUIRK2_USE_RESERVED_MAX_TIMEOUT; + host->quirks2 |= SDHCI_QUIRK2_NON_STANDARD_TUNING; + host->quirks2 |= SDHCI_QUIRK2_USE_PIO_FOR_EMMC_TUNING; + + if (host->quirks2 & SDHCI_QUIRK2_ALWAYS_USE_BASE_CLOCK) + host->quirks2 |= SDHCI_QUIRK2_DIVIDE_TOUT_BY_4; host_version = readw_relaxed((host->ioaddr + SDHCI_HOST_VERSION)); dev_dbg(&pdev->dev, "Host Version: 0x%x Vendor Version 0x%x\n", host_version, ((host_version & SDHCI_VENDOR_VER_MASK) >> - SDHCI_VENDOR_VER_SHIFT)); + SDHCI_VENDOR_VER_SHIFT)); + if (((host_version & SDHCI_VENDOR_VER_MASK) >> + SDHCI_VENDOR_VER_SHIFT) == SDHCI_VER_100) { + /* + * Add 40us delay in interrupt handler when + * operating at initialization frequency(400KHz). + */ + host->quirks2 |= SDHCI_QUIRK2_SLOW_INT_CLR; + /* + * Set Software Reset for DAT line in Software + * Reset Register (Bit 2). + */ + host->quirks2 |= SDHCI_QUIRK2_RDWR_TX_ACTIVE_EOT; + } - core_version = readl_relaxed(msm_host->core_mem + CORE_MCI_VERSION); - core_major = (core_version & CORE_VERSION_MAJOR_MASK) >> - CORE_VERSION_MAJOR_SHIFT; - core_minor = core_version & CORE_VERSION_MINOR_MASK; - dev_dbg(&pdev->dev, "MCI Version: 0x%08x, major: 0x%04x, minor: 0x%02x\n", - core_version, core_major, core_minor); + host->quirks2 |= SDHCI_QUIRK2_IGN_DATA_END_BIT_ERROR; - /* - * Support for some capabilities is not advertised by newer - * controller versions and must be explicitly enabled. - */ - if (core_major >= 1 && core_minor != 0x11 && core_minor != 0x12) { - caps = readl_relaxed(host->ioaddr + SDHCI_CAPABILITIES); - caps |= SDHCI_CAN_VDD_300 | SDHCI_CAN_DO_8BIT; - writel_relaxed(caps, host->ioaddr + - CORE_VENDOR_SPEC_CAPABILITIES0); + /* Setup PWRCTL irq */ + msm_host->pwr_irq = platform_get_irq_byname(pdev, "pwr_irq"); + if (msm_host->pwr_irq < 0) { + dev_err(&pdev->dev, "Failed to get pwr_irq by name (%d)\n", + msm_host->pwr_irq); + goto vreg_deinit; + } + ret = devm_request_threaded_irq(&pdev->dev, msm_host->pwr_irq, NULL, + sdhci_msm_pwr_irq, IRQF_ONESHOT, + dev_name(&pdev->dev), host); + if (ret) { + dev_err(&pdev->dev, "Request threaded irq(%d) failed (%d)\n", + msm_host->pwr_irq, ret); + goto vreg_deinit; } + /* Enable pwr irq interrupts */ + sdhci_msm_writel_relaxed(INT_MASK, host, + msm_host_offset->CORE_PWRCTL_MASK); + +#ifdef CONFIG_MMC_CLKGATE + /* Set clock gating delay to be used when CONFIG_MMC_CLKGATE is set */ + msm_host->mmc->clkgate_delay = SDHCI_MSM_MMC_CLK_GATE_DELAY; +#endif + + /* Set host capabilities */ + msm_host->mmc->caps |= msm_host->pdata->mmc_bus_width; + msm_host->mmc->caps |= msm_host->pdata->caps; + msm_host->mmc->caps |= MMC_CAP_AGGRESSIVE_PM; + msm_host->mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY; + msm_host->mmc->caps2 |= msm_host->pdata->caps2; + msm_host->mmc->caps2 |= MMC_CAP2_BOOTPART_NOACC; + msm_host->mmc->caps2 |= MMC_CAP2_HS400_POST_TUNING; + msm_host->mmc->caps2 |= MMC_CAP2_CLK_SCALE; + msm_host->mmc->caps2 |= MMC_CAP2_SANITIZE; + msm_host->mmc->caps2 |= MMC_CAP2_MAX_DISCARD_SIZE; + msm_host->mmc->caps2 |= MMC_CAP2_SLEEP_AWAKE; + msm_host->mmc->pm_caps |= MMC_PM_KEEP_POWER | MMC_PM_WAKE_SDIO_IRQ; + + if (msm_host->pdata->nonremovable) + msm_host->mmc->caps |= MMC_CAP_NONREMOVABLE; + + if (msm_host->pdata->nonhotplug) + msm_host->mmc->caps2 |= MMC_CAP2_NONHOTPLUG; + + msm_host->mmc->sdr104_wa = msm_host->pdata->sdr104_wa; + + /* Initialize ICE if present */ + if (msm_host->ice.pdev) { + ret = sdhci_msm_ice_init(host); + if (ret) { + dev_err(&pdev->dev, "%s: SDHCi ICE init failed (%d)\n", + mmc_hostname(host->mmc), ret); + ret = -EINVAL; + goto vreg_deinit; + } + host->is_crypto_en = true; + /* Packed commands cannot be encrypted/decrypted using ICE */ + msm_host->mmc->caps2 &= ~(MMC_CAP2_PACKED_WR | + MMC_CAP2_PACKED_WR_CONTROL); + } + + init_completion(&msm_host->pwr_irq_completion); + + if (gpio_is_valid(msm_host->pdata->status_gpio)) { + /* + * Set up the card detect GPIO in active configuration before + * configuring it as an IRQ. Otherwise, it can be in some + * weird/inconsistent state resulting in flood of interrupts. + */ + sdhci_msm_setup_pins(msm_host->pdata, true); + + /* + * This delay is needed for stabilizing the card detect GPIO + * line after changing the pull configs. + */ + usleep_range(10000, 10500); + ret = mmc_gpio_request_cd(msm_host->mmc, + msm_host->pdata->status_gpio, 0); + if (ret) { + dev_err(&pdev->dev, "%s: Failed to request card detection IRQ %d\n", + __func__, ret); + goto vreg_deinit; + } + } + + if ((sdhci_readl(host, SDHCI_CAPABILITIES) & SDHCI_CAN_64BIT) && + (dma_supported(mmc_dev(host->mmc), DMA_BIT_MASK(64)))) { + host->dma_mask = DMA_BIT_MASK(64); + mmc_dev(host->mmc)->dma_mask = &host->dma_mask; + mmc_dev(host->mmc)->coherent_dma_mask = host->dma_mask; + } else if (dma_supported(mmc_dev(host->mmc), DMA_BIT_MASK(32))) { + host->dma_mask = DMA_BIT_MASK(32); + mmc_dev(host->mmc)->dma_mask = &host->dma_mask; + mmc_dev(host->mmc)->coherent_dma_mask = host->dma_mask; + } else { + dev_err(&pdev->dev, "%s: Failed to set dma mask\n", __func__); + } + + msm_host->pdata->sdiowakeup_irq = platform_get_irq_byname(pdev, + "sdiowakeup_irq"); + if (sdhci_is_valid_gpio_wakeup_int(msm_host)) { + dev_info(&pdev->dev, "%s: sdiowakeup_irq = %d\n", __func__, + msm_host->pdata->sdiowakeup_irq); + msm_host->is_sdiowakeup_enabled = true; + ret = request_irq(msm_host->pdata->sdiowakeup_irq, + sdhci_msm_sdiowakeup_irq, + IRQF_SHARED | IRQF_TRIGGER_HIGH, + "sdhci-msm sdiowakeup", host); + if (ret) { + dev_err(&pdev->dev, "%s: request sdiowakeup IRQ %d: failed: %d\n", + __func__, msm_host->pdata->sdiowakeup_irq, ret); + msm_host->pdata->sdiowakeup_irq = -1; + msm_host->is_sdiowakeup_enabled = false; + goto vreg_deinit; + } else { + spin_lock_irqsave(&host->lock, flags); + sdhci_msm_cfg_sdiowakeup_gpio_irq(host, false); + msm_host->sdio_pending_processing = false; + spin_unlock_irqrestore(&host->lock, flags); + } + } + + sdhci_msm_cmdq_init(host, pdev); ret = sdhci_add_host(host); + if (ret) { + dev_err(&pdev->dev, "Add host failed (%d)\n", ret); + goto vreg_deinit; + } + + msm_host->pltfm_init_done = true; + + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_set_autosuspend_delay(&pdev->dev, MSM_AUTOSUSPEND_DELAY_MS); + pm_runtime_use_autosuspend(&pdev->dev); + + msm_host->msm_bus_vote.max_bus_bw.show = show_sdhci_max_bus_bw; + msm_host->msm_bus_vote.max_bus_bw.store = store_sdhci_max_bus_bw; + sysfs_attr_init(&msm_host->msm_bus_vote.max_bus_bw.attr); + msm_host->msm_bus_vote.max_bus_bw.attr.name = "max_bus_bw"; + msm_host->msm_bus_vote.max_bus_bw.attr.mode = S_IRUGO | S_IWUSR; + ret = device_create_file(&pdev->dev, + &msm_host->msm_bus_vote.max_bus_bw); if (ret) - goto clk_disable; + goto remove_host; - return 0; + if (!gpio_is_valid(msm_host->pdata->status_gpio)) { + msm_host->polling.show = show_polling; + msm_host->polling.store = store_polling; + sysfs_attr_init(&msm_host->polling.attr); + msm_host->polling.attr.name = "polling"; + msm_host->polling.attr.mode = S_IRUGO | S_IWUSR; + ret = device_create_file(&pdev->dev, &msm_host->polling); + if (ret) + goto remove_max_bus_bw_file; + } + msm_host->auto_cmd21_attr.show = show_auto_cmd21; + msm_host->auto_cmd21_attr.store = store_auto_cmd21; + sysfs_attr_init(&msm_host->auto_cmd21_attr.attr); + msm_host->auto_cmd21_attr.attr.name = "enable_auto_cmd21"; + msm_host->auto_cmd21_attr.attr.mode = S_IRUGO | S_IWUSR; + ret = device_create_file(&pdev->dev, &msm_host->auto_cmd21_attr); + if (ret) { + pr_err("%s: %s: failed creating auto-cmd21 attr: %d\n", + mmc_hostname(host->mmc), __func__, ret); + device_remove_file(&pdev->dev, &msm_host->auto_cmd21_attr); + } + /* Successful initialization */ + goto out; + +remove_max_bus_bw_file: + device_remove_file(&pdev->dev, &msm_host->msm_bus_vote.max_bus_bw); +remove_host: + dead = (readl_relaxed(host->ioaddr + SDHCI_INT_STATUS) == 0xffffffff); + pm_runtime_disable(&pdev->dev); + sdhci_remove_host(host, dead); +vreg_deinit: + sdhci_msm_vreg_init(&pdev->dev, msm_host->pdata, false); +bus_unregister: + if (msm_host->msm_bus_vote.client_handle) + sdhci_msm_bus_cancel_work_and_set_vote(host, 0); + sdhci_msm_bus_unregister(msm_host); +sleep_clk_disable: + if (!IS_ERR(msm_host->sleep_clk)) + clk_disable_unprepare(msm_host->sleep_clk); +ff_clk_disable: + if (!IS_ERR(msm_host->ff_clk)) + clk_disable_unprepare(msm_host->ff_clk); clk_disable: - clk_disable_unprepare(msm_host->clk); + if (!IS_ERR(msm_host->clk)) + clk_disable_unprepare(msm_host->clk); pclk_disable: - clk_disable_unprepare(msm_host->pclk); + if (!IS_ERR(msm_host->pclk)) + clk_disable_unprepare(msm_host->pclk); bus_clk_disable: - if (!IS_ERR(msm_host->bus_clk)) + if (!IS_ERR_OR_NULL(msm_host->bus_clk)) clk_disable_unprepare(msm_host->bus_clk); pltfm_free: sdhci_pltfm_free(pdev); +out_host_free: + devm_kfree(&pdev->dev, msm_host); +out: + pr_debug("%s: Exit %s\n", dev_name(&pdev->dev), __func__); return ret; } @@ -571,28 +4776,270 @@ static int sdhci_msm_remove(struct platform_device *pdev) struct sdhci_host *host = platform_get_drvdata(pdev); struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = pltfm_host->priv; + struct sdhci_msm_pltfm_data *pdata = msm_host->pdata; int dead = (readl_relaxed(host->ioaddr + SDHCI_INT_STATUS) == - 0xffffffff); + 0xffffffff); + pr_debug("%s: %s\n", dev_name(&pdev->dev), __func__); + if (!gpio_is_valid(msm_host->pdata->status_gpio)) + device_remove_file(&pdev->dev, &msm_host->polling); + device_remove_file(&pdev->dev, &msm_host->msm_bus_vote.max_bus_bw); + pm_runtime_disable(&pdev->dev); sdhci_remove_host(host, dead); sdhci_pltfm_free(pdev); - clk_disable_unprepare(msm_host->clk); - clk_disable_unprepare(msm_host->pclk); - if (!IS_ERR(msm_host->bus_clk)) - clk_disable_unprepare(msm_host->bus_clk); + + sdhci_msm_vreg_init(&pdev->dev, msm_host->pdata, false); + + sdhci_msm_setup_pins(pdata, true); + sdhci_msm_setup_pins(pdata, false); + + if (msm_host->msm_bus_vote.client_handle) { + sdhci_msm_bus_cancel_work_and_set_vote(host, 0); + sdhci_msm_bus_unregister(msm_host); + } return 0; } +#ifdef CONFIG_PM +static int sdhci_msm_cfg_sdio_wakeup(struct sdhci_host *host, bool enable) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + unsigned long flags; + int ret = 0; + + if (!(host->mmc->card && mmc_card_sdio(host->mmc->card) && + sdhci_is_valid_gpio_wakeup_int(msm_host) && + mmc_card_wake_sdio_irq(host->mmc))) { + msm_host->sdio_pending_processing = false; + return 1; + } + + spin_lock_irqsave(&host->lock, flags); + if (enable) { + /* configure DAT1 gpio if applicable */ + if (sdhci_is_valid_gpio_wakeup_int(msm_host)) { + msm_host->sdio_pending_processing = false; + ret = enable_irq_wake(msm_host->pdata->sdiowakeup_irq); + if (!ret) + sdhci_msm_cfg_sdiowakeup_gpio_irq(host, true); + goto out; + } else { + pr_err("%s: sdiowakeup_irq(%d) invalid\n", + mmc_hostname(host->mmc), enable); + } + } else { + if (sdhci_is_valid_gpio_wakeup_int(msm_host)) { + ret = disable_irq_wake(msm_host->pdata->sdiowakeup_irq); + sdhci_msm_cfg_sdiowakeup_gpio_irq(host, false); + msm_host->sdio_pending_processing = false; + } else { + pr_err("%s: sdiowakeup_irq(%d)invalid\n", + mmc_hostname(host->mmc), enable); + + } + } +out: + if (ret) + pr_err("%s: %s: %sable wakeup: failed: %d gpio: %d\n", + mmc_hostname(host->mmc), __func__, enable ? "en" : "dis", + ret, msm_host->pdata->sdiowakeup_irq); + spin_unlock_irqrestore(&host->lock, flags); + return ret; +} + + +static int sdhci_msm_runtime_suspend(struct device *dev) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + ktime_t start = ktime_get(); + int ret; + + if (host->mmc->card && mmc_card_sdio(host->mmc->card)) + goto defer_disable_host_irq; + + sdhci_cfg_irq(host, false, true); + +defer_disable_host_irq: + disable_irq(msm_host->pwr_irq); + + /* + * Remove the vote immediately only if clocks are off in which + * case we might have queued work to remove vote but it may not + * be completed before runtime suspend or system suspend. + */ + if (!atomic_read(&msm_host->clks_on)) { + if (msm_host->msm_bus_vote.client_handle) + sdhci_msm_bus_cancel_work_and_set_vote(host, 0); + } + + if (host->is_crypto_en) { + ret = sdhci_msm_ice_suspend(host); + if (ret < 0) + pr_err("%s: failed to suspend crypto engine %d\n", + mmc_hostname(host->mmc), ret); + } + trace_sdhci_msm_runtime_suspend(mmc_hostname(host->mmc), 0, + ktime_to_us(ktime_sub(ktime_get(), start))); + return 0; +} + +static int sdhci_msm_runtime_resume(struct device *dev) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + ktime_t start = ktime_get(); + int ret; + + if (host->is_crypto_en) { + ret = sdhci_msm_enable_controller_clock(host); + if (ret) { + pr_err("%s: Failed to enable reqd clocks\n", + mmc_hostname(host->mmc)); + goto skip_ice_resume; + } + ret = sdhci_msm_ice_resume(host); + if (ret) + pr_err("%s: failed to resume crypto engine %d\n", + mmc_hostname(host->mmc), ret); + } +skip_ice_resume: + if (host->mmc->card && mmc_card_sdio(host->mmc->card)) + goto defer_enable_host_irq; + + sdhci_cfg_irq(host, true, true); + +defer_enable_host_irq: + enable_irq(msm_host->pwr_irq); + + trace_sdhci_msm_runtime_resume(mmc_hostname(host->mmc), 0, + ktime_to_us(ktime_sub(ktime_get(), start))); + return 0; +} + +static int sdhci_msm_suspend(struct device *dev) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int ret = 0; + int sdio_cfg = 0; + ktime_t start = ktime_get(); + + if (gpio_is_valid(msm_host->pdata->status_gpio) && + (msm_host->mmc->slot.cd_irq >= 0)) + disable_irq(msm_host->mmc->slot.cd_irq); + + if (pm_runtime_suspended(dev)) { + pr_debug("%s: %s: already runtime suspended\n", + mmc_hostname(host->mmc), __func__); + goto out; + } + ret = sdhci_msm_runtime_suspend(dev); +out: + sdhci_msm_disable_controller_clock(host); + if (host->mmc->card && mmc_card_sdio(host->mmc->card)) { + sdio_cfg = sdhci_msm_cfg_sdio_wakeup(host, true); + if (sdio_cfg) + sdhci_cfg_irq(host, false, true); + } + + trace_sdhci_msm_suspend(mmc_hostname(host->mmc), ret, + ktime_to_us(ktime_sub(ktime_get(), start))); + return ret; +} + +static int sdhci_msm_resume(struct device *dev) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int ret = 0; + int sdio_cfg = 0; + ktime_t start = ktime_get(); + + if (gpio_is_valid(msm_host->pdata->status_gpio) && + (msm_host->mmc->slot.cd_irq >= 0)) + enable_irq(msm_host->mmc->slot.cd_irq); + + if (pm_runtime_suspended(dev)) { + pr_debug("%s: %s: runtime suspended, defer system resume\n", + mmc_hostname(host->mmc), __func__); + goto out; + } + + ret = sdhci_msm_runtime_resume(dev); +out: + if (host->mmc->card && mmc_card_sdio(host->mmc->card)) { + sdio_cfg = sdhci_msm_cfg_sdio_wakeup(host, false); + if (sdio_cfg) + sdhci_cfg_irq(host, true, true); + } + + trace_sdhci_msm_resume(mmc_hostname(host->mmc), ret, + ktime_to_us(ktime_sub(ktime_get(), start))); + return ret; +} + +static int sdhci_msm_suspend_noirq(struct device *dev) +{ + struct sdhci_host *host = dev_get_drvdata(dev); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = pltfm_host->priv; + int ret = 0; + + /* + * ksdioirqd may be running, hence retry + * suspend in case the clocks are ON + */ + if (atomic_read(&msm_host->clks_on)) { + pr_warn("%s: %s: clock ON after suspend, aborting suspend\n", + mmc_hostname(host->mmc), __func__); + ret = -EAGAIN; + } + + if (host->mmc->card && mmc_card_sdio(host->mmc->card)) + if (msm_host->sdio_pending_processing) + ret = -EBUSY; + + return ret; +} + +static const struct dev_pm_ops sdhci_msm_pmops = { + SET_SYSTEM_SLEEP_PM_OPS(sdhci_msm_suspend, sdhci_msm_resume) + SET_RUNTIME_PM_OPS(sdhci_msm_runtime_suspend, sdhci_msm_runtime_resume, + NULL) + .suspend_noirq = sdhci_msm_suspend_noirq, +}; + +#define SDHCI_MSM_PMOPS (&sdhci_msm_pmops) + +#else +#define SDHCI_MSM_PMOPS NULL +#endif +static const struct of_device_id sdhci_msm_dt_match[] = { + {.compatible = "qcom,sdhci-msm"}, + {.compatible = "qcom,sdhci-msm-v5"}, + {}, +}; +MODULE_DEVICE_TABLE(of, sdhci_msm_dt_match); + static struct platform_driver sdhci_msm_driver = { - .probe = sdhci_msm_probe, - .remove = sdhci_msm_remove, - .driver = { - .name = "sdhci_msm", - .of_match_table = sdhci_msm_dt_match, + .probe = sdhci_msm_probe, + .remove = sdhci_msm_remove, + .driver = { + .name = "sdhci_msm", + .owner = THIS_MODULE, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .of_match_table = sdhci_msm_dt_match, + .pm = SDHCI_MSM_PMOPS, }, }; module_platform_driver(sdhci_msm_driver); -MODULE_DESCRIPTION("Qualcomm Secure Digital Host Controller Interface driver"); +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Secure Digital Host Controller Interface driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/mmc/host/sdhci-msm.h b/drivers/mmc/host/sdhci-msm.h new file mode 100644 index 000000000000..79949c2c537f --- /dev/null +++ b/drivers/mmc/host/sdhci-msm.h @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef __SDHCI_MSM_H__ +#define __SDHCI_MSM_H__ + +#include <linux/mmc/mmc.h> +#include <linux/pm_qos.h> +#include "sdhci-pltfm.h" + +/* This structure keeps information per regulator */ +struct sdhci_msm_reg_data { + /* voltage regulator handle */ + struct regulator *reg; + /* regulator name */ + const char *name; + /* voltage level to be set */ + u32 low_vol_level; + u32 high_vol_level; + /* Load values for low power and high power mode */ + u32 lpm_uA; + u32 hpm_uA; + + /* is this regulator enabled? */ + bool is_enabled; + /* is this regulator needs to be always on? */ + bool is_always_on; + /* is low power mode setting required for this regulator? */ + bool lpm_sup; + bool set_voltage_sup; +}; + +/* + * This structure keeps information for all the + * regulators required for a SDCC slot. + */ +struct sdhci_msm_slot_reg_data { + /* keeps VDD/VCC regulator info */ + struct sdhci_msm_reg_data *vdd_data; + /* keeps VDD IO regulator info */ + struct sdhci_msm_reg_data *vdd_io_data; +}; + +struct sdhci_msm_gpio { + u32 no; + const char *name; + bool is_enabled; +}; + +struct sdhci_msm_gpio_data { + struct sdhci_msm_gpio *gpio; + u8 size; +}; + +struct sdhci_msm_pin_data { + /* + * = 1 if controller pins are using gpios + * = 0 if controller has dedicated MSM pads + */ + u8 is_gpio; + struct sdhci_msm_gpio_data *gpio_data; +}; + +struct sdhci_pinctrl_data { + struct pinctrl *pctrl; + struct pinctrl_state *pins_active; + struct pinctrl_state *pins_sleep; +}; + +struct sdhci_msm_bus_voting_data { + struct msm_bus_scale_pdata *bus_pdata; + unsigned int *bw_vecs; + unsigned int bw_vecs_size; +}; + +struct sdhci_msm_cpu_group_map { + int nr_groups; + cpumask_t *mask; +}; + +struct sdhci_msm_pm_qos_latency { + s32 latency[SDHCI_POWER_POLICY_NUM]; +}; + +struct sdhci_msm_pm_qos_data { + struct sdhci_msm_cpu_group_map cpu_group_map; + enum pm_qos_req_type irq_req_type; + int irq_cpu; + struct sdhci_msm_pm_qos_latency irq_latency; + struct sdhci_msm_pm_qos_latency *cmdq_latency; + struct sdhci_msm_pm_qos_latency *latency; + bool irq_valid; + bool cmdq_valid; + bool legacy_valid; +}; + +/* + * PM QoS for group voting management - each cpu group defined is associated + * with 1 instance of this structure. + */ +struct sdhci_msm_pm_qos_group { + struct pm_qos_request req; + struct delayed_work unvote_work; + atomic_t counter; + s32 latency; +}; + +/* PM QoS HW IRQ voting */ +struct sdhci_msm_pm_qos_irq { + struct pm_qos_request req; + struct delayed_work unvote_work; + struct device_attribute enable_attr; + struct device_attribute status_attr; + atomic_t counter; + s32 latency; + bool enabled; +}; + +struct sdhci_msm_pltfm_data { + /* Supported UHS-I Modes */ + u32 caps; + + /* More capabilities */ + u32 caps2; + + unsigned long mmc_bus_width; + struct sdhci_msm_slot_reg_data *vreg_data; + bool nonremovable; + bool nonhotplug; + bool largeaddressbus; + bool pin_cfg_sts; + struct sdhci_msm_pin_data *pin_data; + struct sdhci_pinctrl_data *pctrl_data; + int status_gpio; /* card detection GPIO that is configured as IRQ */ + struct sdhci_msm_bus_voting_data *voting_data; + u32 *sup_clk_table; + unsigned char sup_clk_cnt; + int sdiowakeup_irq; + u32 *sup_ice_clk_table; + unsigned char sup_ice_clk_cnt; + u32 ice_clk_max; + u32 ice_clk_min; + struct sdhci_msm_pm_qos_data pm_qos_data; + bool sdr104_wa; +}; + +struct sdhci_msm_bus_vote { + uint32_t client_handle; + uint32_t curr_vote; + int min_bw_vote; + int max_bw_vote; + bool is_max_bw_needed; + struct delayed_work vote_work; + struct device_attribute max_bus_bw; +}; + +struct sdhci_msm_ice_data { + struct qcom_ice_variant_ops *vops; + struct platform_device *pdev; + int state; +}; + +struct sdhci_msm_debug_data { + struct mmc_host copy_mmc; + struct mmc_card copy_card; + struct sdhci_host copy_host; +}; + +struct sdhci_msm_host { + struct platform_device *pdev; + void __iomem *core_mem; /* MSM SDCC mapped address */ + void __iomem *cryptoio; /* ICE HCI mapped address */ + bool ice_hci_support; + int pwr_irq; /* power irq */ + struct clk *clk; /* main SD/MMC bus clock */ + struct clk *pclk; /* SDHC peripheral bus clock */ + struct clk *bus_clk; /* SDHC bus voter clock */ + struct clk *ff_clk; /* CDC calibration fixed feedback clock */ + struct clk *sleep_clk; /* CDC calibration sleep clock */ + struct clk *ice_clk; /* SDHC peripheral ICE clock */ + atomic_t clks_on; /* Set if clocks are enabled */ + struct sdhci_msm_pltfm_data *pdata; + struct mmc_host *mmc; + struct sdhci_msm_debug_data cached_data; + struct sdhci_pltfm_data sdhci_msm_pdata; + u32 curr_pwr_state; + u32 curr_io_level; + struct completion pwr_irq_completion; + struct sdhci_msm_bus_vote msm_bus_vote; + struct device_attribute polling; + u32 clk_rate; /* Keeps track of current clock rate that is set */ + bool tuning_done; + bool calibration_done; + u8 saved_tuning_phase; + bool en_auto_cmd21; + struct device_attribute auto_cmd21_attr; + bool is_sdiowakeup_enabled; + bool sdio_pending_processing; + atomic_t controller_clock; + bool use_cdclp533; + bool use_updated_dll_reset; + bool use_14lpp_dll; + bool enhanced_strobe; + bool rclk_delay_fix; + u32 caps_0; + struct sdhci_msm_ice_data ice; + u32 ice_clk_rate; + struct sdhci_msm_pm_qos_group *pm_qos; + int pm_qos_prev_cpu; + struct device_attribute pm_qos_group_enable_attr; + struct device_attribute pm_qos_group_status_attr; + bool pm_qos_group_enable; + struct sdhci_msm_pm_qos_irq pm_qos_irq; + bool tuning_in_progress; + bool mci_removed; + const struct sdhci_msm_offset *offset; + bool core_3_0v_support; + bool pltfm_init_done; +}; + +extern char *saved_command_line; + +void sdhci_msm_pm_qos_irq_init(struct sdhci_host *host); +void sdhci_msm_pm_qos_irq_vote(struct sdhci_host *host); +void sdhci_msm_pm_qos_irq_unvote(struct sdhci_host *host, bool async); + +void sdhci_msm_pm_qos_cpu_init(struct sdhci_host *host, + struct sdhci_msm_pm_qos_latency *latency); +void sdhci_msm_pm_qos_cpu_vote(struct sdhci_host *host, + struct sdhci_msm_pm_qos_latency *latency, int cpu); +bool sdhci_msm_pm_qos_cpu_unvote(struct sdhci_host *host, int cpu, bool async); + + +#endif /* __SDHCI_MSM_H__ */ diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 62d37d2ac557..0033fea0a800 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -30,8 +30,12 @@ #include <linux/mmc/card.h> #include <linux/mmc/sdio.h> #include <linux/mmc/slot-gpio.h> +#include <linux/mmc/sdio.h> + +#include <trace/events/mmc.h> #include "sdhci.h" +#include "cmdq_hci.h" #define DRIVER_NAME "sdhci" @@ -45,6 +49,9 @@ #define MAX_TUNING_LOOP 40 +#define SDHCI_DBG_DUMP_RS_INTERVAL (10 * HZ) +#define SDHCI_DBG_DUMP_RS_BURST 2 + static unsigned int debug_quirks = 0; static unsigned int debug_quirks2; @@ -52,10 +59,13 @@ static void sdhci_finish_data(struct sdhci_host *); static void sdhci_finish_command(struct sdhci_host *); static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode); +static int sdhci_enhanced_strobe(struct mmc_host *mmc); static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable); static int sdhci_pre_dma_transfer(struct sdhci_host *host, struct mmc_data *data); static int sdhci_do_get_cd(struct sdhci_host *host); +static bool sdhci_check_state(struct sdhci_host *); +static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable); #ifdef CONFIG_PM static int sdhci_runtime_pm_get(struct sdhci_host *host); @@ -79,60 +89,102 @@ static void sdhci_runtime_pm_bus_off(struct sdhci_host *host) } #endif +static void sdhci_dump_state(struct sdhci_host *host) +{ + struct mmc_host *mmc = host->mmc; + + #ifdef CONFIG_MMC_CLKGATE + pr_info("%s: clk: %d clk-gated: %d claimer: %s pwr: %d host->irq = %d\n", + mmc_hostname(mmc), host->clock, mmc->clk_gated, + mmc->claimer->comm, host->pwr, + (host->flags & SDHCI_HOST_IRQ_STATUS)); + #else + pr_info("%s: clk: %d claimer: %s pwr: %d\n", + mmc_hostname(mmc), host->clock, + mmc->claimer->comm, host->pwr); + #endif + pr_info("%s: rpmstatus[pltfm](runtime-suspend:usage_count:disable_depth)(%d:%d:%d)\n", + mmc_hostname(mmc), mmc->parent->power.runtime_status, + atomic_read(&mmc->parent->power.usage_count), + mmc->parent->power.disable_depth); +} + static void sdhci_dumpregs(struct sdhci_host *host) { - pr_debug(DRIVER_NAME ": =========== REGISTER DUMP (%s)===========\n", + MMC_TRACE(host->mmc, + "%s: 0x04=0x%08x 0x06=0x%08x 0x0E=0x%08x 0x30=0x%08x 0x34=0x%08x 0x38=0x%08x\n", + __func__, + sdhci_readw(host, SDHCI_BLOCK_SIZE), + sdhci_readw(host, SDHCI_BLOCK_COUNT), + sdhci_readw(host, SDHCI_COMMAND), + sdhci_readl(host, SDHCI_INT_STATUS), + sdhci_readl(host, SDHCI_INT_ENABLE), + sdhci_readl(host, SDHCI_SIGNAL_ENABLE)); + mmc_stop_tracing(host->mmc); + + pr_info(DRIVER_NAME ": =========== REGISTER DUMP (%s)===========\n", mmc_hostname(host->mmc)); - pr_debug(DRIVER_NAME ": Sys addr: 0x%08x | Version: 0x%08x\n", + pr_info(DRIVER_NAME ": Sys addr: 0x%08x | Version: 0x%08x\n", sdhci_readl(host, SDHCI_DMA_ADDRESS), sdhci_readw(host, SDHCI_HOST_VERSION)); - pr_debug(DRIVER_NAME ": Blk size: 0x%08x | Blk cnt: 0x%08x\n", + pr_info(DRIVER_NAME ": Blk size: 0x%08x | Blk cnt: 0x%08x\n", sdhci_readw(host, SDHCI_BLOCK_SIZE), sdhci_readw(host, SDHCI_BLOCK_COUNT)); - pr_debug(DRIVER_NAME ": Argument: 0x%08x | Trn mode: 0x%08x\n", + pr_info(DRIVER_NAME ": Argument: 0x%08x | Trn mode: 0x%08x\n", sdhci_readl(host, SDHCI_ARGUMENT), sdhci_readw(host, SDHCI_TRANSFER_MODE)); - pr_debug(DRIVER_NAME ": Present: 0x%08x | Host ctl: 0x%08x\n", + pr_info(DRIVER_NAME ": Present: 0x%08x | Host ctl: 0x%08x\n", sdhci_readl(host, SDHCI_PRESENT_STATE), sdhci_readb(host, SDHCI_HOST_CONTROL)); - pr_debug(DRIVER_NAME ": Power: 0x%08x | Blk gap: 0x%08x\n", + pr_info(DRIVER_NAME ": Power: 0x%08x | Blk gap: 0x%08x\n", sdhci_readb(host, SDHCI_POWER_CONTROL), sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL)); - pr_debug(DRIVER_NAME ": Wake-up: 0x%08x | Clock: 0x%08x\n", + pr_info(DRIVER_NAME ": Wake-up: 0x%08x | Clock: 0x%08x\n", sdhci_readb(host, SDHCI_WAKE_UP_CONTROL), sdhci_readw(host, SDHCI_CLOCK_CONTROL)); - pr_debug(DRIVER_NAME ": Timeout: 0x%08x | Int stat: 0x%08x\n", + pr_info(DRIVER_NAME ": Timeout: 0x%08x | Int stat: 0x%08x\n", sdhci_readb(host, SDHCI_TIMEOUT_CONTROL), sdhci_readl(host, SDHCI_INT_STATUS)); - pr_debug(DRIVER_NAME ": Int enab: 0x%08x | Sig enab: 0x%08x\n", + pr_info(DRIVER_NAME ": Int enab: 0x%08x | Sig enab: 0x%08x\n", sdhci_readl(host, SDHCI_INT_ENABLE), sdhci_readl(host, SDHCI_SIGNAL_ENABLE)); - pr_debug(DRIVER_NAME ": AC12 err: 0x%08x | Slot int: 0x%08x\n", - sdhci_readw(host, SDHCI_ACMD12_ERR), + pr_info(DRIVER_NAME ": AC12 err: 0x%08x | Slot int: 0x%08x\n", + host->auto_cmd_err_sts, sdhci_readw(host, SDHCI_SLOT_INT_STATUS)); - pr_debug(DRIVER_NAME ": Caps: 0x%08x | Caps_1: 0x%08x\n", + pr_info(DRIVER_NAME ": Caps: 0x%08x | Caps_1: 0x%08x\n", sdhci_readl(host, SDHCI_CAPABILITIES), sdhci_readl(host, SDHCI_CAPABILITIES_1)); - pr_debug(DRIVER_NAME ": Cmd: 0x%08x | Max curr: 0x%08x\n", + pr_info(DRIVER_NAME ": Cmd: 0x%08x | Max curr: 0x%08x\n", sdhci_readw(host, SDHCI_COMMAND), sdhci_readl(host, SDHCI_MAX_CURRENT)); - pr_debug(DRIVER_NAME ": Host ctl2: 0x%08x\n", + pr_info(DRIVER_NAME ": Resp 1: 0x%08x | Resp 0: 0x%08x\n", + sdhci_readl(host, SDHCI_RESPONSE + 0x4), + sdhci_readl(host, SDHCI_RESPONSE)); + pr_info(DRIVER_NAME ": Resp 3: 0x%08x | Resp 2: 0x%08x\n", + sdhci_readl(host, SDHCI_RESPONSE + 0xC), + sdhci_readl(host, SDHCI_RESPONSE + 0x8)); + pr_info(DRIVER_NAME ": Host ctl2: 0x%08x\n", sdhci_readw(host, SDHCI_HOST_CONTROL2)); if (host->flags & SDHCI_USE_ADMA) { if (host->flags & SDHCI_USE_64_BIT_DMA) - pr_debug(DRIVER_NAME ": ADMA Err: 0x%08x | ADMA Ptr: 0x%08x%08x\n", + pr_info(DRIVER_NAME ": ADMA Err: 0x%08x | ADMA Ptr: 0x%08x%08x\n", readl(host->ioaddr + SDHCI_ADMA_ERROR), readl(host->ioaddr + SDHCI_ADMA_ADDRESS_HI), readl(host->ioaddr + SDHCI_ADMA_ADDRESS)); else - pr_debug(DRIVER_NAME ": ADMA Err: 0x%08x | ADMA Ptr: 0x%08x\n", + pr_info(DRIVER_NAME ": ADMA Err: 0x%08x | ADMA Ptr: 0x%08x\n", readl(host->ioaddr + SDHCI_ADMA_ERROR), readl(host->ioaddr + SDHCI_ADMA_ADDRESS)); } - pr_debug(DRIVER_NAME ": ===========================================\n"); + host->mmc->err_occurred = true; + + if (host->ops->dump_vendor_regs) + host->ops->dump_vendor_regs(host); + sdhci_dump_state(host); + pr_info(DRIVER_NAME ": ===========================================\n"); } /*****************************************************************************\ @@ -177,6 +229,7 @@ void sdhci_reset(struct sdhci_host *host, u8 mask) { unsigned long timeout; +retry_reset: sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET); if (mask & SDHCI_RESET_ALL) { @@ -187,19 +240,60 @@ void sdhci_reset(struct sdhci_host *host, u8 mask) } /* Wait max 100 ms */ - timeout = 100; + timeout = 100000; + + if (host->ops->check_power_status && host->pwr && + (mask & SDHCI_RESET_ALL)) + host->ops->check_power_status(host, REQ_BUS_OFF); + + /* clear pending normal/error interrupt status */ + sdhci_writel(host, sdhci_readl(host, SDHCI_INT_STATUS), + SDHCI_INT_STATUS); /* hw clears the bit when it's done */ while (sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask) { if (timeout == 0) { pr_err("%s: Reset 0x%x never completed.\n", mmc_hostname(host->mmc), (int)mask); + MMC_TRACE(host->mmc, "%s: Reset 0x%x never completed\n", + __func__, (int)mask); + if ((host->quirks2 & SDHCI_QUIRK2_USE_RESET_WORKAROUND) + && host->ops->reset_workaround) { + if (!host->reset_wa_applied) { + /* + * apply the workaround and issue + * reset again. + */ + host->ops->reset_workaround(host, 1); + host->reset_wa_applied = 1; + host->reset_wa_cnt++; + goto retry_reset; + } else { + pr_err("%s: Reset 0x%x failed with workaround\n", + mmc_hostname(host->mmc), + (int)mask); + /* clear the workaround */ + host->ops->reset_workaround(host, 0); + host->reset_wa_applied = 0; + } + } + sdhci_dumpregs(host); return; } timeout--; - mdelay(1); + udelay(1); } + + if ((host->quirks2 & SDHCI_QUIRK2_USE_RESET_WORKAROUND) && + host->ops->reset_workaround && host->reset_wa_applied) { + pr_info("%s: Reset 0x%x successful with workaround\n", + mmc_hostname(host->mmc), (int)mask); + /* clear the workaround */ + host->ops->reset_workaround(host, 0); + host->reset_wa_applied = 0; + } + } EXPORT_SYMBOL_GPL(sdhci_reset); @@ -221,6 +315,8 @@ static void sdhci_do_reset(struct sdhci_host *host, u8 mask) /* Resetting the controller clears many */ host->preset_enabled = false; } + if (host->is_crypto_en) + host->crypto_reset_reqd = true; } static void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios); @@ -236,7 +332,7 @@ static void sdhci_init(struct sdhci_host *host, int soft) SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT | SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC | SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END | - SDHCI_INT_RESPONSE; + SDHCI_INT_RESPONSE | SDHCI_INT_AUTO_CMD_ERR; sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); @@ -279,9 +375,12 @@ static void sdhci_led_control(struct led_classdev *led, struct sdhci_host *host = container_of(led, struct sdhci_host, led); unsigned long flags; + if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) + return; + spin_lock_irqsave(&host->lock, flags); - if (host->runtime_suspended) + if (host->runtime_suspended || sdhci_check_state(host)) goto out; if (brightness == LED_OFF) @@ -598,7 +697,10 @@ static void sdhci_adma_table_post(struct sdhci_host *host, void *align; char *buffer; unsigned long flags; - bool has_unaligned; + bool has_unaligned = false; + u32 command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)); + + trace_mmc_adma_table_post(command, data->sg_len); if (data->flags & MMC_DATA_READ) direction = DMA_FROM_DEVICE; @@ -648,6 +750,7 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd) u8 count; struct mmc_data *data = cmd->data; unsigned target_timeout, current_timeout; + u32 curr_clk = 0; /* In KHz */ /* * If the host controller provides us with an incorrect timeout @@ -693,7 +796,14 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd) * (1) / (2) > 2^6 */ count = 0; - current_timeout = (1 << 13) * 1000 / host->timeout_clk; + if (host->quirks2 & SDHCI_QUIRK2_ALWAYS_USE_BASE_CLOCK) { + curr_clk = host->clock / 1000; + if (host->quirks2 & SDHCI_QUIRK2_DIVIDE_TOUT_BY_4) + curr_clk /= 4; + current_timeout = (1 << 13) * 1000 / curr_clk; + } else { + current_timeout = (1 << 13) * 1000 / host->timeout_clk; + } while (current_timeout < target_timeout) { count++; current_timeout <<= 1; @@ -701,10 +811,12 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd) break; } - if (count >= 0xF) { - DBG("%s: Too large timeout 0x%x requested for CMD%d!\n", - mmc_hostname(host->mmc), count, cmd->opcode); - count = 0xE; + if (!(host->quirks2 & SDHCI_QUIRK2_USE_RESERVED_MAX_TIMEOUT)) { + if (count >= 0xF) { + DBG("%s: Too large timeout 0x%x requested for CMD%d!\n", + mmc_hostname(host->mmc), count, cmd->opcode); + count = 0xE; + } } return count; @@ -736,6 +848,17 @@ static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd) } } +static void sdhci_set_blk_size_reg(struct sdhci_host *host, unsigned int blksz, + unsigned int sdma_boundary) +{ + if (host->flags & SDHCI_USE_ADMA) + sdhci_writew(host, SDHCI_MAKE_BLKSZ(0, blksz), + SDHCI_BLOCK_SIZE); + else + sdhci_writew(host, SDHCI_MAKE_BLKSZ(sdma_boundary, blksz), + SDHCI_BLOCK_SIZE); +} + static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) { u8 ctrl; @@ -751,7 +874,7 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) return; /* Sanity checks */ - BUG_ON(data->blksz * data->blocks > 524288); + BUG_ON(data->blksz * data->blocks > host->mmc->max_req_size); BUG_ON(data->blksz > host->mmc->max_blk_size); BUG_ON(data->blocks > 65535); @@ -762,6 +885,10 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) host->flags |= SDHCI_REQ_USE_DMA; + if ((host->quirks2 & SDHCI_QUIRK2_USE_PIO_FOR_EMMC_TUNING) && + cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200) + host->flags &= ~SDHCI_REQ_USE_DMA; + /* * FIXME: This doesn't account for merging when mapping the * scatterlist. @@ -828,6 +955,7 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) if (host->flags & SDHCI_REQ_USE_DMA) { if (host->flags & SDHCI_USE_ADMA) { + trace_mmc_adma_table_pre(cmd->opcode, data->sg_len); ret = sdhci_adma_table_pre(host, data); if (ret) { /* @@ -898,9 +1026,13 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) sdhci_set_transfer_irqs(host); /* Set the DMA boundary value and block size */ - sdhci_writew(host, SDHCI_MAKE_BLKSZ(SDHCI_DEFAULT_BOUNDARY_ARG, - data->blksz), SDHCI_BLOCK_SIZE); + sdhci_set_blk_size_reg(host, data->blksz, SDHCI_DEFAULT_BOUNDARY_ARG); sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT); + MMC_TRACE(host->mmc, + "%s: 0x28=0x%08x 0x3E=0x%08x 0x06=0x%08x\n", __func__, + sdhci_readb(host, SDHCI_HOST_CONTROL), + sdhci_readw(host, SDHCI_HOST_CONTROL2), + sdhci_readw(host, SDHCI_BLOCK_COUNT)); } static void sdhci_set_transfer_mode(struct sdhci_host *host, @@ -942,12 +1074,26 @@ static void sdhci_set_transfer_mode(struct sdhci_host *host, } } - if (data->flags & MMC_DATA_READ) + if (data->flags & MMC_DATA_READ) { mode |= SDHCI_TRNS_READ; + if (host->ops->toggle_cdr) { + if ((cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200) || + (cmd->opcode == MMC_SEND_TUNING_BLOCK_HS400) || + (cmd->opcode == MMC_SEND_TUNING_BLOCK)) + host->ops->toggle_cdr(host, false); + else + host->ops->toggle_cdr(host, true); + } + } + if (host->ops->toggle_cdr && (data->flags & MMC_DATA_WRITE)) + host->ops->toggle_cdr(host, false); if (host->flags & SDHCI_REQ_USE_DMA) mode |= SDHCI_TRNS_DMA; sdhci_writew(host, mode, SDHCI_TRANSFER_MODE); + MMC_TRACE(host->mmc, "%s: 0x00=0x%08x 0x0C=0x%08x\n", __func__, + sdhci_readw(host, SDHCI_ARGUMENT2), + sdhci_readw(host, SDHCI_TRANSFER_MODE)); } static void sdhci_finish_data(struct sdhci_host *host) @@ -959,6 +1105,8 @@ static void sdhci_finish_data(struct sdhci_host *host) data = host->data; host->data = NULL; + MMC_TRACE(host->mmc, "%s: 0x24=0x%08x\n", __func__, + sdhci_readl(host, SDHCI_PRESENT_STATE)); if (host->flags & SDHCI_REQ_USE_DMA) { if (host->flags & SDHCI_USE_ADMA) sdhci_adma_table_post(host, data); @@ -1017,7 +1165,7 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) WARN_ON(host->cmd); /* Wait max 10 ms */ - timeout = 10; + timeout = 10000; mask = SDHCI_CMD_INHIBIT; if ((cmd->data != NULL) || (cmd->flags & MMC_RSP_BUSY)) @@ -1032,13 +1180,16 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) if (timeout == 0) { pr_err("%s: Controller never released " "inhibit bit(s).\n", mmc_hostname(host->mmc)); + MMC_TRACE(host->mmc, + "%s :Controller never released inhibit bit(s)\n", + __func__); sdhci_dumpregs(host); cmd->error = -EIO; tasklet_schedule(&host->finish_tasklet); return; } timeout--; - mdelay(1); + udelay(1); } timeout = jiffies; @@ -1084,7 +1235,15 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200) flags |= SDHCI_CMD_DATA; + if (cmd->data) + host->data_start_time = ktime_get(); + trace_mmc_cmd_rw_start(cmd->opcode, cmd->arg, cmd->flags); sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND); + MMC_TRACE(host->mmc, + "%s: updated 0x8=0x%08x 0xC=0x%08x 0xE=0x%08x\n", __func__, + sdhci_readl(host, SDHCI_ARGUMENT), + sdhci_readw(host, SDHCI_TRANSFER_MODE), + sdhci_readw(host, SDHCI_COMMAND)); } EXPORT_SYMBOL_GPL(sdhci_send_command); @@ -1105,15 +1264,20 @@ static void sdhci_finish_command(struct sdhci_host *host) sdhci_readb(host, SDHCI_RESPONSE + (3-i)*4-1); } + MMC_TRACE(host->mmc, + "%s: resp 0: 0x%08x resp 1: 0x%08x resp 2: 0x%08x resp 3: 0x%08x\n", + __func__, host->cmd->resp[0], host->cmd->resp[1], + host->cmd->resp[2], host->cmd->resp[3]); } else { host->cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE); + MMC_TRACE(host->mmc, "%s: resp 0: 0x%08x\n", + __func__, host->cmd->resp[0]); } } - host->cmd->error = 0; - /* Finished CMD23, now send actual command. */ if (host->cmd == host->mrq->sbc) { + host->cmd->error = 0; host->cmd = NULL; sdhci_send_command(host, host->mrq->cmd); } else { @@ -1173,7 +1337,8 @@ void sdhci_set_clock(struct sdhci_host *host, unsigned int clock) host->mmc->actual_clock = 0; - sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL); + if (host->clock) + sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL); if (host->quirks2 & SDHCI_QUIRK2_NEED_DELAY_AFTER_INT_CLK_RST) mdelay(1); @@ -1257,6 +1422,10 @@ void sdhci_set_clock(struct sdhci_host *host, unsigned int clock) clock_set: if (real_div) host->mmc->actual_clock = (host->max_clk * clk_mul) / real_div; + + if (host->quirks2 & SDHCI_QUIRK2_ALWAYS_USE_BASE_CLOCK) + div = 0; + clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT; clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN) << SDHCI_DIVIDER_HI_SHIFT; @@ -1264,19 +1433,19 @@ clock_set: sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); /* Wait max 20 ms */ - timeout = 20; + timeout = 20000; while (!((clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL)) & SDHCI_CLOCK_INT_STABLE)) { if (timeout == 0) { pr_err("%s: Internal clock never " "stabilised.\n", mmc_hostname(host->mmc)); + MMC_TRACE(host->mmc, + "%s: Internal clock never stabilised.\n", __func__); sdhci_dumpregs(host); return; } timeout--; - spin_unlock_irq(&host->lock); - usleep_range(900, 1100); - spin_lock_irq(&host->lock); + udelay(1); } clk |= SDHCI_CLOCK_CARD_EN; @@ -1330,6 +1499,8 @@ static void sdhci_set_power(struct sdhci_host *host, unsigned char mode, if (pwr == 0) { sdhci_writeb(host, 0, SDHCI_POWER_CONTROL); + if (host->ops->check_power_status) + host->ops->check_power_status(host, REQ_BUS_OFF); if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON) sdhci_runtime_pm_bus_off(host); vdd = 0; @@ -1338,20 +1509,27 @@ static void sdhci_set_power(struct sdhci_host *host, unsigned char mode, * Spec says that we should clear the power reg before setting * a new value. Some controllers don't seem to like this though. */ - if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE)) + if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE)) { sdhci_writeb(host, 0, SDHCI_POWER_CONTROL); - + if (host->ops->check_power_status) + host->ops->check_power_status(host, REQ_BUS_OFF); + } /* * At least the Marvell CaFe chip gets confused if we set the * voltage and set turn on power at the same time, so set the * voltage first. */ - if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER) + if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER) { sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL); + if (host->ops->check_power_status) + host->ops->check_power_status(host, REQ_BUS_ON); + } pwr |= SDHCI_POWER_ON; sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL); + if (host->ops->check_power_status) + host->ops->check_power_status(host, REQ_BUS_ON); if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON) sdhci_runtime_pm_bus_on(host); @@ -1371,6 +1549,148 @@ static void sdhci_set_power(struct sdhci_host *host, unsigned char mode, * * \*****************************************************************************/ +static int sdhci_enable(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (host->ops->platform_bus_voting) + host->ops->platform_bus_voting(host, 1); + + return 0; +} + +static int sdhci_disable(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (host->ops->platform_bus_voting) + host->ops->platform_bus_voting(host, 0); + + return 0; +} + +static void sdhci_notify_halt(struct mmc_host *mmc, bool halt) +{ + struct sdhci_host *host = mmc_priv(mmc); + + pr_debug("%s: halt notification was sent, halt=%d\n", + mmc_hostname(mmc), halt); + if (host->flags & SDHCI_USE_64_BIT_DMA) { + if (halt) + host->desc_sz = 16; + else + host->desc_sz = SDHCI_ADMA2_64_DESC_SZ; + } +} + +static inline void sdhci_update_power_policy(struct sdhci_host *host, + enum sdhci_power_policy policy) +{ + host->power_policy = policy; +} + +static int sdhci_notify_load(struct mmc_host *mmc, enum mmc_load state) +{ + int err = 0; + struct sdhci_host *host = mmc_priv(mmc); + + switch (state) { + case MMC_LOAD_HIGH: + sdhci_update_power_policy(host, SDHCI_PERFORMANCE_MODE); + break; + case MMC_LOAD_LOW: + sdhci_update_power_policy(host, SDHCI_POWER_SAVE_MODE); + break; + default: + err = -EINVAL; + break; + } + + if (host->ops->notify_load) + err = host->ops->notify_load(host, state); + + return err; +} + +static bool sdhci_check_state(struct sdhci_host *host) +{ + if (!host->clock || !host->pwr) + return true; + else + return false; +} + +static bool sdhci_check_auto_tuning(struct sdhci_host *host, + struct mmc_command *cmd) +{ + if (((cmd->opcode != MMC_READ_SINGLE_BLOCK) && + (cmd->opcode != MMC_READ_MULTIPLE_BLOCK) && + (cmd->opcode != SD_IO_RW_EXTENDED)) || (host->clock < 100000000)) + return false; + else if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200 || + host->mmc->ios.timing == MMC_TIMING_UHS_SDR104) + return true; + else + return false; +} + +static int sdhci_get_tuning_cmd(struct sdhci_host *host) +{ + if (!host->mmc || !host->mmc->card) + return 0; + /* + * If we are here, all conditions have already been true + * and the card can either be an eMMC or SD/SDIO + */ + if (mmc_card_mmc(host->mmc->card)) + return MMC_SEND_TUNING_BLOCK_HS200; + else + return MMC_SEND_TUNING_BLOCK; +} + +static int sdhci_crypto_cfg(struct sdhci_host *host, struct mmc_request *mrq, + u32 slot) +{ + int err = 0; + + if (host->crypto_reset_reqd && host->ops->crypto_engine_reset) { + err = host->ops->crypto_engine_reset(host); + if (err) { + pr_err("%s: crypto reset failed\n", + mmc_hostname(host->mmc)); + goto out; + } + host->crypto_reset_reqd = false; + } + + if (host->ops->crypto_engine_cfg) { + err = host->ops->crypto_engine_cfg(host, mrq, slot); + if (err) { + pr_err("%s: failed to configure crypto\n", + mmc_hostname(host->mmc)); + goto out; + } + } +out: + return err; +} + +static int sdhci_crypto_cfg_end(struct sdhci_host *host, + struct mmc_request *mrq) +{ + int err = 0; + + if (host->ops->crypto_engine_cfg_end) { + err = host->ops->crypto_engine_cfg_end(host, mrq); + if (err) { + pr_err("%s: failed to configure crypto\n", + mmc_hostname(host->mmc)); + return err; + } + } + return 0; +} + static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) { struct sdhci_host *host; @@ -1380,16 +1700,41 @@ static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) host = mmc_priv(mmc); sdhci_runtime_pm_get(host); + if (sdhci_check_state(host)) { + sdhci_dump_state(host); + WARN(1, "sdhci in bad state"); + mrq->cmd->error = -EIO; + if (mrq->data) + mrq->data->error = -EIO; + mmc_request_done(host->mmc, mrq); + sdhci_runtime_pm_put(host); + return; + } - /* Firstly check card presence */ - present = mmc->ops->get_cd(mmc); + /* + * Firstly check card presence from cd-gpio. The return could + * be one of the following possibilities: + * negative: cd-gpio is not available + * zero: cd-gpio is used, and card is removed + * one: cd-gpio is used, and card is present + */ + present = sdhci_do_get_cd(host); + if (present < 0) { + /* If polling, assume that the card is always present. */ + if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) + present = 1; + else + present = sdhci_readl(host, SDHCI_PRESENT_STATE) & + SDHCI_CARD_PRESENT; + } spin_lock_irqsave(&host->lock, flags); WARN_ON(host->mrq != NULL); #ifndef SDHCI_USE_LEDS_CLASS - sdhci_activate_led(host); + if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_LED_CONTROL)) + sdhci_activate_led(host); #endif /* @@ -1409,6 +1754,22 @@ static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) host->mrq->cmd->error = -ENOMEDIUM; tasklet_schedule(&host->finish_tasklet); } else { + if (host->ops->config_auto_tuning_cmd) { + if (sdhci_check_auto_tuning(host, mrq->cmd)) + host->ops->config_auto_tuning_cmd(host, true, + sdhci_get_tuning_cmd(host)); + else + host->ops->config_auto_tuning_cmd(host, false, + sdhci_get_tuning_cmd(host)); + } + + if (host->is_crypto_en) { + spin_unlock_irqrestore(&host->lock, flags); + if (sdhci_crypto_cfg(host, mrq, 0)) + goto end_req; + spin_lock_irqsave(&host->lock, flags); + } + if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23)) sdhci_send_command(host, mrq->sbc); else @@ -1417,6 +1778,16 @@ static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) mmiowb(); spin_unlock_irqrestore(&host->lock, flags); + return; +end_req: + mrq->cmd->error = -EIO; + if (mrq->data) + mrq->data->error = -EIO; + host->mrq = NULL; + MMC_TRACE(host->mmc, "Request failed due to ice config\n"); + sdhci_dumpregs(host); + mmc_request_done(host->mmc, mrq); + sdhci_runtime_pm_put(host); } void sdhci_set_bus_width(struct sdhci_host *host, int width) @@ -1465,38 +1836,50 @@ void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing) } EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling); +void sdhci_cfg_irq(struct sdhci_host *host, bool enable, bool sync) +{ + if (enable && !(host->flags & SDHCI_HOST_IRQ_STATUS)) { + enable_irq(host->irq); + host->flags |= SDHCI_HOST_IRQ_STATUS; + } else if (!enable && (host->flags & SDHCI_HOST_IRQ_STATUS)) { + if (sync) + disable_irq(host->irq); + else + disable_irq_nosync(host->irq); + host->flags &= ~SDHCI_HOST_IRQ_STATUS; + } +} +EXPORT_SYMBOL(sdhci_cfg_irq); + static void sdhci_do_set_ios(struct sdhci_host *host, struct mmc_ios *ios) { unsigned long flags; u8 ctrl; struct mmc_host *mmc = host->mmc; - - spin_lock_irqsave(&host->lock, flags); + int ret; if (host->flags & SDHCI_DEVICE_DEAD) { - spin_unlock_irqrestore(&host->lock, flags); if (!IS_ERR(mmc->supply.vmmc) && ios->power_mode == MMC_POWER_OFF) mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0); return; } - /* - * Reset the chip on each power off. - * Should clear out any weird states. - */ - if (ios->power_mode == MMC_POWER_OFF) { - sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); - sdhci_reinit(host); - } - if (host->version >= SDHCI_SPEC_300 && (ios->power_mode == MMC_POWER_UP) && !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) sdhci_enable_preset_value(host, false); - if (!ios->clock || ios->clock != host->clock) { + spin_lock_irqsave(&host->lock, flags); + if (host->mmc && host->mmc->card && + mmc_card_sdio(host->mmc->card)) + sdhci_cfg_irq(host, false, false); + + if (ios->clock && + ((ios->clock != host->clock) || (ios->timing != host->timing))) { + spin_unlock_irqrestore(&host->lock, flags); host->ops->set_clock(host, ios->clock); + spin_lock_irqsave(&host->lock, flags); host->clock = ios->clock; if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK && @@ -1511,8 +1894,45 @@ static void sdhci_do_set_ios(struct sdhci_host *host, struct mmc_ios *ios) host->mmc->max_busy_timeout /= host->timeout_clk; } } + if (ios->clock && host->sdio_irq_async_status) + sdhci_enable_sdio_irq_nolock(host, false); + spin_unlock_irqrestore(&host->lock, flags); - sdhci_set_power(host, ios->power_mode, ios->vdd); + /* + * The controller clocks may be off during power-up and we may end up + * enabling card clock before giving power to the card. Hence, during + * MMC_POWER_UP enable the controller clock and turn-on the regulators. + * The mmc_power_up would provide the necessary delay before turning on + * the clocks to the card. + */ + if (ios->power_mode & MMC_POWER_UP) { + if (host->ops->enable_controller_clock) { + ret = host->ops->enable_controller_clock(host); + if (ret) { + pr_err("%s: enabling controller clock: failed: %d\n", + mmc_hostname(host->mmc), ret); + } else { + sdhci_set_power(host, ios->power_mode, ios->vdd); + } + } + } + + spin_lock_irqsave(&host->lock, flags); + if (!host->clock) { + if (host->mmc && host->mmc->card && + mmc_card_sdio(host->mmc->card)) + sdhci_cfg_irq(host, true, false); + spin_unlock_irqrestore(&host->lock, flags); + return; + } + spin_unlock_irqrestore(&host->lock, flags); + + if (!host->ops->enable_controller_clock && (ios->power_mode & + (MMC_POWER_UP | + MMC_POWER_ON))) + sdhci_set_power(host, ios->power_mode, ios->vdd); + + spin_lock_irqsave(&host->lock, flags); if (host->ops->platform_send_init_74_clocks) host->ops->platform_send_init_74_clocks(host, ios->power_mode); @@ -1580,7 +2000,11 @@ static void sdhci_do_set_ios(struct sdhci_host *host, struct mmc_ios *ios) sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); /* Re-enable SD Clock */ - host->ops->set_clock(host, host->clock); + if (ios->clock) { + spin_unlock_irqrestore(&host->lock, flags); + host->ops->set_clock(host, host->clock); + spin_lock_irqsave(&host->lock, flags); + } } /* Reset SD Clock Enable */ @@ -1607,10 +2031,15 @@ static void sdhci_do_set_ios(struct sdhci_host *host, struct mmc_ios *ios) } /* Re-enable SD Clock */ - host->ops->set_clock(host, host->clock); + if (ios->clock) { + spin_unlock_irqrestore(&host->lock, flags); + host->ops->set_clock(host, host->clock); + spin_lock_irqsave(&host->lock, flags); + } } else sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); + spin_unlock_irqrestore(&host->lock, flags); /* * Some (ENE) controllers go apeshit on some ios operation, * signalling timeout and CRC errors even on CMD0. Resetting @@ -1619,8 +2048,25 @@ static void sdhci_do_set_ios(struct sdhci_host *host, struct mmc_ios *ios) if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS) sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); - mmiowb(); + /* + * Reset the chip on each power off. + * Should clear out any weird states. + */ + if (ios->power_mode == MMC_POWER_OFF) { + sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); + sdhci_reinit(host); + sdhci_set_power(host, ios->power_mode, ios->vdd); + } + if (!ios->clock) + host->ops->set_clock(host, ios->clock); + + spin_lock_irqsave(&host->lock, flags); + if (host->mmc && host->mmc->card && + mmc_card_sdio(host->mmc->card)) + sdhci_cfg_irq(host, true, false); spin_unlock_irqrestore(&host->lock, flags); + + mmiowb(); } static void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) @@ -1732,16 +2178,28 @@ static int sdhci_get_ro(struct mmc_host *mmc) static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable) { - if (!(host->flags & SDHCI_DEVICE_DEAD)) { + u16 ctrl = 0; + + if (host->flags & SDHCI_DEVICE_DEAD) + return; + + if (mmc_card_and_host_support_async_int(host->mmc)) { + ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2); if (enable) - host->ier |= SDHCI_INT_CARD_INT; + ctrl |= SDHCI_CTRL_ASYNC_INT_ENABLE; else - host->ier &= ~SDHCI_INT_CARD_INT; - - sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); - sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); - mmiowb(); + ctrl &= ~SDHCI_CTRL_ASYNC_INT_ENABLE; + sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2); } + + if (enable) + host->ier |= SDHCI_INT_CARD_INT; + else + host->ier &= ~SDHCI_INT_CARD_INT; + + sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); + sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); + mmiowb(); } static void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable) @@ -1784,6 +2242,8 @@ static int sdhci_do_start_signal_voltage_switch(struct sdhci_host *host, /* Set 1.8V Signal Enable in the Host Control2 register to 0 */ ctrl &= ~SDHCI_CTRL_VDD_180; sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2); + if (host->ops->check_power_status) + host->ops->check_power_status(host, REQ_IO_HIGH); if (!IS_ERR(mmc->supply.vqmmc)) { ret = regulator_set_voltage(mmc->supply.vqmmc, 2700000, @@ -1823,6 +2283,8 @@ static int sdhci_do_start_signal_voltage_switch(struct sdhci_host *host, */ ctrl |= SDHCI_CTRL_VDD_180; sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2); + if (host->ops->check_power_status) + host->ops->check_power_status(host, REQ_IO_LOW); /* Some controller need to do more when switching */ if (host->ops->voltage_switch) @@ -1893,6 +2355,19 @@ static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios) return 0; } +static int sdhci_enhanced_strobe(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + int err = 0; + + sdhci_runtime_pm_get(host); + if (host->ops->enhanced_strobe) + err = host->ops->enhanced_strobe(host); + sdhci_runtime_pm_put(host); + + return err; +} + static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) { struct sdhci_host *host = mmc_priv(mmc); @@ -1922,9 +2397,10 @@ static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) switch (host->timing) { /* HS400 tuning is done in HS200 mode */ case MMC_TIMING_MMC_HS400: - err = -EINVAL; - goto out_unlock; - + if (!(mmc->caps2 & MMC_CAP2_HS400_POST_TUNING)) { + err = -EINVAL; + goto out_unlock; + } case MMC_TIMING_MMC_HS200: /* * Periodic re-tuning for HS400 is not expected to be needed, so @@ -1950,7 +2426,13 @@ static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) if (host->ops->platform_execute_tuning) { spin_unlock_irqrestore(&host->lock, flags); + /* + * Make sure re-tuning won't get triggered for the CRC errors + * occurred while executing tuning + */ + mmc_retune_disable(mmc); err = host->ops->platform_execute_tuning(host, opcode); + mmc_retune_enable(mmc); sdhci_runtime_pm_put(host); return err; } @@ -2002,14 +2484,11 @@ static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) */ if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200) { if (mmc->ios.bus_width == MMC_BUS_WIDTH_8) - sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 128), - SDHCI_BLOCK_SIZE); + sdhci_set_blk_size_reg(host, 128, 7); else if (mmc->ios.bus_width == MMC_BUS_WIDTH_4) - sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64), - SDHCI_BLOCK_SIZE); + sdhci_set_blk_size_reg(host, 64, 7); } else { - sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64), - SDHCI_BLOCK_SIZE); + sdhci_set_blk_size_reg(host, 64, 7); } /* @@ -2132,6 +2611,9 @@ static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable) if (host->version < SDHCI_SPEC_300) return; + if (host->quirks2 & SDHCI_QUIRK2_BROKEN_PRESET_VALUE) + return; + /* * We only enable or disable Preset Value if they are not already * enabled or disabled respectively. Otherwise, we bail out. @@ -2169,6 +2651,8 @@ static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq, DMA_TO_DEVICE : DMA_FROM_DEVICE); data->host_cookie = COOKIE_UNMAPPED; } + if (host->ops->post_req) + host->ops->post_req(host, mrq); } static int sdhci_pre_dma_transfer(struct sdhci_host *host, @@ -2205,6 +2689,9 @@ static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq, if (host->flags & SDHCI_REQ_USE_DMA) sdhci_pre_dma_transfer(host, mrq->data); + + if (host->ops->pre_req) + host->ops->pre_req(host, mrq); } static void sdhci_card_event(struct mmc_host *mmc) @@ -2238,7 +2725,29 @@ static void sdhci_card_event(struct mmc_host *mmc) spin_unlock_irqrestore(&host->lock, flags); } +static int sdhci_late_init(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (host->ops->init) + host->ops->init(host); + + return 0; +} + +static void sdhci_force_err_irq(struct mmc_host *mmc, u64 errmask) +{ + struct sdhci_host *host = mmc_priv(mmc); + u16 mask = errmask & 0xFFFF; + + pr_err("%s: Force raise error mask:0x%04x\n", __func__, mask); + sdhci_runtime_pm_get(host); + sdhci_writew(host, mask, SDHCI_SET_INT_ERROR); + sdhci_runtime_pm_put(host); +} + static const struct mmc_host_ops sdhci_ops = { + .init = sdhci_late_init, .request = sdhci_request, .post_req = sdhci_post_req, .pre_req = sdhci_pre_req, @@ -2250,9 +2759,15 @@ static const struct mmc_host_ops sdhci_ops = { .start_signal_voltage_switch = sdhci_start_signal_voltage_switch, .prepare_hs400_tuning = sdhci_prepare_hs400_tuning, .execute_tuning = sdhci_execute_tuning, + .enhanced_strobe = sdhci_enhanced_strobe, .select_drive_strength = sdhci_select_drive_strength, .card_event = sdhci_card_event, .card_busy = sdhci_card_busy, + .enable = sdhci_enable, + .disable = sdhci_disable, + .notify_load = sdhci_notify_load, + .notify_halt = sdhci_notify_halt, + .force_err_irq = sdhci_force_err_irq, }; /*****************************************************************************\ @@ -2304,19 +2819,25 @@ static void sdhci_tasklet_finish(unsigned long param) controllers do not like that. */ sdhci_do_reset(host, SDHCI_RESET_CMD); sdhci_do_reset(host, SDHCI_RESET_DATA); + } else { + if (host->quirks2 & SDHCI_QUIRK2_RDWR_TX_ACTIVE_EOT) + sdhci_reset(host, SDHCI_RESET_DATA); } host->mrq = NULL; host->cmd = NULL; host->data = NULL; + host->auto_cmd_err_sts = 0; #ifndef SDHCI_USE_LEDS_CLASS - sdhci_deactivate_led(host); + if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_LED_CONTROL)) + sdhci_deactivate_led(host); #endif mmiowb(); spin_unlock_irqrestore(&host->lock, flags); + sdhci_crypto_cfg_end(host, mrq); mmc_request_done(host->mmc, mrq); sdhci_runtime_pm_put(host); } @@ -2333,9 +2854,15 @@ static void sdhci_timeout_timer(unsigned long data) if (host->mrq) { pr_err("%s: Timeout waiting for hardware " "interrupt.\n", mmc_hostname(host->mmc)); + MMC_TRACE(host->mmc, "Timeout waiting for h/w interrupt\n"); sdhci_dumpregs(host); if (host->data) { + pr_info("%s: bytes to transfer: %d transferred: %d\n", + mmc_hostname(host->mmc), + (host->data->blksz * host->data->blocks), + (sdhci_readw(host, SDHCI_BLOCK_SIZE) & 0xFFF) * + sdhci_readw(host, SDHCI_BLOCK_COUNT)); host->data->error = -ETIMEDOUT; sdhci_finish_data(host); } else { @@ -2360,23 +2887,63 @@ static void sdhci_timeout_timer(unsigned long data) static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *mask) { + u16 auto_cmd_status; BUG_ON(intmask == 0); if (!host->cmd) { pr_err("%s: Got command interrupt 0x%08x even " "though no command operation was in progress.\n", mmc_hostname(host->mmc), (unsigned)intmask); + MMC_TRACE(host->mmc, + "Got command interrupt 0x%08x even though no command operation was in progress.\n", + (unsigned)intmask); sdhci_dumpregs(host); return; } + trace_mmc_cmd_rw_end(host->cmd->opcode, intmask, + sdhci_readl(host, SDHCI_RESPONSE)); + if (intmask & SDHCI_INT_TIMEOUT) host->cmd->error = -ETIMEDOUT; else if (intmask & (SDHCI_INT_CRC | SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) host->cmd->error = -EILSEQ; + if (intmask & SDHCI_INT_AUTO_CMD_ERR) { + auto_cmd_status = host->auto_cmd_err_sts; + pr_err_ratelimited("%s: %s: AUTO CMD err sts 0x%08x\n", + mmc_hostname(host->mmc), __func__, auto_cmd_status); + if (auto_cmd_status & (SDHCI_AUTO_CMD12_NOT_EXEC | + SDHCI_AUTO_CMD_INDEX_ERR | + SDHCI_AUTO_CMD_ENDBIT_ERR)) + host->cmd->error = -EIO; + else if (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT_ERR) + host->cmd->error = -ETIMEDOUT; + else if (auto_cmd_status & SDHCI_AUTO_CMD_CRC_ERR) + host->cmd->error = -EILSEQ; + } + if (host->cmd->error) { + /* + * If this command initiates a data phase and a response + * CRC error is signalled, the card can start transferring + * data - the card may have received the command without + * error. We must not terminate the mmc_request early. + * + * If the card did not receive the command or returned an + * error which prevented it sending data, the data phase + * will time out. + * + * Even in case of cmd INDEX OR ENDBIT error we + * handle it the same way. + */ + if (host->cmd->data && + (((intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) == + SDHCI_INT_CRC) || (host->cmd->error == -EILSEQ))) { + host->cmd = NULL; + return; + } tasklet_schedule(&host->finish_tasklet); return; } @@ -2450,13 +3017,17 @@ static void sdhci_adma_show_error(struct sdhci_host *host) { } static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) { u32 command; + bool pr_msg = false; BUG_ON(intmask == 0); + command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)); + trace_mmc_data_rw_end(command, intmask); + /* CMD19 generates _only_ Buffer Read Ready interrupt */ if (intmask & SDHCI_INT_DATA_AVAIL) { - command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)); - if (command == MMC_SEND_TUNING_BLOCK || - command == MMC_SEND_TUNING_BLOCK_HS200) { + if (!(host->quirks2 & SDHCI_QUIRK2_NON_STANDARD_TUNING) && + (command == MMC_SEND_TUNING_BLOCK || + command == MMC_SEND_TUNING_BLOCK_HS200)) { host->tuning_done = 1; wake_up(&host->buf_ready_int); return; @@ -2487,11 +3058,17 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) host->busy_handle = 1; return; } + if (host->quirks2 & + SDHCI_QUIRK2_IGNORE_DATATOUT_FOR_R1BCMD) + return; } pr_err("%s: Got data interrupt 0x%08x even " "though no data operation was in progress.\n", mmc_hostname(host->mmc), (unsigned)intmask); + MMC_TRACE(host->mmc, + "Got data interrupt 0x%08x even though no data operation was in progress.\n", + (unsigned)intmask); sdhci_dumpregs(host); return; @@ -2502,8 +3079,7 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) else if (intmask & SDHCI_INT_DATA_END_BIT) host->data->error = -EILSEQ; else if ((intmask & SDHCI_INT_DATA_CRC) && - SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)) - != MMC_BUS_TEST_R) + (command != MMC_BUS_TEST_R)) host->data->error = -EILSEQ; else if (intmask & SDHCI_INT_ADMA_ERROR) { pr_err("%s: ADMA error\n", mmc_hostname(host->mmc)); @@ -2512,10 +3088,34 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) if (host->ops->adma_workaround) host->ops->adma_workaround(host, intmask); } - - if (host->data->error) + if (host->data->error) { + if (intmask & (SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT + | SDHCI_INT_DATA_END_BIT)) { + command = SDHCI_GET_CMD(sdhci_readw(host, + SDHCI_COMMAND)); + if ((command != MMC_SEND_TUNING_BLOCK_HS200) && + (command != MMC_SEND_TUNING_BLOCK)) + pr_msg = true; + } else { + pr_msg = true; + } + if (pr_msg && __ratelimit(&host->dbg_dump_rs)) { + pr_err("%s: data txfr (0x%08x) error: %d after %lld ms\n", + mmc_hostname(host->mmc), intmask, + host->data->error, ktime_to_ms(ktime_sub( + ktime_get(), host->data_start_time))); + MMC_TRACE(host->mmc, + "data txfr (0x%08x) error: %d after %lld ms\n", + intmask, host->data->error, + ktime_to_ms(ktime_sub(ktime_get(), + host->data_start_time))); + + if (!host->mmc->sdr104_wa || + (host->mmc->ios.timing != MMC_TIMING_UHS_SDR104)) + sdhci_dumpregs(host); + } sdhci_finish_data(host); - else { + } else { if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL)) sdhci_transfer_pio(host); @@ -2561,6 +3161,58 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) } } +#ifdef CONFIG_MMC_CQ_HCI +static int sdhci_get_cmd_err(u32 intmask) +{ + if (intmask & SDHCI_INT_TIMEOUT) + return -ETIMEDOUT; + else if (intmask & (SDHCI_INT_CRC | SDHCI_INT_END_BIT | + SDHCI_INT_INDEX)) + return -EILSEQ; + return 0; +} + +static int sdhci_get_data_err(u32 intmask) +{ + if (intmask & SDHCI_INT_DATA_TIMEOUT) + return -ETIMEDOUT; + else if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC)) + return -EILSEQ; + else if (intmask & SDHCI_INT_ADMA_ERROR) + return -EIO; + return 0; +} + +static irqreturn_t sdhci_cmdq_irq(struct sdhci_host *host, u32 intmask) +{ + int err = 0; + u32 mask = 0; + irqreturn_t ret; + + if (intmask & SDHCI_INT_CMD_MASK) + err = sdhci_get_cmd_err(intmask); + else if (intmask & SDHCI_INT_DATA_MASK) + err = sdhci_get_data_err(intmask); + + ret = cmdq_irq(host->mmc, err); + if (err) { + /* Clear the error interrupts */ + mask = intmask & SDHCI_INT_ERROR_MASK; + sdhci_writel(host, mask, SDHCI_INT_STATUS); + } + return ret; + +} + +#else +static irqreturn_t sdhci_cmdq_irq(struct sdhci_host *host, u32 intmask) +{ + pr_err("%s: Received cmdq-irq when disabled !!!!\n", + mmc_hostname(host->mmc)); + return IRQ_NONE; +} +#endif + static irqreturn_t sdhci_irq(int irq, void *dev_id) { irqreturn_t result = IRQ_NONE; @@ -2575,6 +3227,31 @@ static irqreturn_t sdhci_irq(int irq, void *dev_id) return IRQ_NONE; } + if (!host->clock && host->mmc->card && + mmc_card_sdio(host->mmc->card)) { + if (!mmc_card_and_host_support_async_int(host->mmc)) { + spin_unlock(&host->lock); + return IRQ_NONE; + } + /* + * async card interrupt is level sensitive and received + * when clocks are off. + * If sdio card has asserted async interrupt, in that + * case we need to disable host->irq. + * Later we can disable card interrupt and re-enable + * host->irq. + */ + + pr_debug("%s: %s: sdio_async intr. received\n", + mmc_hostname(host->mmc), __func__); + sdhci_cfg_irq(host, false, false); + host->sdio_irq_async_status = true; + host->thread_isr |= SDHCI_INT_CARD_INT; + result = IRQ_WAKE_THREAD; + spin_unlock(&host->lock); + return result; + } + intmask = sdhci_readl(host, SDHCI_INT_STATUS); if (!intmask || intmask == 0xffffffff) { result = IRQ_NONE; @@ -2582,6 +3259,22 @@ static irqreturn_t sdhci_irq(int irq, void *dev_id) } do { + if (host->mmc->card && mmc_card_cmdq(host->mmc->card) && + !mmc_host_halt(host->mmc) && !mmc_host_cq_disable(host->mmc)) { + pr_debug("*** %s: cmdq intr: 0x%08x\n", + mmc_hostname(host->mmc), + intmask); + result = sdhci_cmdq_irq(host, intmask); + if (result == IRQ_HANDLED) + goto out; + } + + MMC_TRACE(host->mmc, + "%s: intmask: 0x%x\n", __func__, intmask); + + if (intmask & SDHCI_INT_AUTO_CMD_ERR) + host->auto_cmd_err_sts = sdhci_readw(host, + SDHCI_AUTO_CMD_ERR); /* Clear selected interrupts. */ mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK | SDHCI_INT_BUS_POWER); @@ -2620,12 +3313,20 @@ static irqreturn_t sdhci_irq(int irq, void *dev_id) result = IRQ_WAKE_THREAD; } - if (intmask & SDHCI_INT_CMD_MASK) + if (intmask & SDHCI_INT_CMD_MASK) { + if ((host->quirks2 & SDHCI_QUIRK2_SLOW_INT_CLR) && + (host->clock <= 400000)) + udelay(40); sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask); + } - if (intmask & SDHCI_INT_DATA_MASK) + if (intmask & SDHCI_INT_DATA_MASK) { + if ((host->quirks2 & SDHCI_QUIRK2_SLOW_INT_CLR) && + (host->clock <= 400000)) + udelay(40); sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK); + } if (intmask & SDHCI_INT_BUS_POWER) pr_err("%s: Card is consuming too much power!\n", @@ -2659,6 +3360,8 @@ out: if (unexpected) { pr_err("%s: Unexpected interrupt 0x%08x.\n", mmc_hostname(host->mmc), unexpected); + MMC_TRACE(host->mmc, "Unexpected interrupt 0x%08x.\n", + unexpected); sdhci_dumpregs(host); } @@ -2685,8 +3388,11 @@ static irqreturn_t sdhci_thread_irq(int irq, void *dev_id) sdio_run_irqs(host->mmc); spin_lock_irqsave(&host->lock, flags); - if (host->flags & SDHCI_SDIO_IRQ_ENABLED) + if (host->flags & SDHCI_SDIO_IRQ_ENABLED) { + if (host->sdio_irq_async_status) + host->sdio_irq_async_status = false; sdhci_enable_sdio_irq_nolock(host, true); + } spin_unlock_irqrestore(&host->lock, flags); } @@ -2903,11 +3609,255 @@ struct sdhci_host *sdhci_alloc_host(struct device *dev, host->mmc_host_ops = sdhci_ops; mmc->ops = &host->mmc_host_ops; + spin_lock_init(&host->lock); + ratelimit_state_init(&host->dbg_dump_rs, SDHCI_DBG_DUMP_RS_INTERVAL, + SDHCI_DBG_DUMP_RS_BURST); + return host; } EXPORT_SYMBOL_GPL(sdhci_alloc_host); +#ifdef CONFIG_MMC_CQ_HCI +static void sdhci_cmdq_set_transfer_params(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + u8 ctrl; + + if (host->version >= SDHCI_SPEC_200) { + ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); + ctrl &= ~SDHCI_CTRL_DMA_MASK; + if (host->flags & SDHCI_USE_64_BIT_DMA) + ctrl |= SDHCI_CTRL_ADMA64; + else + ctrl |= SDHCI_CTRL_ADMA32; + sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); + } + if (host->ops->toggle_cdr && !host->cdr_support) + host->ops->toggle_cdr(host, false); +} + +static void sdhci_cmdq_clear_set_irqs(struct mmc_host *mmc, bool clear) +{ + struct sdhci_host *host = mmc_priv(mmc); + u32 ier = 0; + + ier &= ~SDHCI_INT_ALL_MASK; + + if (clear) { + ier = SDHCI_INT_CMDQ_EN | SDHCI_INT_ERROR_MASK; + sdhci_writel(host, ier, SDHCI_INT_ENABLE); + sdhci_writel(host, ier, SDHCI_SIGNAL_ENABLE); + } else { + ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT | + SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT | + SDHCI_INT_INDEX | SDHCI_INT_END_BIT | + SDHCI_INT_CRC | SDHCI_INT_TIMEOUT | + SDHCI_INT_DATA_END | SDHCI_INT_RESPONSE | + SDHCI_INT_AUTO_CMD_ERR; + sdhci_writel(host, ier, SDHCI_INT_ENABLE); + sdhci_writel(host, ier, SDHCI_SIGNAL_ENABLE); + } +} + +static void sdhci_cmdq_set_data_timeout(struct mmc_host *mmc, u32 val) +{ + struct sdhci_host *host = mmc_priv(mmc); + + sdhci_writeb(host, val, SDHCI_TIMEOUT_CONTROL); +} + +static void sdhci_cmdq_dump_vendor_regs(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + + sdhci_dumpregs(host); +} + +static int sdhci_cmdq_init(struct sdhci_host *host, struct mmc_host *mmc, + bool dma64) +{ + return cmdq_init(host->cq_host, mmc, dma64); +} + +static void sdhci_cmdq_set_block_size(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + + sdhci_set_blk_size_reg(host, 512, 0); +} + +static void sdhci_enhanced_strobe_mask(struct mmc_host *mmc, bool set) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (host->ops->enhanced_strobe_mask) + host->ops->enhanced_strobe_mask(host, set); +} + +static void sdhci_cmdq_clear_set_dumpregs(struct mmc_host *mmc, bool set) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (host->ops->clear_set_dumpregs) + host->ops->clear_set_dumpregs(host, set); +} +static int sdhci_cmdq_crypto_cfg(struct mmc_host *mmc, + struct mmc_request *mrq, u32 slot, u64 *ice_ctx) +{ + struct sdhci_host *host = mmc_priv(mmc); + int err = 0; + + if (!host->is_crypto_en) + return 0; + + if (host->crypto_reset_reqd && host->ops->crypto_engine_reset) { + err = host->ops->crypto_engine_reset(host); + if (err) { + pr_err("%s: crypto reset failed\n", + mmc_hostname(host->mmc)); + goto out; + } + host->crypto_reset_reqd = false; + } + + if (host->ops->crypto_engine_cmdq_cfg) { + err = host->ops->crypto_engine_cmdq_cfg(host, mrq, + slot, ice_ctx); + if (err) { + pr_err("%s: failed to configure crypto\n", + mmc_hostname(host->mmc)); + goto out; + } + } +out: + return err; +} + +static int sdhci_cmdq_crypto_cfg_end(struct mmc_host *mmc, + struct mmc_request *mrq) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (!host->is_crypto_en) + return 0; + + return sdhci_crypto_cfg_end(host, mrq); +} + +static void sdhci_cmdq_crypto_cfg_reset(struct mmc_host *mmc, unsigned int slot) +{ + struct sdhci_host *host = mmc_priv(mmc); + + if (!host->is_crypto_en) + return; + + if (host->ops->crypto_cfg_reset) + host->ops->crypto_cfg_reset(host, slot); +} + +static void sdhci_cmdq_post_cqe_halt(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + + sdhci_writel(host, sdhci_readl(host, SDHCI_INT_ENABLE) | + SDHCI_INT_RESPONSE, SDHCI_INT_ENABLE); + sdhci_writel(host, SDHCI_INT_RESPONSE, SDHCI_INT_STATUS); +} +#else +static void sdhci_cmdq_set_transfer_params(struct mmc_host *mmc) +{ + +} +static void sdhci_cmdq_clear_set_irqs(struct mmc_host *mmc, bool clear) +{ + +} + +static void sdhci_cmdq_set_data_timeout(struct mmc_host *mmc, u32 val) +{ + +} + +static void sdhci_cmdq_dump_vendor_regs(struct mmc_host *mmc) +{ + +} + +static int sdhci_cmdq_init(struct sdhci_host *host, struct mmc_host *mmc, + bool dma64) +{ + return -ENOSYS; +} + +static void sdhci_cmdq_set_block_size(struct mmc_host *mmc) +{ + +} + +static void sdhci_enhanced_strobe_mask(struct mmc_host *mmc, bool set) +{ + +} + +static void sdhci_cmdq_clear_set_dumpregs(struct mmc_host *mmc, bool set) +{ + +} +static int sdhci_cmdq_crypto_cfg(struct mmc_host *mmc, + struct mmc_request *mrq, u32 slot, u64 *ice_ctx) +{ + return 0; +} + +static int sdhci_cmdq_crypto_cfg_end(struct mmc_host *mmc, + struct mmc_request *mrq) +{ + return 0; +} + +static void sdhci_cmdq_crypto_cfg_reset(struct mmc_host *mmc, unsigned int slot) +{ + +} +static void sdhci_cmdq_post_cqe_halt(struct mmc_host *mmc) +{ +} +#endif + +static const struct cmdq_host_ops sdhci_cmdq_ops = { + .clear_set_irqs = sdhci_cmdq_clear_set_irqs, + .set_data_timeout = sdhci_cmdq_set_data_timeout, + .dump_vendor_regs = sdhci_cmdq_dump_vendor_regs, + .set_block_size = sdhci_cmdq_set_block_size, + .clear_set_dumpregs = sdhci_cmdq_clear_set_dumpregs, + .enhanced_strobe_mask = sdhci_enhanced_strobe_mask, + .crypto_cfg = sdhci_cmdq_crypto_cfg, + .crypto_cfg_end = sdhci_cmdq_crypto_cfg_end, + .crypto_cfg_reset = sdhci_cmdq_crypto_cfg_reset, + .post_cqe_halt = sdhci_cmdq_post_cqe_halt, + .set_transfer_params = sdhci_cmdq_set_transfer_params, +}; + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT +static int sdhci_is_adma2_64bit(struct sdhci_host *host) +{ + u32 caps; + + caps = (host->quirks & SDHCI_QUIRK_MISSING_CAPS) ? host->caps : + sdhci_readl(host, SDHCI_CAPABILITIES); + + if (caps & SDHCI_CAN_64BIT) + return 1; + return 0; +} +#else +static int sdhci_is_adma2_64bit(struct sdhci_host *host) +{ + return 0; +} +#endif + int sdhci_add_host(struct sdhci_host *host) { struct mmc_host *mmc; @@ -2980,7 +3930,7 @@ int sdhci_add_host(struct sdhci_host *host) * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to * implement. */ - if (sdhci_readl(host, SDHCI_CAPABILITIES) & SDHCI_CAN_64BIT) + if (sdhci_is_adma2_64bit(host)) host->flags |= SDHCI_USE_64_BIT_DMA; if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) { @@ -3135,6 +4085,9 @@ int sdhci_add_host(struct sdhci_host *host) mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23; mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD; + if (caps[0] & SDHCI_CAN_ASYNC_INT) + mmc->caps2 |= MMC_CAP2_ASYNC_SDIO_IRQ_4BIT_MODE; + if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12) host->flags |= SDHCI_AUTO_CMD12; @@ -3167,7 +4120,8 @@ int sdhci_add_host(struct sdhci_host *host) if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) && !(mmc->caps & MMC_CAP_NONREMOVABLE) && - IS_ERR_VALUE(mmc_gpio_get_cd(host->mmc))) + (IS_ERR_VALUE(mmc_gpio_get_cd(host->mmc)) && + !(mmc->caps2 & MMC_CAP2_NONHOTPLUG))) mmc->caps |= MMC_CAP_NEEDS_POLL; /* If there are external regulators, get them */ @@ -3264,10 +4218,15 @@ int sdhci_add_host(struct sdhci_host *host) * value. */ max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT); - if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) { - int curr = regulator_get_current_limit(mmc->supply.vmmc); - if (curr > 0) { + if (!max_current_caps) { + u32 curr = 0; + + if (!IS_ERR(mmc->supply.vmmc)) + curr = regulator_get_current_limit(mmc->supply.vmmc); + else if (host->ops->get_current_limit) + curr = host->ops->get_current_limit(host); + if (curr > 0) { /* convert to SDHCI_MAX_CURRENT format */ curr = curr/1000; /* convert to mA */ curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER; @@ -3332,8 +4291,6 @@ int sdhci_add_host(struct sdhci_host *host) return -ENODEV; } - spin_lock_init(&host->lock); - /* * Maximum number of segments. Depends on if the hardware * can do scatter/gather or not. @@ -3399,6 +4356,8 @@ int sdhci_add_host(struct sdhci_host *host) init_waitqueue_head(&host->buf_ready_int); + host->flags |= SDHCI_HOST_IRQ_STATUS; + sdhci_init(host, 0); ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, @@ -3414,33 +4373,54 @@ int sdhci_add_host(struct sdhci_host *host) #endif #ifdef SDHCI_USE_LEDS_CLASS - snprintf(host->led_name, sizeof(host->led_name), - "%s::", mmc_hostname(mmc)); - host->led.name = host->led_name; - host->led.brightness = LED_OFF; - host->led.default_trigger = mmc_hostname(mmc); - host->led.brightness_set = sdhci_led_control; - - ret = led_classdev_register(mmc_dev(mmc), &host->led); - if (ret) { - pr_err("%s: Failed to register LED device: %d\n", - mmc_hostname(mmc), ret); - goto reset; + if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_LED_CONTROL)) { + snprintf(host->led_name, sizeof(host->led_name), + "%s::", mmc_hostname(mmc)); + host->led.name = host->led_name; + host->led.brightness = LED_OFF; + host->led.default_trigger = mmc_hostname(mmc); + host->led.brightness_set = sdhci_led_control; + + ret = led_classdev_register(mmc_dev(mmc), &host->led); + if (ret) { + pr_err("%s: Failed to register LED device: %d\n", + mmc_hostname(mmc), ret); + goto reset; + } } #endif mmiowb(); - mmc_add_host(mmc); + if (host->quirks2 & SDHCI_QUIRK2_IGN_DATA_END_BIT_ERROR) { + host->ier = (host->ier & ~SDHCI_INT_DATA_END_BIT); + sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); + sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); + } - pr_info("%s: SDHCI controller on %s [%s] using %s\n", - mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)), + if (mmc->caps2 & MMC_CAP2_CMD_QUEUE) { + bool dma64 = (host->flags & SDHCI_USE_64_BIT_DMA) ? + true : false; + ret = sdhci_cmdq_init(host, mmc, dma64); + if (ret) + pr_err("%s: CMDQ init: failed (%d)\n", + mmc_hostname(host->mmc), ret); + else + host->cq_host->ops = &sdhci_cmdq_ops; + } + + pr_info("%s: SDHCI controller on %s [%s] using %s in %s mode\n", + mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)), (host->flags & SDHCI_USE_ADMA) ? - (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" : - (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO"); + ((host->flags & SDHCI_USE_64_BIT_DMA) ? + "64-bit ADMA" : "32-bit ADMA") : + ((host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO"), + ((mmc->caps2 & MMC_CAP2_CMD_QUEUE) && !ret) ? + "CMDQ" : "legacy"); sdhci_enable_card_detection(host); + mmc_add_host(mmc); return 0; #ifdef SDHCI_USE_LEDS_CLASS @@ -3481,10 +4461,11 @@ void sdhci_remove_host(struct sdhci_host *host, int dead) sdhci_disable_card_detection(host); - mmc_remove_host(mmc); + mmc_remove_host(host->mmc); #ifdef SDHCI_USE_LEDS_CLASS - led_classdev_unregister(&host->led); + if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_LED_CONTROL)) + led_classdev_unregister(&host->led); #endif if (!dead) diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index 0115e9907bf8..300be7fd0f24 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -17,7 +17,7 @@ #include <linux/compiler.h> #include <linux/types.h> #include <linux/io.h> - +#include <linux/ratelimit.h> #include <linux/mmc/host.h> /* @@ -137,22 +137,32 @@ #define SDHCI_INT_DATA_CRC 0x00200000 #define SDHCI_INT_DATA_END_BIT 0x00400000 #define SDHCI_INT_BUS_POWER 0x00800000 -#define SDHCI_INT_ACMD12ERR 0x01000000 +#define SDHCI_INT_AUTO_CMD_ERR 0x01000000 #define SDHCI_INT_ADMA_ERROR 0x02000000 #define SDHCI_INT_NORMAL_MASK 0x00007FFF #define SDHCI_INT_ERROR_MASK 0xFFFF8000 #define SDHCI_INT_CMD_MASK (SDHCI_INT_RESPONSE | SDHCI_INT_TIMEOUT | \ - SDHCI_INT_CRC | SDHCI_INT_END_BIT | SDHCI_INT_INDEX) + SDHCI_INT_CRC | SDHCI_INT_END_BIT | SDHCI_INT_INDEX | \ + SDHCI_INT_AUTO_CMD_ERR) + #define SDHCI_INT_DATA_MASK (SDHCI_INT_DATA_END | SDHCI_INT_DMA_END | \ SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL | \ SDHCI_INT_DATA_TIMEOUT | SDHCI_INT_DATA_CRC | \ SDHCI_INT_DATA_END_BIT | SDHCI_INT_ADMA_ERROR | \ SDHCI_INT_BLK_GAP) + +#define SDHCI_INT_CMDQ_EN (0x1 << 14) #define SDHCI_INT_ALL_MASK ((unsigned int)-1) -#define SDHCI_ACMD12_ERR 0x3C +#define SDHCI_AUTO_CMD_ERR 0x3C +#define SDHCI_AUTO_CMD12_NOT_EXEC 0x0001 +#define SDHCI_AUTO_CMD_TIMEOUT_ERR 0x0002 +#define SDHCI_AUTO_CMD_CRC_ERR 0x0004 +#define SDHCI_AUTO_CMD_ENDBIT_ERR 0x0008 +#define SDHCI_AUTO_CMD_INDEX_ERR 0x0010 +#define SDHCI_AUTO_CMD12_NOT_ISSUED 0x0080 #define SDHCI_HOST_CONTROL2 0x3E #define SDHCI_CTRL_UHS_MASK 0x0007 @@ -170,6 +180,7 @@ #define SDHCI_CTRL_DRV_TYPE_D 0x0030 #define SDHCI_CTRL_EXEC_TUNING 0x0040 #define SDHCI_CTRL_TUNED_CLK 0x0080 +#define SDHCI_CTRL_ASYNC_INT_ENABLE 0x4000 #define SDHCI_CTRL_PRESET_VAL_ENABLE 0x8000 #define SDHCI_CAPABILITIES 0x40 @@ -190,6 +201,7 @@ #define SDHCI_CAN_VDD_300 0x02000000 #define SDHCI_CAN_VDD_180 0x04000000 #define SDHCI_CAN_64BIT 0x10000000 +#define SDHCI_CAN_ASYNC_INT 0x20000000 #define SDHCI_SUPPORT_SDR50 0x00000001 #define SDHCI_SUPPORT_SDR104 0x00000002 @@ -320,6 +332,12 @@ enum sdhci_cookie { COOKIE_GIVEN, }; +enum sdhci_power_policy { + SDHCI_PERFORMANCE_MODE, + SDHCI_POWER_SAVE_MODE, + SDHCI_POWER_POLICY_NUM /* Always keep this one last */ +}; + struct sdhci_host { /* Data set by hardware interface driver */ const char *hw_name; /* Hardware bus name */ @@ -423,6 +441,84 @@ struct sdhci_host { */ #define SDHCI_QUIRK2_NEED_DELAY_AFTER_INT_CLK_RST (1<<16) +/* + * Read Transfer Active/ Write Transfer Active may be not + * de-asserted after end of transaction. Issue reset for DAT line. + */ +#define SDHCI_QUIRK2_RDWR_TX_ACTIVE_EOT (1<<17) +/* + * Slow interrupt clearance at 400KHz may cause + * host controller driver interrupt handler to + * be called twice. + */ +#define SDHCI_QUIRK2_SLOW_INT_CLR (1<<18) + +/* + * If the base clock can be scalable, then there should be no further + * clock dividing as the input clock itself will be scaled down to + * required frequency. + */ +#define SDHCI_QUIRK2_ALWAYS_USE_BASE_CLOCK (1<<19) + +/* + * Ignore data timeout error for R1B commands as there will be no + * data associated and the busy timeout value for these commands + * could be lager than the maximum timeout value that controller + * can handle. + */ +#define SDHCI_QUIRK2_IGNORE_DATATOUT_FOR_R1BCMD (1<<20) + +/* + * The preset value registers are not properly initialized by + * some hardware and hence preset value must not be enabled for + * such controllers. + */ +#define SDHCI_QUIRK2_BROKEN_PRESET_VALUE (1<<21) +/* + * Some controllers define the usage of 0xF in data timeout counter + * register (0x2E) which is actually a reserved bit as per + * specification. + */ +#define SDHCI_QUIRK2_USE_RESERVED_MAX_TIMEOUT (1<<22) +/* + * This is applicable for controllers that advertize timeout clock + * value in capabilities register (bit 5-0) as just 50MHz whereas the + * base clock frequency is 200MHz. So, the controller internally + * multiplies the value in timeout control register by 4 with the + * assumption that driver always uses fixed timeout clock value from + * capabilities register to calculate the timeout. But when the driver + * uses SDHCI_QUIRK2_ALWAYS_USE_BASE_CLOCK base clock frequency is directly + * controller by driver and it's rate varies upto max. 200MHz. This new quirk + * will be used in such cases to avoid controller mulplication when timeout is + * calculated based on the base clock. + */ +#define SDHCI_QUIRK2_DIVIDE_TOUT_BY_4 (1 << 23) + +/* + * Some SDHC controllers are unable to handle data-end bit error in + * 1-bit mode of SDIO. + */ +#define SDHCI_QUIRK2_IGN_DATA_END_BIT_ERROR (1<<24) + +/* Controller has nonstandard clock management */ +#define SDHCI_QUIRK_NONSTANDARD_CLOCK (1<<25) +/* Use reset workaround in case sdhci reset timeouts */ +#define SDHCI_QUIRK2_USE_RESET_WORKAROUND (1<<26) +/* Some controllers doesn't have have any LED control */ +#define SDHCI_QUIRK2_BROKEN_LED_CONTROL (1<<27) +/* + * Some controllers doesn't follow the tuning procedure as defined in spec. + * The tuning data has to be compared from SW driver to validate the correct + * phase. + */ +#define SDHCI_QUIRK2_NON_STANDARD_TUNING (1 << 28) +/* + * Some controllers may use PIO mode to workaround HW issues in ADMA for + * eMMC tuning commands. + */ +#define SDHCI_QUIRK2_USE_PIO_FOR_EMMC_TUNING (1 << 23) + + int irq; /* Device IRQ */ void __iomem *ioaddr; /* Mapped address */ @@ -432,6 +528,7 @@ struct sdhci_host { struct mmc_host *mmc; /* MMC structure */ struct mmc_host_ops mmc_host_ops; /* MMC host ops */ u64 dma_mask; /* custom DMA mask */ + u64 coherent_dma_mask; #if defined(CONFIG_LEDS_CLASS) || defined(CONFIG_LEDS_CLASS_MODULE) struct led_classdev led; /* LED control */ @@ -453,6 +550,7 @@ struct sdhci_host { #define SDHCI_SDR104_NEEDS_TUNING (1<<10) /* SDR104/HS200 needs tuning */ #define SDHCI_USE_64_BIT_DMA (1<<12) /* Use 64-bit DMA */ #define SDHCI_HS400_TUNING (1<<13) /* Tuning for HS400 */ +#define SDHCI_HOST_IRQ_STATUS (1<<14) /* host->irq status */ unsigned int version; /* SDHCI spec. version */ @@ -466,6 +564,7 @@ struct sdhci_host { bool runtime_suspended; /* Host is runtime suspended */ bool bus_on; /* Bus power prevents runtime suspend */ bool preset_enabled; /* Preset is enabled */ + bool cdr_support; struct mmc_request *mrq; /* Current request */ struct mmc_command *cmd; /* Current command */ @@ -514,6 +613,20 @@ struct sdhci_host { unsigned int tuning_count; /* Timer count for re-tuning */ unsigned int tuning_mode; /* Re-tuning mode supported by host */ #define SDHCI_TUNING_MODE_1 0 + ktime_t data_start_time; + + enum sdhci_power_policy power_policy; + + bool is_crypto_en; + bool crypto_reset_reqd; + bool sdio_irq_async_status; + + u32 auto_cmd_err_sts; + struct ratelimit_state dbg_dump_rs; + struct cmdq_host *cq_host; + int reset_wa_applied; /* reset workaround status */ + ktime_t reset_wa_t; /* time when the reset workaround is applied */ + int reset_wa_cnt; /* total number of times workaround is used */ unsigned long private[0] ____cacheline_aligned; }; @@ -543,16 +656,46 @@ struct sdhci_ops { unsigned int (*get_ro)(struct sdhci_host *host); void (*reset)(struct sdhci_host *host, u8 mask); int (*platform_execute_tuning)(struct sdhci_host *host, u32 opcode); + int (*crypto_engine_cfg)(struct sdhci_host *host, + struct mmc_request *mrq, u32 slot); + int (*crypto_engine_cmdq_cfg)(struct sdhci_host *host, + struct mmc_request *mrq, u32 slot, u64 *ice_ctx); + int (*crypto_engine_cfg_end)(struct sdhci_host *host, + struct mmc_request *mrq); + int (*crypto_engine_reset)(struct sdhci_host *host); + void (*crypto_cfg_reset)(struct sdhci_host *host, unsigned int slot); void (*set_uhs_signaling)(struct sdhci_host *host, unsigned int uhs); void (*hw_reset)(struct sdhci_host *host); void (*adma_workaround)(struct sdhci_host *host, u32 intmask); + unsigned int (*get_max_segments)(void); void (*platform_init)(struct sdhci_host *host); +#define REQ_BUS_OFF (1 << 0) +#define REQ_BUS_ON (1 << 1) +#define REQ_IO_LOW (1 << 2) +#define REQ_IO_HIGH (1 << 3) void (*card_event)(struct sdhci_host *host); + int (*enhanced_strobe)(struct sdhci_host *host); + void (*platform_bus_voting)(struct sdhci_host *host, u32 enable); + void (*check_power_status)(struct sdhci_host *host, u32 req_type); + int (*config_auto_tuning_cmd)(struct sdhci_host *host, + bool enable, + u32 type); + int (*enable_controller_clock)(struct sdhci_host *host); + void (*clear_set_dumpregs)(struct sdhci_host *host, bool set); + void (*enhanced_strobe_mask)(struct sdhci_host *host, bool set); + void (*dump_vendor_regs)(struct sdhci_host *host); + void (*toggle_cdr)(struct sdhci_host *host, bool enable); void (*voltage_switch)(struct sdhci_host *host); int (*select_drive_strength)(struct sdhci_host *host, struct mmc_card *card, unsigned int max_dtr, int host_drv, int card_drv, int *drv_type); + int (*notify_load)(struct sdhci_host *host, enum mmc_load state); + void (*reset_workaround)(struct sdhci_host *host, u32 enable); + void (*init)(struct sdhci_host *host); + void (*pre_req)(struct sdhci_host *host, struct mmc_request *req); + void (*post_req)(struct sdhci_host *host, struct mmc_request *req); + unsigned int (*get_current_limit)(struct sdhci_host *host); }; #ifdef CONFIG_MMC_SDHCI_IO_ACCESSORS @@ -672,4 +815,5 @@ extern int sdhci_runtime_suspend_host(struct sdhci_host *host); extern int sdhci_runtime_resume_host(struct sdhci_host *host); #endif +void sdhci_cfg_irq(struct sdhci_host *host, bool enable, bool sync); #endif /* __SDHCI_HW_H */ |