Merge "spi: spi_qsd: Improve latencies for small transfers"

2 files changed, 133 insertions, 76 deletions

diff --git a/drivers/spi/spi_qsd.c b/drivers/spi/spi_qsd.c
index 934ac0e642d9..a5d59e7f2d90 100644
--- a/drivers/spi/spi_qsd.c
+++ b/drivers/spi/spi_qsd.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2015, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2008-2016, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -45,6 +45,8 @@
 #include <linux/msm-bus-board.h>
 #include "spi_qsd.h"
 
+#define SPI_MAX_BYTES_PER_WORD			(4)
+
 static int msm_spi_pm_resume_runtime(struct device *device);
 static int msm_spi_pm_suspend_runtime(struct device *device);
 static inline void msm_spi_dma_unmap_buffers(struct msm_spi *dd);
@@ -438,10 +440,12 @@ static void msm_spi_read_word_from_fifo(struct msm_spi *dd)
 	u32   data_in;
 	int   i;
 	int   shift;
+	int   read_bytes = (dd->pack_words ?
+				SPI_MAX_BYTES_PER_WORD : dd->bytes_per_word);
 
 	data_in = readl_relaxed(dd->base + SPI_INPUT_FIFO);
 	if (dd->read_buf) {
-		for (i = 0; (i < dd->bytes_per_word) &&
+		for (i = 0; (i < read_bytes) &&
 			     dd->rx_bytes_remaining; i++) {
 			/* The data format depends on bytes_per_word:
 			   4 bytes: 0x12345678
@@ -454,8 +458,8 @@ static void msm_spi_read_word_from_fifo(struct msm_spi *dd)
 			dd->rx_bytes_remaining--;
 		}
 	} else {
-		if (dd->rx_bytes_remaining >= dd->bytes_per_word)
-			dd->rx_bytes_remaining -= dd->bytes_per_word;
+		if (dd->rx_bytes_remaining >= read_bytes)
+			dd->rx_bytes_remaining -= read_bytes;
 		else
 			dd->rx_bytes_remaining = 0;
 	}
@@ -552,7 +556,7 @@ msm_spi_set_bpw_and_no_io_flags(struct msm_spi *dd, u32 *config, int n)
 	if (n != (*config & SPI_CFG_N))
 		*config = (*config & ~SPI_CFG_N) | n;
 
-	if (dd->mode == SPI_BAM_MODE) {
+	if (dd->tx_mode == SPI_BAM_MODE) {
 		if (dd->read_buf == NULL)
 			*config |= SPI_NO_INPUT;
 		if (dd->write_buf == NULL)
@@ -617,25 +621,34 @@ static void msm_spi_set_spi_config(struct msm_spi *dd, int bpw)
 static void msm_spi_set_mx_counts(struct msm_spi *dd, u32 n_words)
 {
 	/*
-	 * n_words cannot exceed fifo_size, and only one READ COUNT
-	 * interrupt is generated per transaction, so for transactions
-	 * larger than fifo size READ COUNT must be disabled.
-	 * For those transactions we usually move to Data Mover mode.
+	 * For FIFO mode:
+	 *   - Set the MX_OUTPUT_COUNT/MX_INPUT_COUNT registers to 0
+	 *   - Set the READ/WRITE_COUNT registers to 0 (infinite mode)
+	 *     or num bytes (finite mode) if less than fifo worth of data.
+	 * For Block mode:
+	 *  - Set the MX_OUTPUT/MX_INPUT_COUNT registers to num xfer bytes.
+	 *  - Set the READ/WRITE_COUNT registers to 0.
 	 */
-	if (dd->mode == SPI_FIFO_MODE) {
-		if (n_words <= dd->input_fifo_size) {
-			writel_relaxed(n_words,
-				       dd->base + SPI_MX_READ_COUNT);
-			msm_spi_set_write_count(dd, n_words);
-		} else {
-			writel_relaxed(0, dd->base + SPI_MX_READ_COUNT);
-			msm_spi_set_write_count(dd, 0);
-		}
-		if (dd->qup_ver == SPI_QUP_VERSION_BFAM) {
-			/* must be zero for FIFO */
-			writel_relaxed(0, dd->base + SPI_MX_INPUT_COUNT);
+	if (dd->tx_mode != SPI_BAM_MODE) {
+		if (dd->tx_mode == SPI_FIFO_MODE) {
+			if (n_words <= dd->input_fifo_size)
+				msm_spi_set_write_count(dd, n_words);
+			else
+				msm_spi_set_write_count(dd, 0);
 			writel_relaxed(0, dd->base + SPI_MX_OUTPUT_COUNT);
-		}
+		} else
+			writel_relaxed(n_words, dd->base + SPI_MX_OUTPUT_COUNT);
+
+		if (dd->rx_mode == SPI_FIFO_MODE) {
+			if (n_words <= dd->input_fifo_size)
+				writel_relaxed(n_words,
+						dd->base + SPI_MX_READ_COUNT);
+			else
+				writel_relaxed(0,
+						dd->base + SPI_MX_READ_COUNT);
+			writel_relaxed(0, dd->base + SPI_MX_INPUT_COUNT);
+		} else
+			writel_relaxed(n_words, dd->base + SPI_MX_INPUT_COUNT);
 	} else {
 		/* must be zero for BAM and DMOV */
 		writel_relaxed(0, dd->base + SPI_MX_READ_COUNT);
@@ -882,7 +895,7 @@ xfr_err:
 static int
 msm_spi_bam_next_transfer(struct msm_spi *dd)
 {
-	if (dd->mode != SPI_BAM_MODE)
+	if (dd->tx_mode != SPI_BAM_MODE)
 		return 0;
 
 	if (dd->tx_bytes_remaining > 0) {
@@ -901,7 +914,7 @@ msm_spi_bam_next_transfer(struct msm_spi *dd)
 static int msm_spi_dma_send_next(struct msm_spi *dd)
 {
 	int ret = 0;
-	if (dd->mode == SPI_BAM_MODE)
+	if (dd->tx_mode == SPI_BAM_MODE)
 		ret = msm_spi_bam_next_transfer(dd);
 	return ret;
 }
@@ -932,32 +945,38 @@ static inline irqreturn_t msm_spi_qup_irq(int irq, void *dev_id)
 	}
 
 	op = readl_relaxed(dd->base + SPI_OPERATIONAL);
+	writel_relaxed(op, dd->base + SPI_OPERATIONAL);
+	/*
+	 * Ensure service flag was cleared before further
+	 * processing of interrupt.
+	 */
+	mb();
 	if (op & SPI_OP_INPUT_SERVICE_FLAG) {
-		writel_relaxed(SPI_OP_INPUT_SERVICE_FLAG,
-			       dd->base + SPI_OPERATIONAL);
-		/*
-		 * Ensure service flag was cleared before further
-		 * processing of interrupt.
-		 */
-		mb();
 		ret |= msm_spi_input_irq(irq, dev_id);
 	}
 
 	if (op & SPI_OP_OUTPUT_SERVICE_FLAG) {
-		writel_relaxed(SPI_OP_OUTPUT_SERVICE_FLAG,
-			       dd->base + SPI_OPERATIONAL);
-		/*
-		 * Ensure service flag was cleared before further
-		 * processing of interrupt.
-		 */
-		mb();
 		ret |= msm_spi_output_irq(irq, dev_id);
 	}
 
-	if (dd->done) {
+	if (dd->tx_mode != SPI_BAM_MODE) {
+		if (!dd->rx_done) {
+			if (dd->rx_bytes_remaining == 0)
+				dd->rx_done = true;
+		}
+		if (!dd->tx_done) {
+			if (!dd->tx_bytes_remaining &&
+					(op & SPI_OP_IP_FIFO_NOT_EMPTY)) {
+				dd->tx_done = true;
+			}
+		}
+	}
+	if (dd->tx_done && dd->rx_done) {
+		msm_spi_set_state(dd, SPI_OP_STATE_RESET);
+		dd->tx_done = false;
+		dd->rx_done = false;
 		complete(&dd->rx_transfer_complete);
 		complete(&dd->tx_transfer_complete);
-		dd->done = 0;
 	}
 	return ret;
 }
@@ -968,17 +987,23 @@ static irqreturn_t msm_spi_input_irq(int irq, void *dev_id)
 
 	dd->stat_rx++;
 
-	if (dd->mode == SPI_MODE_NONE)
+	if (dd->rx_mode == SPI_MODE_NONE)
 		return IRQ_HANDLED;
 
-	if (dd->mode == SPI_FIFO_MODE) {
+	if (dd->rx_mode == SPI_FIFO_MODE) {
 		while ((readl_relaxed(dd->base + SPI_OPERATIONAL) &
 			SPI_OP_IP_FIFO_NOT_EMPTY) &&
 			(dd->rx_bytes_remaining > 0)) {
 			msm_spi_read_word_from_fifo(dd);
 		}
-		if (dd->rx_bytes_remaining == 0)
-			msm_spi_complete(dd);
+	} else if (dd->rx_mode == SPI_BLOCK_MODE) {
+		int count = 0;
+
+		while (dd->rx_bytes_remaining &&
+				(count < dd->input_block_size)) {
+			msm_spi_read_word_from_fifo(dd);
+			count += SPI_MAX_BYTES_PER_WORD;
+		}
 	}
 
 	return IRQ_HANDLED;
@@ -989,18 +1014,20 @@ static void msm_spi_write_word_to_fifo(struct msm_spi *dd)
 	u32    word;
 	u8     byte;
 	int    i;
+	int   write_bytes =
+		(dd->pack_words ? SPI_MAX_BYTES_PER_WORD : dd->bytes_per_word);
 
 	word = 0;
 	if (dd->write_buf) {
-		for (i = 0; (i < dd->bytes_per_word) &&
+		for (i = 0; (i < write_bytes) &&
 			     dd->tx_bytes_remaining; i++) {
 			dd->tx_bytes_remaining--;
 			byte = *dd->write_buf++;
 			word |= (byte << (BITS_PER_BYTE * i));
 		}
 	} else
-		if (dd->tx_bytes_remaining > dd->bytes_per_word)
-			dd->tx_bytes_remaining -= dd->bytes_per_word;
+		if (dd->tx_bytes_remaining > write_bytes)
+			dd->tx_bytes_remaining -= write_bytes;
 		else
 			dd->tx_bytes_remaining = 0;
 	dd->write_xfr_cnt++;
@@ -1012,11 +1039,22 @@ static inline void msm_spi_write_rmn_to_fifo(struct msm_spi *dd)
 {
 	int count = 0;
 
-	while ((dd->tx_bytes_remaining > 0) && (count < dd->input_fifo_size) &&
-	       !(readl_relaxed(dd->base + SPI_OPERATIONAL) &
-		SPI_OP_OUTPUT_FIFO_FULL)) {
-		msm_spi_write_word_to_fifo(dd);
-		count++;
+	if (dd->tx_mode == SPI_FIFO_MODE) {
+		while ((dd->tx_bytes_remaining > 0) &&
+			(count < dd->input_fifo_size) &&
+		       !(readl_relaxed(dd->base + SPI_OPERATIONAL)
+						& SPI_OP_OUTPUT_FIFO_FULL)) {
+			msm_spi_write_word_to_fifo(dd);
+			count++;
+		}
+	}
+
+	if (dd->tx_mode == SPI_BLOCK_MODE) {
+		while (dd->tx_bytes_remaining &&
+				(count < dd->output_block_size)) {
+			msm_spi_write_word_to_fifo(dd);
+			count += SPI_MAX_BYTES_PER_WORD;
+		}
 	}
 }
 
@@ -1026,11 +1064,11 @@ static irqreturn_t msm_spi_output_irq(int irq, void *dev_id)
 
 	dd->stat_tx++;
 
-	if (dd->mode == SPI_MODE_NONE)
+	if (dd->tx_mode == SPI_MODE_NONE)
 		return IRQ_HANDLED;
 
 	/* Output FIFO is empty. Transmit any outstanding write data. */
-	if (dd->mode == SPI_FIFO_MODE)
+	if ((dd->tx_mode == SPI_FIFO_MODE) || (dd->tx_mode == SPI_BLOCK_MODE))
 		msm_spi_write_rmn_to_fifo(dd);
 
 	return IRQ_HANDLED;
@@ -1106,7 +1144,7 @@ error:
 static int msm_spi_dma_map_buffers(struct msm_spi *dd)
 {
 	int ret = 0;
-	if (dd->mode == SPI_BAM_MODE)
+	if (dd->tx_mode == SPI_BAM_MODE)
 		ret = msm_spi_bam_map_buffers(dd);
 	return ret;
 }
@@ -1135,7 +1173,7 @@ static void msm_spi_bam_unmap_buffers(struct msm_spi *dd)
 
 static inline void msm_spi_dma_unmap_buffers(struct msm_spi *dd)
 {
-	if (dd->mode == SPI_BAM_MODE)
+	if (dd->tx_mode == SPI_BAM_MODE)
 		msm_spi_bam_unmap_buffers(dd);
 }
 
@@ -1197,9 +1235,11 @@ static void
 msm_spi_set_transfer_mode(struct msm_spi *dd, u8 bpw, u32 read_count)
 {
 	if (msm_spi_use_dma(dd, dd->cur_transfer, bpw)) {
-		dd->mode = SPI_BAM_MODE;
+		dd->tx_mode = SPI_BAM_MODE;
+		dd->rx_mode = SPI_BAM_MODE;
 	} else {
-		dd->mode = SPI_FIFO_MODE;
+		dd->rx_mode = SPI_FIFO_MODE;
+		dd->tx_mode = SPI_FIFO_MODE;
 		dd->read_len = dd->cur_transfer->len;
 		dd->write_len = dd->cur_transfer->len;
 	}
@@ -1215,14 +1255,19 @@ static void msm_spi_set_qup_io_modes(struct msm_spi *dd)
 	spi_iom = readl_relaxed(dd->base + SPI_IO_MODES);
 	/* Set input and output transfer mode: FIFO, DMOV, or BAM */
 	spi_iom &= ~(SPI_IO_M_INPUT_MODE | SPI_IO_M_OUTPUT_MODE);
-	spi_iom = (spi_iom | (dd->mode << OUTPUT_MODE_SHIFT));
-	spi_iom = (spi_iom | (dd->mode << INPUT_MODE_SHIFT));
-	/* Turn on packing for data mover */
-	if (dd->mode == SPI_BAM_MODE)
+	spi_iom = (spi_iom | (dd->tx_mode << OUTPUT_MODE_SHIFT));
+	spi_iom = (spi_iom | (dd->rx_mode << INPUT_MODE_SHIFT));
+	/* Always enable packing for all % 8 bits_per_word */
+	if (dd->cur_transfer->bits_per_word &&
+		((dd->cur_transfer->bits_per_word == 8) ||
+		(dd->cur_transfer->bits_per_word == 16) ||
+		(dd->cur_transfer->bits_per_word == 32))) {
 		spi_iom |= SPI_IO_M_PACK_EN | SPI_IO_M_UNPACK_EN;
-	else {
+		dd->pack_words = true;
+	} else {
 		spi_iom &= ~(SPI_IO_M_PACK_EN | SPI_IO_M_UNPACK_EN);
 		spi_iom |= SPI_IO_M_OUTPUT_BIT_SHIFT_EN;
+		dd->pack_words = false;
 	}
 
 	/*if (dd->mode == SPI_BAM_MODE) {
@@ -1280,7 +1325,7 @@ static void msm_spi_set_qup_op_mask(struct msm_spi *dd)
 {
 	/* mask INPUT and OUTPUT service flags in to prevent IRQs on FIFO status
 	 * change in BAM mode */
-	u32 mask = (dd->mode == SPI_BAM_MODE) ?
+	u32 mask = (dd->tx_mode == SPI_BAM_MODE) ?
 		QUP_OP_MASK_OUTPUT_SERVICE_FLAG | QUP_OP_MASK_INPUT_SERVICE_FLAG
 		: 0;
 	writel_relaxed(mask, dd->base + QUP_OPERATIONAL_MASK);
@@ -1321,6 +1366,8 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
 	dd->rx_bytes_remaining = dd->cur_msg_len;
 	dd->read_buf           = dd->cur_transfer->rx_buf;
 	dd->write_buf          = dd->cur_transfer->tx_buf;
+	dd->tx_done = false;
+	dd->rx_done = false;
 	init_completion(&dd->tx_transfer_complete);
 	init_completion(&dd->rx_transfer_complete);
 	if (dd->cur_transfer->bits_per_word)
@@ -1351,10 +1398,12 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
 
 	msm_spi_set_transfer_mode(dd, bpw, read_count);
 	msm_spi_set_mx_counts(dd, read_count);
-	if (dd->mode == SPI_BAM_MODE) {
+	if (dd->tx_mode == SPI_BAM_MODE) {
 		ret = msm_spi_dma_map_buffers(dd);
 		if (ret < 0) {
 			pr_err("Mapping DMA buffers\n");
+			dd->tx_mode = SPI_MODE_NONE;
+			dd->rx_mode = SPI_MODE_NONE;
 			return ret;
 		}
 	}
@@ -1368,11 +1417,11 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
 	   the first. Restricting this to one write avoids contention
 	   issues and race conditions between this thread and the int handler
 	*/
-	if (dd->mode == SPI_FIFO_MODE) {
+	if (dd->tx_mode != SPI_BAM_MODE) {
 		if (msm_spi_prepare_for_write(dd))
 			goto transfer_end;
 		msm_spi_start_write(dd, read_count);
-	} else if (dd->mode == SPI_BAM_MODE) {
+	} else {
 		if ((msm_spi_bam_begin_transfer(dd)) < 0) {
 			dev_err(dd->dev, "%s: BAM transfer setup failed\n",
 				__func__);
@@ -1388,11 +1437,11 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
 	 * might fire before the first word is written resulting in a
 	 * possible race condition.
 	 */
-	if (dd->mode != SPI_BAM_MODE)
+	if (dd->tx_mode != SPI_BAM_MODE)
 		if (msm_spi_set_state(dd, SPI_OP_STATE_RUN)) {
 			dev_warn(dd->dev,
 				"%s: Failed to set QUP to run-state. Mode:%d",
-				__func__, dd->mode);
+				__func__, dd->tx_mode);
 			goto transfer_end;
 		}
 
@@ -1422,10 +1471,11 @@ static int msm_spi_process_transfer(struct msm_spi *dd)
 	msm_spi_udelay(dd->xfrs_delay_usec);
 
 transfer_end:
-	if ((dd->mode == SPI_BAM_MODE) && status)
+	if ((dd->tx_mode == SPI_BAM_MODE) && status)
 		msm_spi_bam_flush(dd);
 	msm_spi_dma_unmap_buffers(dd);
-	dd->mode = SPI_MODE_NONE;
+	dd->tx_mode = SPI_MODE_NONE;
+	dd->rx_mode = SPI_MODE_NONE;
 
 	msm_spi_set_state(dd, SPI_OP_STATE_RESET);
 	if (!dd->cur_transfer->cs_change)
@@ -2350,7 +2400,8 @@ static int init_resources(struct platform_device *pdev)
 	pclk_enabled = 0;
 
 	dd->transfer_pending = 0;
-	dd->mode = SPI_MODE_NONE;
+	dd->tx_mode = SPI_MODE_NONE;
+	dd->rx_mode = SPI_MODE_NONE;
 
 	rc = msm_spi_request_irq(dd, pdev, master);
 	if (rc)
diff --git a/drivers/spi/spi_qsd.h b/drivers/spi/spi_qsd.h
index cfc0e754f255..fb906939c03a 100644
--- a/drivers/spi/spi_qsd.h
+++ b/drivers/spi/spi_qsd.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2014-2016, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -113,6 +113,8 @@
 #define INPUT_MODE_SHIFT              QSD_REG(10)         QUP_REG(12)
 
 /* SPI_OPERATIONAL fields */
+#define SPI_OP_IN_BLK_RD_REQ_FLAG     0x00002000
+#define SPI_OP_OUT_BLK_WR_REQ_FLAG    0x00001000
 #define SPI_OP_MAX_INPUT_DONE_FLAG    0x00000800
 #define SPI_OP_MAX_OUTPUT_DONE_FLAG   0x00000400
 #define SPI_OP_INPUT_SERVICE_FLAG     0x00000200
@@ -318,7 +320,8 @@ struct msm_spi {
 	bool                     transfer_pending;
 	wait_queue_head_t        continue_suspend;
 	/* DMA data */
-	enum msm_spi_mode        mode;
+	enum msm_spi_mode        tx_mode;
+	enum msm_spi_mode        rx_mode;
 	bool                     use_dma;
 	int                      tx_dma_chan;
 	int                      tx_dma_crci;
@@ -349,7 +352,8 @@ struct msm_spi {
 #endif
 	struct msm_spi_platform_data *pdata; /* Platform data */
 	/* When set indicates multiple transfers in a single message */
-	bool                     done;
+	bool                     rx_done;
+	bool                     tx_done;
 	u32                      cur_msg_len;
 	/* Used in FIFO mode to keep track of the transfer being processed */
 	struct spi_transfer     *cur_tx_transfer;
@@ -367,6 +371,7 @@ struct msm_spi {
 	struct pinctrl_state	*pins_active;
 	struct pinctrl_state	*pins_sleep;
 	bool			is_init_complete;
+	bool			pack_words;
 };
 
 /* Forward declaration */
@@ -523,7 +528,8 @@ static inline void msm_spi_set_write_count(struct msm_spi *dd, int val)
 
 static inline void msm_spi_complete(struct msm_spi *dd)
 {
-	dd->done = 1;
+	dd->tx_done = true;
+	dd->rx_done = true;
 }
 
 static inline void msm_spi_enable_error_flags(struct msm_spi *dd)