msm: kgsl: Add Qualcomm GPU driver

Snapshot of the Qualcom Adreno GPU driver (KGSL) as of msm-3.18 commit
commit e70ad0cd5efd ("Promotion of kernel.lnx.3.18-151201.").

Signed-off-by: Jordan Crouse <jcrouse@codeaurora.org>

1 files changed, 1357 insertions, 0 deletions

diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c
new file mode 100644
index 000000000000..a80707385e3b
--- /dev/null
+++ b/drivers/gpu/msm/adreno_ringbuffer.c
@@ -0,0 +1,1357 @@
+/* Copyright (c) 2002,2007-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/log2.h>
+#include <linux/time.h>
+#include <linux/delay.h>
+
+#include "kgsl.h"
+#include "kgsl_sharedmem.h"
+#include "kgsl_cffdump.h"
+#include "kgsl_trace.h"
+#include "kgsl_pwrctrl.h"
+
+#include "adreno.h"
+#include "adreno_pm4types.h"
+#include "adreno_ringbuffer.h"
+
+#include "a3xx_reg.h"
+#include "adreno_a5xx.h"
+
+#define GSL_RB_NOP_SIZEDWORDS				2
+
+#define ADRENO_RB_PREEMPT_TOKEN_IB_DWORDS	50
+#define ADRENO_RB_PREEMPT_TOKEN_DWORDS		125
+
+#define RB_HOSTPTR(_rb, _pos) \
+	((unsigned int *) ((_rb)->buffer_desc.hostptr + \
+		((_pos) * sizeof(unsigned int))))
+
+#define RB_GPUADDR(_rb, _pos) \
+	((_rb)->buffer_desc.gpuaddr + ((_pos) * sizeof(unsigned int)))
+
+static void _cff_write_ringbuffer(struct adreno_ringbuffer *rb)
+{
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
+	struct kgsl_device *device = &adreno_dev->dev;
+	uint64_t gpuaddr;
+	unsigned int *hostptr;
+	size_t size;
+
+	if (device->cff_dump_enable == 0)
+		return;
+
+	/*
+	 * This code is predicated on the fact that we write a full block of
+	 * stuff without wrapping
+	 */
+	BUG_ON(rb->wptr < rb->last_wptr);
+
+	size = (rb->wptr - rb->last_wptr) * sizeof(unsigned int);
+
+	hostptr = RB_HOSTPTR(rb, rb->last_wptr);
+	gpuaddr = RB_GPUADDR(rb, rb->last_wptr);
+
+	kgsl_cffdump_memcpy(device, gpuaddr, hostptr, size);
+}
+
+void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb,
+		struct adreno_submit_time *time)
+{
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
+	BUG_ON(rb->wptr == 0);
+
+	/* Write the changes to CFF if so enabled */
+	_cff_write_ringbuffer(rb);
+
+	/*
+	 * Read the current GPU ticks and wallclock for most accurate
+	 * profiling
+	 */
+
+	if (time != NULL) {
+		/*
+		 * Here we are attempting to create a mapping between the
+		 * GPU time domain (alwayson counter) and the CPU time domain
+		 * (local_clock) by sampling both values as close together as
+		 * possible. This is useful for many types of debugging and
+		 * profiling. In order to make this mapping as accurate as
+		 * possible, we must turn off interrupts to avoid running
+		 * interrupt handlers between the two samples.
+		 */
+		unsigned long flags;
+		local_irq_save(flags);
+
+		/* Read always on registers */
+		if (!adreno_is_a3xx(adreno_dev)) {
+			adreno_readreg64(adreno_dev,
+				ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO,
+				ADRENO_REG_RBBM_ALWAYSON_COUNTER_HI,
+				&time->ticks);
+
+			/*
+			 * Mask hi bits as they may be incorrect on
+			 * a4x and some a5x
+			 */
+			if (ADRENO_GPUREV(adreno_dev) >= 400 &&
+				ADRENO_GPUREV(adreno_dev) <= ADRENO_REV_A530)
+				time->ticks &= 0xFFFFFFFF;
+		}
+		else
+			time->ticks = 0;
+
+		/* Get the kernel clock for time since boot */
+		time->ktime = local_clock();
+
+		/* Get the timeofday for the wall time (for the user) */
+		getnstimeofday(&time->utime);
+
+		local_irq_restore(flags);
+	}
+
+	/* Memory barrier before informing the hardware of new commands */
+	mb();
+
+	if (adreno_preempt_state(adreno_dev, ADRENO_DISPATCHER_PREEMPT_CLEAR) &&
+		(adreno_dev->cur_rb == rb)) {
+		/*
+		 * Let the pwrscale policy know that new commands have
+		 * been submitted.
+		 */
+		kgsl_pwrscale_busy(rb->device);
+		adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, rb->wptr);
+	}
+}
+
+int adreno_ringbuffer_submit_spin(struct adreno_ringbuffer *rb,
+		struct adreno_submit_time *time, unsigned int timeout)
+{
+	adreno_ringbuffer_submit(rb, NULL);
+	return adreno_spin_idle(rb->device, timeout);
+}
+
+static int
+adreno_ringbuffer_waitspace(struct adreno_ringbuffer *rb,
+				unsigned int numcmds, int wptr_ahead)
+{
+	int nopcount = 0;
+	unsigned int freecmds;
+	unsigned int wptr = rb->wptr;
+	unsigned int *cmds = NULL;
+	uint64_t gpuaddr;
+	unsigned long wait_time;
+	unsigned long wait_timeout = msecs_to_jiffies(ADRENO_IDLE_TIMEOUT);
+	unsigned int rptr;
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
+
+	/* if wptr ahead, fill the remaining with NOPs */
+	if (wptr_ahead) {
+		/* -1 for header */
+		nopcount = KGSL_RB_DWORDS - rb->wptr - 1;
+
+		cmds = RB_HOSTPTR(rb, rb->wptr);
+		gpuaddr = RB_GPUADDR(rb, rb->wptr);
+
+		rptr = adreno_get_rptr(rb);
+		/* For non current rb we don't expect the rptr to move */
+		if ((adreno_dev->cur_rb != rb ||
+				!adreno_preempt_state(adreno_dev,
+				ADRENO_DISPATCHER_PREEMPT_CLEAR)) &&
+			!rptr)
+			return -ENOSPC;
+
+		/* Make sure that rptr is not 0 before submitting
+		 * commands at the end of ringbuffer. We do not
+		 * want the rptr and wptr to become equal when
+		 * the ringbuffer is not empty */
+		wait_time = jiffies + wait_timeout;
+		while (!rptr) {
+			rptr = adreno_get_rptr(rb);
+			if (time_after(jiffies, wait_time))
+				return -ETIMEDOUT;
+		}
+
+		rb->wptr = 0;
+	}
+
+	rptr = adreno_get_rptr(rb);
+	freecmds = rptr - rb->wptr;
+	if (freecmds == 0 || freecmds > numcmds)
+		goto done;
+
+	/* non current rptr will not advance anyway or if preemption underway */
+	if (adreno_dev->cur_rb != rb ||
+		!adreno_preempt_state(adreno_dev,
+			ADRENO_DISPATCHER_PREEMPT_CLEAR)) {
+		rb->wptr = wptr;
+		return -ENOSPC;
+	}
+
+	wait_time = jiffies + wait_timeout;
+	/* wait for space in ringbuffer */
+	while (1) {
+		rptr = adreno_get_rptr(rb);
+
+		freecmds = rptr - rb->wptr;
+
+		if (freecmds == 0 || freecmds > numcmds)
+			break;
+
+		if (time_after(jiffies, wait_time)) {
+			KGSL_DRV_ERR(rb->device,
+			"Timed out waiting for freespace in RB rptr: 0x%x, wptr: 0x%x, rb id %d\n",
+			rptr, wptr, rb->id);
+			return -ETIMEDOUT;
+		}
+	}
+done:
+	if (wptr_ahead) {
+		*cmds = cp_packet(adreno_dev, CP_NOP, nopcount);
+		kgsl_cffdump_write(rb->device, gpuaddr, *cmds);
+
+	}
+	return 0;
+}
+
+unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb,
+					unsigned int numcmds)
+{
+	unsigned int *ptr = NULL;
+	int ret = 0;
+	unsigned int rptr;
+	BUG_ON(numcmds >= KGSL_RB_DWORDS);
+
+	rptr = adreno_get_rptr(rb);
+	/* check for available space */
+	if (rb->wptr >= rptr) {
+		/* wptr ahead or equal to rptr */
+		/* reserve dwords for nop packet */
+		if ((rb->wptr + numcmds) > (KGSL_RB_DWORDS -
+				GSL_RB_NOP_SIZEDWORDS))
+			ret = adreno_ringbuffer_waitspace(rb, numcmds, 1);
+	} else {
+		/* wptr behind rptr */
+		if ((rb->wptr + numcmds) >= rptr)
+			ret = adreno_ringbuffer_waitspace(rb, numcmds, 0);
+		/* check for remaining space */
+		/* reserve dwords for nop packet */
+		if (!ret && (rb->wptr + numcmds) > (KGSL_RB_DWORDS -
+				GSL_RB_NOP_SIZEDWORDS))
+			ret = adreno_ringbuffer_waitspace(rb, numcmds, 1);
+	}
+
+	if (!ret) {
+		rb->last_wptr = rb->wptr;
+
+		ptr = (unsigned int *)rb->buffer_desc.hostptr + rb->wptr;
+		rb->wptr += numcmds;
+	} else
+		ptr = ERR_PTR(ret);
+
+	return ptr;
+}
+
+/**
+ * _ringbuffer_setup_common() - Ringbuffer start
+ * @rb: Pointer to adreno ringbuffer
+ *
+ * Setup ringbuffer for GPU.
+ */
+static void _ringbuffer_setup_common(struct adreno_ringbuffer *rb)
+{
+	struct kgsl_device *device = rb->device;
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+	struct adreno_ringbuffer *rb_temp;
+	int i;
+
+	FOR_EACH_RINGBUFFER(adreno_dev, rb_temp, i) {
+		kgsl_sharedmem_set(rb_temp->device,
+			&(rb_temp->buffer_desc), 0,
+			0xAA, KGSL_RB_SIZE);
+		rb_temp->wptr = 0;
+		rb_temp->rptr = 0;
+		rb_temp->wptr_preempt_end = 0xFFFFFFFF;
+		rb_temp->starve_timer_state =
+		ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT;
+		adreno_iommu_set_pt_generate_rb_cmds(rb_temp,
+					device->mmu.defaultpagetable);
+	}
+
+	/*
+	 * The size of the ringbuffer in the hardware is the log2
+	 * representation of the size in quadwords (sizedwords / 2).
+	 * Also disable the host RPTR shadow register as it might be unreliable
+	 * in certain circumstances.
+	 */
+
+	adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL,
+		(ilog2(KGSL_RB_DWORDS >> 1) & 0x3F) |
+		(1 << 27));
+
+	adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_BASE,
+			  ADRENO_REG_CP_RB_BASE_HI, rb->buffer_desc.gpuaddr);
+
+	/* CP ROQ queue sizes (bytes) - RB:16, ST:16, IB1:32, IB2:64 */
+	if (adreno_is_a3xx(adreno_dev)) {
+		unsigned int val = 0x000E0602;
+
+		if (adreno_is_a305b(adreno_dev) ||
+				adreno_is_a310(adreno_dev) ||
+				adreno_is_a330(adreno_dev))
+			val = 0x003E2008;
+		kgsl_regwrite(device, A3XX_CP_QUEUE_THRESHOLDS, val);
+	}
+}
+
+/**
+ * _ringbuffer_start_common() - Ringbuffer start
+ * @rb: Pointer to adreno ringbuffer
+ *
+ * Start ringbuffer for GPU.
+ */
+static int _ringbuffer_start_common(struct adreno_ringbuffer *rb)
+{
+	int status;
+	struct kgsl_device *device = rb->device;
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+	struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+
+	/* clear ME_HALT to start micro engine */
+	adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, 0);
+
+	/* ME init is GPU specific, so jump into the sub-function */
+	status = gpudev->rb_init(adreno_dev, rb);
+	if (status)
+		return status;
+
+	return status;
+}
+
+/**
+ * adreno_ringbuffer_start() - Ringbuffer start
+ * @adreno_dev: Pointer to adreno device
+ * @start_type: Warm or cold start
+ */
+int adreno_ringbuffer_start(struct adreno_device *adreno_dev,
+	unsigned int start_type)
+{
+	int status;
+	struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev);
+	struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+
+	_ringbuffer_setup_common(rb);
+
+	status = gpudev->microcode_load(adreno_dev, start_type);
+	if (status)
+		return status;
+
+	return _ringbuffer_start_common(rb);
+}
+
+void adreno_ringbuffer_stop(struct adreno_device *adreno_dev)
+{
+	struct kgsl_device *device = &adreno_dev->dev;
+	struct adreno_ringbuffer *rb;
+	int i;
+	FOR_EACH_RINGBUFFER(adreno_dev, rb, i)
+		kgsl_cancel_events(device, &(rb->events));
+}
+
+static int _adreno_ringbuffer_init(struct adreno_device *adreno_dev,
+				struct adreno_ringbuffer *rb, int id)
+{
+	int ret;
+	char name[64];
+
+	rb->device = &adreno_dev->dev;
+	rb->id = id;
+
+	snprintf(name, sizeof(name), "rb_events-%d", id);
+	kgsl_add_event_group(&rb->events, NULL, name,
+		adreno_rb_readtimestamp, rb);
+	rb->timestamp = 0;
+	init_waitqueue_head(&rb->ts_expire_waitq);
+
+	/*
+	 * Allocate mem for storing RB pagetables and commands to
+	 * switch pagetable
+	 */
+	ret = kgsl_allocate_global(&adreno_dev->dev, &rb->pagetable_desc,
+		PAGE_SIZE, 0, KGSL_MEMDESC_PRIVILEGED);
+	if (ret)
+		return ret;
+
+	ret = kgsl_allocate_global(&adreno_dev->dev, &rb->buffer_desc,
+			KGSL_RB_SIZE, KGSL_MEMFLAGS_GPUREADONLY, 0);
+	return ret;
+}
+
+int adreno_ringbuffer_init(struct adreno_device *adreno_dev, bool nopreempt)
+{
+	int status = 0;
+	struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+	struct adreno_ringbuffer *rb;
+	int i;
+
+	if (nopreempt == false && ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION))
+		adreno_dev->num_ringbuffers = gpudev->num_prio_levels;
+	else
+		adreno_dev->num_ringbuffers = 1;
+
+	FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
+		status = _adreno_ringbuffer_init(adreno_dev, rb, i);
+		if (status)
+			break;
+	}
+	if (status)
+		adreno_ringbuffer_close(adreno_dev);
+	else
+		adreno_dev->cur_rb = &(adreno_dev->ringbuffers[0]);
+
+	return status;
+}
+
+static void _adreno_ringbuffer_close(struct adreno_ringbuffer *rb)
+{
+	kgsl_free_global(&rb->pagetable_desc);
+	kgsl_free_global(&rb->preemption_desc);
+
+	memset(&rb->pt_update_desc, 0, sizeof(struct kgsl_memdesc));
+
+	kgsl_free_global(&rb->buffer_desc);
+	kgsl_del_event_group(&rb->events);
+	memset(rb, 0, sizeof(struct adreno_ringbuffer));
+}
+
+void adreno_ringbuffer_close(struct adreno_device *adreno_dev)
+{
+	struct adreno_ringbuffer *rb;
+	int i;
+
+	FOR_EACH_RINGBUFFER(adreno_dev, rb, i)
+		_adreno_ringbuffer_close(rb);
+}
+
+/*
+ * cp_secure_mode() - Put GPU in trusted mode
+ * @adreno_dev: Pointer to adreno device
+ * @cmds: Pointer to cmds to be put in the ringbuffer
+ * @set: 1 - secure mode, 0 - unsecure mode
+ *
+ * Add commands to the ringbuffer to put the GPU in secure mode
+ * or unsecure mode based on the variable set.
+ */
+int cp_secure_mode(struct adreno_device *adreno_dev, uint *cmds,
+				int set)
+{
+	uint *start = cmds;
+
+	if (adreno_is_a4xx(adreno_dev)) {
+		cmds += cp_wait_for_idle(adreno_dev, cmds);
+		/*
+		 * The two commands will stall the PFP until the PFP-ME-AHB
+		 * is drained and the GPU is idle. As soon as this happens,
+		 * the PFP will start moving again.
+		 */
+		cmds += cp_wait_for_me(adreno_dev, cmds);
+
+		/*
+		 * Below commands are processed by ME. GPU will be
+		 * idle when they are processed. But the PFP will continue
+		 * to fetch instructions at the same time.
+		 */
+		*cmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+		*cmds++ = 0;
+		*cmds++ = cp_packet(adreno_dev, CP_WIDE_REG_WRITE, 2);
+		*cmds++ = adreno_getreg(adreno_dev,
+				ADRENO_REG_RBBM_SECVID_TRUST_CONTROL);
+		*cmds++ = set;
+		*cmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+		*cmds++ = 1;
+
+		/* Stall PFP until all above commands are complete */
+		cmds += cp_wait_for_me(adreno_dev, cmds);
+	} else {
+		/*
+		 * A5xx has a separate opcode specifically to put the GPU
+		 * in and out of secure mode.
+		 */
+		*cmds++ = cp_packet(adreno_dev, CP_SET_SECURE_MODE, 1);
+		*cmds++ = set;
+	}
+
+	return cmds - start;
+}
+
+static int
+adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
+				unsigned int flags, unsigned int *cmds,
+				unsigned int sizedwords, uint32_t timestamp,
+				struct adreno_submit_time *time)
+{
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
+	struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+	struct kgsl_device *device = rb->device;
+	unsigned int *ringcmds, *start;
+	unsigned int total_sizedwords = sizedwords;
+	unsigned int i;
+	unsigned int context_id = 0;
+	uint64_t gpuaddr = rb->device->memstore.gpuaddr;
+	bool profile_ready;
+	struct adreno_context *drawctxt = rb->drawctxt_active;
+	struct kgsl_context *context = NULL;
+	bool secured_ctxt = false;
+	uint64_t cond_addr;
+
+	if (drawctxt != NULL && kgsl_context_detached(&drawctxt->base) &&
+		!(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
+		return -ENOENT;
+
+	rb->timestamp++;
+
+	/* If this is a internal IB, use the global timestamp for it */
+	if (!drawctxt || (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
+		timestamp = rb->timestamp;
+	else {
+		context_id = drawctxt->base.id;
+		context = &drawctxt->base;
+	}
+
+	/*
+	 * Note that we cannot safely take drawctxt->mutex here without
+	 * potential mutex inversion with device->mutex which is held
+	 * here. As a result, any other code that accesses this variable
+	 * must also use device->mutex.
+	 */
+	if (drawctxt) {
+		drawctxt->internal_timestamp = rb->timestamp;
+		if (drawctxt->base.flags & KGSL_CONTEXT_SECURE)
+			secured_ctxt = true;
+	}
+
+	/*
+	 * If in stream ib profiling is enabled and there are counters
+	 * assigned, then space needs to be reserved for profiling.  This
+	 * space in the ringbuffer is always consumed (might be filled with
+	 * NOPs in error case.  profile_ready needs to be consistent through
+	 * the _addcmds call since it is allocating additional ringbuffer
+	 * command space.
+	 */
+	profile_ready = drawctxt &&
+		adreno_profile_assignments_ready(&adreno_dev->profile) &&
+		!(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE);
+
+	/* reserve space to temporarily turn off protected mode
+	*  error checking if needed
+	*/
+	total_sizedwords += flags & KGSL_CMD_FLAGS_PMODE ? 4 : 0;
+	/* 2 dwords to store the start of command sequence */
+	total_sizedwords += 2;
+	/* internal ib command identifier for the ringbuffer */
+	total_sizedwords += (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE) ? 2 : 0;
+
+	total_sizedwords += (secured_ctxt) ? 26 : 0;
+
+	/* context rollover */
+	if (adreno_is_a3xx(adreno_dev))
+		total_sizedwords += 3;
+
+	/* For HLSQ updates below */
+	if (adreno_is_a4xx(adreno_dev) || adreno_is_a3xx(adreno_dev))
+		total_sizedwords += 4;
+
+	if (gpudev->preemption_pre_ibsubmit &&
+				adreno_is_preemption_enabled(adreno_dev))
+		total_sizedwords += 20;
+
+	if (gpudev->preemption_post_ibsubmit &&
+				adreno_is_preemption_enabled(adreno_dev))
+		total_sizedwords += 13;
+
+	/*
+	 * a5xx uses 64 bit memory address. pm4 commands that involve read/write
+	 * from memory take 4 bytes more than a4xx because of 64 bit addressing.
+	 * This function is shared between gpucores, so reserve the max size
+	 * required in ringbuffer and adjust the write pointer depending on
+	 * gpucore at the end of this function.
+	 */
+	total_sizedwords += 4; /* sop timestamp */
+	total_sizedwords += 5; /* eop timestamp */
+
+	if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
+		total_sizedwords += 4; /* global timestamp without cache
+					* flush for non-zero context */
+	}
+
+	if (flags & KGSL_CMD_FLAGS_WFI)
+		total_sizedwords += 2; /* WFI */
+
+	if (profile_ready)
+		total_sizedwords += 8;   /* space for pre_ib and post_ib */
+
+	/* Add space for the power on shader fixup if we need it */
+	if (flags & KGSL_CMD_FLAGS_PWRON_FIXUP)
+		total_sizedwords += 9;
+
+	/* WAIT_MEM_WRITES - needed in the stall on fault case
+	 * to prevent out of order CP operations that can result
+	 * in a CACHE_FLUSH_TS interrupt storm */
+	if (test_bit(KGSL_FT_PAGEFAULT_GPUHALT_ENABLE,
+				&adreno_dev->ft_pf_policy))
+		total_sizedwords += 1;
+
+	ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
+	if (IS_ERR(ringcmds))
+		return PTR_ERR(ringcmds);
+
+	start = ringcmds;
+
+	*ringcmds++ = cp_packet(adreno_dev, CP_NOP, 1);
+	*ringcmds++ = KGSL_CMD_IDENTIFIER;
+
+	if (adreno_is_preemption_enabled(adreno_dev) &&
+				gpudev->preemption_pre_ibsubmit) {
+		cond_addr = device->memstore.gpuaddr +
+					KGSL_MEMSTORE_OFFSET(context_id,
+					 preempted);
+		ringcmds += gpudev->preemption_pre_ibsubmit(
+					adreno_dev, rb, ringcmds, context,
+					cond_addr, NULL);
+	}
+
+	if (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE) {
+		*ringcmds++ = cp_packet(adreno_dev, CP_NOP, 1);
+		*ringcmds++ = KGSL_CMD_INTERNAL_IDENTIFIER;
+	}
+
+	if (flags & KGSL_CMD_FLAGS_PWRON_FIXUP) {
+		/* Disable protected mode for the fixup */
+		*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+		*ringcmds++ = 0;
+
+		*ringcmds++ = cp_packet(adreno_dev, CP_NOP, 1);
+		*ringcmds++ = KGSL_PWRON_FIXUP_IDENTIFIER;
+		*ringcmds++ = cp_mem_packet(adreno_dev,
+				CP_INDIRECT_BUFFER_PFE, 2, 1);
+		ringcmds += cp_gpuaddr(adreno_dev, ringcmds,
+				adreno_dev->pwron_fixup.gpuaddr);
+		*ringcmds++ = adreno_dev->pwron_fixup_dwords;
+
+		/* Re-enable protected mode */
+		*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+		*ringcmds++ = 1;
+	}
+
+	/* Add any IB required for profiling if it is enabled */
+	if (profile_ready)
+		adreno_profile_preib_processing(adreno_dev, drawctxt,
+				&flags, &ringcmds);
+
+	/* start-of-pipeline timestamp */
+	*ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1);
+	if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
+		ringcmds += cp_gpuaddr(adreno_dev, ringcmds,
+			gpuaddr + KGSL_MEMSTORE_OFFSET(context_id,
+			soptimestamp));
+	else
+		ringcmds += cp_gpuaddr(adreno_dev, ringcmds,
+			gpuaddr + KGSL_MEMSTORE_RB_OFFSET(rb, soptimestamp));
+	*ringcmds++ = timestamp;
+
+	if (secured_ctxt)
+		ringcmds += cp_secure_mode(adreno_dev, ringcmds, 1);
+
+	if (flags & KGSL_CMD_FLAGS_PMODE) {
+		/* disable protected mode error checking */
+		*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+		*ringcmds++ = 0;
+	}
+
+	for (i = 0; i < sizedwords; i++)
+		*ringcmds++ = cmds[i];
+
+	if (flags & KGSL_CMD_FLAGS_PMODE) {
+		/* re-enable protected mode error checking */
+		*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+		*ringcmds++ = 1;
+	}
+
+	/*
+	 * Flush HLSQ lazy updates to make sure there are no
+	 * resources pending for indirect loads after the timestamp
+	 */
+	if (adreno_is_a4xx(adreno_dev) || adreno_is_a3xx(adreno_dev)) {
+		*ringcmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1);
+		*ringcmds++ = 0x07; /* HLSQ_FLUSH */
+		ringcmds += cp_wait_for_idle(adreno_dev, ringcmds);
+	}
+
+	/* Add any postIB required for profiling if it is enabled and has
+	   assigned counters */
+	if (profile_ready)
+		adreno_profile_postib_processing(adreno_dev, &flags, &ringcmds);
+
+	/*
+	 * WAIT_MEM_WRITES - needed in the stall on fault case to prevent
+	 * out of order CP operations that can result in a CACHE_FLUSH_TS
+	 * interrupt storm
+	 */
+	if (test_bit(KGSL_FT_PAGEFAULT_GPUHALT_ENABLE,
+				&adreno_dev->ft_pf_policy))
+		*ringcmds++ = cp_packet(adreno_dev, CP_WAIT_MEM_WRITES, 0);
+
+	/*
+	 * end-of-pipeline timestamp.  If per context timestamps is not
+	 * enabled, then drawctxt will be NULL or internal command flag will be
+	 * set and hence the rb timestamp will be used in else statement below.
+	 */
+	*ringcmds++ = cp_mem_packet(adreno_dev, CP_EVENT_WRITE, 3, 1);
+	if (drawctxt || (flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE))
+		*ringcmds++ = CACHE_FLUSH_TS | (1 << 31);
+	else
+		*ringcmds++ = CACHE_FLUSH_TS;
+
+	if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) {
+		ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr +
+				KGSL_MEMSTORE_OFFSET(context_id, eoptimestamp));
+		*ringcmds++ = timestamp;
+		*ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1);
+		ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr +
+				KGSL_MEMSTORE_RB_OFFSET(rb, eoptimestamp));
+		*ringcmds++ = rb->timestamp;
+	} else {
+		ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr +
+				KGSL_MEMSTORE_RB_OFFSET(rb, eoptimestamp));
+		*ringcmds++ = timestamp;
+	}
+
+	if (adreno_is_a3xx(adreno_dev)) {
+		/* Dummy set-constant to trigger context rollover */
+		*ringcmds++ = cp_packet(adreno_dev, CP_SET_CONSTANT, 2);
+		*ringcmds++ =
+			(0x4<<16) | (A3XX_HLSQ_CL_KERNEL_GROUP_X_REG - 0x2000);
+		*ringcmds++ = 0;
+	}
+
+	if (flags & KGSL_CMD_FLAGS_WFI) {
+		ringcmds += cp_wait_for_idle(adreno_dev, ringcmds);
+	}
+
+	if (secured_ctxt)
+		ringcmds += cp_secure_mode(adreno_dev, ringcmds, 0);
+
+	if (gpudev->preemption_post_ibsubmit &&
+				adreno_is_preemption_enabled(adreno_dev))
+		ringcmds += gpudev->preemption_post_ibsubmit(adreno_dev,
+					rb, ringcmds, &drawctxt->base);
+
+	/*
+	 * If we have more ringbuffer commands than space reserved
+	 * in ringbuffer BUG() to fix this because it will lead to
+	 * weird errors.
+	 */
+	if ((ringcmds - start) > total_sizedwords)
+		BUG();
+	/*
+	 *  Allocate total_sizedwords space in RB, this is the max space
+	 *  required. If we have commands less than the space reserved in RB
+	 *  adjust the wptr accordingly.
+	 */
+	rb->wptr = rb->wptr - (total_sizedwords - (ringcmds - start));
+
+	adreno_ringbuffer_submit(rb, time);
+
+	return 0;
+}
+
+int
+adreno_ringbuffer_issuecmds(struct adreno_ringbuffer *rb,
+				unsigned int flags,
+				unsigned int *cmds,
+				int sizedwords)
+{
+	flags |= KGSL_CMD_FLAGS_INTERNAL_ISSUE;
+
+	return adreno_ringbuffer_addcmds(rb, flags, cmds,
+		sizedwords, 0, NULL);
+}
+
+/**
+ * _ringbuffer_verify_ib() - Check if an IB's size is within a permitted limit
+ * @device: The kgsl device pointer
+ * @ibdesc: Pointer to the IB descriptor
+ */
+static inline bool _ringbuffer_verify_ib(struct kgsl_device_private *dev_priv,
+		struct kgsl_context *context, struct kgsl_memobj_node *ib)
+{
+	struct kgsl_device *device = dev_priv->device;
+	struct kgsl_process_private *private = dev_priv->process_priv;
+
+	/* The maximum allowable size for an IB in the CP is 0xFFFFF dwords */
+	if (ib->size == 0 || ((ib->size >> 2) > 0xFFFFF)) {
+		pr_context(device, context, "ctxt %d invalid ib size %lld\n",
+			context->id, ib->size);
+		return false;
+	}
+
+	/* Make sure that the address is mapped */
+	if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, ib->gpuaddr)) {
+		pr_context(device, context, "ctxt %d invalid ib gpuaddr %llX\n",
+			context->id, ib->gpuaddr);
+		return false;
+	}
+
+	return true;
+}
+
+int
+adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv,
+				struct kgsl_context *context,
+				struct kgsl_cmdbatch *cmdbatch,
+				uint32_t *timestamp)
+{
+	struct kgsl_device *device = dev_priv->device;
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+	struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
+	struct kgsl_memobj_node *ib;
+	int ret;
+
+	if (kgsl_context_invalid(context))
+		return -EDEADLK;
+
+	/* Verify the IBs before they get queued */
+	list_for_each_entry(ib, &cmdbatch->cmdlist, node)
+		if (_ringbuffer_verify_ib(dev_priv, context, ib) == false)
+			return -EINVAL;
+
+	/* wait for the suspend gate */
+	wait_for_completion(&device->cmdbatch_gate);
+
+	/*
+	 * Clear the wake on touch bit to indicate an IB has been
+	 * submitted since the last time we set it. But only clear
+	 * it when we have rendering commands.
+	 */
+	if (!(cmdbatch->flags & KGSL_CMDBATCH_MARKER)
+		&& !(cmdbatch->flags & KGSL_CMDBATCH_SYNC))
+		device->flags &= ~KGSL_FLAG_WAKE_ON_TOUCH;
+
+	/* Queue the command in the ringbuffer */
+	ret = adreno_dispatcher_queue_cmd(adreno_dev, drawctxt, cmdbatch,
+		timestamp);
+
+	/*
+	 * Return -EPROTO if the device has faulted since the last time we
+	 * checked - userspace uses this to perform post-fault activities
+	 */
+	if (!ret && test_and_clear_bit(ADRENO_CONTEXT_FAULT, &context->priv))
+		ret = -EPROTO;
+
+	return ret;
+}
+
+static void adreno_ringbuffer_set_constraint(struct kgsl_device *device,
+			struct kgsl_cmdbatch *cmdbatch)
+{
+	struct kgsl_context *context = cmdbatch->context;
+	/*
+	 * Check if the context has a constraint and constraint flags are
+	 * set.
+	 */
+	if (context->pwr_constraint.type &&
+		((context->flags & KGSL_CONTEXT_PWR_CONSTRAINT) ||
+			(cmdbatch->flags & KGSL_CONTEXT_PWR_CONSTRAINT)))
+		kgsl_pwrctrl_set_constraint(device, &context->pwr_constraint,
+						context->id);
+}
+
+static inline int _get_alwayson_counter(struct adreno_device *adreno_dev,
+		unsigned int *cmds, uint64_t gpuaddr)
+{
+	unsigned int *p = cmds;
+
+	*p++ = cp_mem_packet(adreno_dev, CP_REG_TO_MEM, 2, 1);
+
+	/*
+	 * For a4x and some a5x the alwayson_hi read through CPU
+	 * will be masked. Only do 32 bit CP reads for keeping the
+	 * numbers consistent
+	 */
+	if (ADRENO_GPUREV(adreno_dev) >= 400 &&
+		ADRENO_GPUREV(adreno_dev) <= ADRENO_REV_A530)
+		*p++ = adreno_getreg(adreno_dev,
+			ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO);
+	else
+		*p++ = adreno_getreg(adreno_dev,
+			ADRENO_REG_RBBM_ALWAYSON_COUNTER_LO) |
+			(1 << 30) | (2 << 18);
+	p += cp_gpuaddr(adreno_dev, p, gpuaddr);
+
+	return (unsigned int)(p - cmds);
+}
+
+/* adreno_rindbuffer_submitcmd - submit userspace IBs to the GPU */
+int adreno_ringbuffer_submitcmd(struct adreno_device *adreno_dev,
+		struct kgsl_cmdbatch *cmdbatch, struct adreno_submit_time *time)
+{
+	struct kgsl_device *device = &adreno_dev->dev;
+	struct kgsl_memobj_node *ib;
+	unsigned int numibs = 0;
+	unsigned int *link;
+	unsigned int *cmds;
+	struct kgsl_context *context;
+	struct adreno_context *drawctxt;
+	bool use_preamble = true;
+	bool cmdbatch_user_profiling = false;
+	bool cmdbatch_kernel_profiling = false;
+	int flags = KGSL_CMD_FLAGS_NONE;
+	int ret;
+	struct adreno_ringbuffer *rb;
+	struct kgsl_cmdbatch_profiling_buffer *profile_buffer = NULL;
+	unsigned int dwords = 0;
+	struct adreno_submit_time local;
+
+	struct kgsl_mem_entry *entry = cmdbatch->profiling_buf_entry;
+	if (entry)
+		profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc,
+					cmdbatch->profiling_buffer_gpuaddr);
+
+	context = cmdbatch->context;
+	drawctxt = ADRENO_CONTEXT(context);
+
+	/* Get the total IBs in the list */
+	list_for_each_entry(ib, &cmdbatch->cmdlist, node)
+		numibs++;
+
+	rb = drawctxt->rb;
+
+	/* process any profiling results that are available into the log_buf */
+	adreno_profile_process_results(adreno_dev);
+
+	/*
+	 * If SKIP CMD flag is set for current context
+	 * a) set SKIPCMD as fault_recovery for current commandbatch
+	 * b) store context's commandbatch fault_policy in current
+	 *    commandbatch fault_policy and clear context's commandbatch
+	 *    fault_policy
+	 * c) force preamble for commandbatch
+	 */
+	if (test_bit(ADRENO_CONTEXT_SKIP_CMD, &drawctxt->base.priv) &&
+		(!test_bit(CMDBATCH_FLAG_SKIP, &cmdbatch->priv))) {
+
+		set_bit(KGSL_FT_SKIPCMD, &cmdbatch->fault_recovery);
+		cmdbatch->fault_policy = drawctxt->fault_policy;
+		set_bit(CMDBATCH_FLAG_FORCE_PREAMBLE, &cmdbatch->priv);
+
+		/* if context is detached print fault recovery */
+		adreno_fault_skipcmd_detached(device, drawctxt, cmdbatch);
+
+		/* clear the drawctxt flags */
+		clear_bit(ADRENO_CONTEXT_SKIP_CMD, &drawctxt->base.priv);
+		drawctxt->fault_policy = 0;
+	}
+
+	/*When preamble is enabled, the preamble buffer with state restoration
+	commands are stored in the first node of the IB chain. We can skip that
+	if a context switch hasn't occured */
+
+	if ((drawctxt->base.flags & KGSL_CONTEXT_PREAMBLE) &&
+		!test_bit(CMDBATCH_FLAG_FORCE_PREAMBLE, &cmdbatch->priv) &&
+		(rb->drawctxt_active == drawctxt))
+		use_preamble = false;
+
+	/*
+	 * In skip mode don't issue the draw IBs but keep all the other
+	 * accoutrements of a submision (including the interrupt) to keep
+	 * the accounting sane. Set start_index and numibs to 0 to just
+	 * generate the start and end markers and skip everything else
+	 */
+	if (test_bit(CMDBATCH_FLAG_SKIP, &cmdbatch->priv)) {
+		use_preamble = false;
+		numibs = 0;
+	}
+
+	/*
+	 * a5xx uses 64 bit memory address. pm4 commands that involve read/write
+	 * from memory take 4 bytes more than a4xx because of 64 bit addressing.
+	 * This function is shared between gpucores, so reserve the max size
+	 * required and adjust the number of commands before calling addcmds.
+	 * Each submission needs 7 dwords max for wrappers and other red tape.
+	 */
+	dwords = 7;
+
+	/* Each IB takes up 30 dwords in worst case */
+	dwords += (numibs * 30);
+
+	if (cmdbatch->flags & KGSL_CMDBATCH_PROFILING &&
+		!adreno_is_a3xx(adreno_dev) && profile_buffer) {
+		cmdbatch_user_profiling = true;
+		dwords += 6;
+
+		/*
+		 * REG_TO_MEM packet on A5xx needs another ordinal.
+		 * Add 2 more dwords since we do profiling before and after.
+		 */
+		if (adreno_is_a5xx(adreno_dev))
+			dwords += 2;
+
+		/*
+		 * we want to use an adreno_submit_time struct to get the
+		 * precise moment when the command is submitted to the
+		 * ringbuffer.  If an upstream caller already passed down a
+		 * pointer piggyback on that otherwise use a local struct
+		 */
+
+		if (time == NULL)
+			time = &local;
+	}
+
+	if (test_bit(CMDBATCH_FLAG_PROFILE, &cmdbatch->priv)) {
+		cmdbatch_kernel_profiling = true;
+		dwords += 6;
+		if (adreno_is_a5xx(adreno_dev))
+			dwords += 2;
+	}
+
+	link = kzalloc(sizeof(unsigned int) *  dwords, GFP_KERNEL);
+	if (!link) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	cmds = link;
+
+	*cmds++ = cp_packet(adreno_dev, CP_NOP, 1);
+	*cmds++ = KGSL_START_OF_IB_IDENTIFIER;
+
+	if (cmdbatch_kernel_profiling) {
+		cmds += _get_alwayson_counter(adreno_dev, cmds,
+			adreno_dev->cmdbatch_profile_buffer.gpuaddr +
+			ADRENO_CMDBATCH_PROFILE_OFFSET(cmdbatch->profile_index,
+				started));
+	}
+
+	/*
+	 * Add cmds to read the GPU ticks at the start of the cmdbatch and
+	 * write it into the appropriate cmdbatch profiling buffer offset
+	 */
+	if (cmdbatch_user_profiling) {
+		cmds += _get_alwayson_counter(adreno_dev, cmds,
+			cmdbatch->profiling_buffer_gpuaddr +
+			offsetof(struct kgsl_cmdbatch_profiling_buffer,
+			gpu_ticks_submitted));
+	}
+
+	if (numibs) {
+		list_for_each_entry(ib, &cmdbatch->cmdlist, node) {
+			/*
+			 * Skip 0 sized IBs - these are presumed to have been
+			 * removed from consideration by the FT policy
+			 */
+			if (ib->priv & MEMOBJ_SKIP ||
+				(ib->priv & MEMOBJ_PREAMBLE &&
+				use_preamble == false))
+				*cmds++ = cp_mem_packet(adreno_dev, CP_NOP,
+						3, 1);
+
+			*cmds++ = cp_mem_packet(adreno_dev,
+					CP_INDIRECT_BUFFER_PFE, 2, 1);
+			cmds += cp_gpuaddr(adreno_dev, cmds, ib->gpuaddr);
+			*cmds++ = (unsigned int) ib->size >> 2;
+			/* preamble is required on only for first command */
+			use_preamble = false;
+		}
+	}
+
+	if (cmdbatch_kernel_profiling) {
+		cmds += _get_alwayson_counter(adreno_dev, cmds,
+			adreno_dev->cmdbatch_profile_buffer.gpuaddr +
+			ADRENO_CMDBATCH_PROFILE_OFFSET(cmdbatch->profile_index,
+				retired));
+	}
+
+	/*
+	 * Add cmds to read the GPU ticks at the end of the cmdbatch and
+	 * write it into the appropriate cmdbatch profiling buffer offset
+	 */
+	if (cmdbatch_user_profiling) {
+		cmds += _get_alwayson_counter(adreno_dev, cmds,
+			cmdbatch->profiling_buffer_gpuaddr +
+			offsetof(struct kgsl_cmdbatch_profiling_buffer,
+			gpu_ticks_retired));
+	}
+
+	*cmds++ = cp_packet(adreno_dev, CP_NOP, 1);
+	*cmds++ = KGSL_END_OF_IB_IDENTIFIER;
+
+	ret = adreno_drawctxt_switch(adreno_dev, rb, drawctxt, cmdbatch->flags);
+
+	/*
+	 * In the unlikely event of an error in the drawctxt switch,
+	 * treat it like a hang
+	 */
+	if (ret)
+		goto done;
+
+	if (test_bit(CMDBATCH_FLAG_WFI, &cmdbatch->priv))
+		flags = KGSL_CMD_FLAGS_WFI;
+
+	/*
+	 * For some targets, we need to execute a dummy shader operation after a
+	 * power collapse
+	 */
+
+	if (test_and_clear_bit(ADRENO_DEVICE_PWRON, &adreno_dev->priv) &&
+		test_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv))
+		flags |= KGSL_CMD_FLAGS_PWRON_FIXUP;
+
+	/* Set the constraints before adding to ringbuffer */
+	adreno_ringbuffer_set_constraint(device, cmdbatch);
+
+	/* CFF stuff executed only if CFF is enabled */
+	kgsl_cffdump_capture_ib_desc(device, context, cmdbatch);
+
+
+	ret = adreno_ringbuffer_addcmds(rb, flags,
+					&link[0], (cmds - link),
+					cmdbatch->timestamp, time);
+
+	if (!ret) {
+		cmdbatch->global_ts = drawctxt->internal_timestamp;
+
+		/* Put the timevalues in the profiling buffer */
+		if (cmdbatch_user_profiling) {
+			profile_buffer->wall_clock_s = time->utime.tv_sec;
+			profile_buffer->wall_clock_ns = time->utime.tv_nsec;
+			profile_buffer->gpu_ticks_queued = time->ticks;
+		}
+	}
+
+	kgsl_cffdump_regpoll(device,
+		adreno_getreg(adreno_dev, ADRENO_REG_RBBM_STATUS) << 2,
+		0x00000000, 0x80000000);
+done:
+	/* Corresponding unmap to the memdesc map of profile_buffer */
+	if (entry)
+		kgsl_memdesc_unmap(&entry->memdesc);
+
+
+	trace_kgsl_issueibcmds(device, context->id, cmdbatch,
+			numibs, cmdbatch->timestamp,
+			cmdbatch->flags, ret, drawctxt->type);
+
+	kfree(link);
+	return ret;
+}
+
+/**
+ * adreno_ringbuffer_mmu_clk_disable_event() - Callback function that
+ * disables the MMU clocks.
+ * @device: Device pointer
+ * @context: The ringbuffer context pointer
+ * @data: Pointer containing the adreno_mmu_disable_clk_param structure
+ * @type: The event call type (RETIRED or CANCELLED)
+ */
+static void adreno_ringbuffer_mmu_clk_disable_event(struct kgsl_device *device,
+			struct kgsl_event_group *group, void *data, int type)
+{
+	kgsl_mmu_disable_clk(&device->mmu);
+}
+
+/*
+ * adreno_ringbuffer_mmu_disable_clk_on_ts() - Sets up event to disable MMU
+ * clocks
+ * @device - The kgsl device pointer
+ * @rb: The ringbuffer in whose event list the event is added
+ * @timestamp: The timestamp on which the event should trigger
+ *
+ * Creates an event to disable the MMU clocks on timestamp and if event
+ * already exists then updates the timestamp of disabling the MMU clocks
+ * with the passed in ts if it is greater than the current value at which
+ * the clocks will be disabled
+ * Return - void
+ */
+void
+adreno_ringbuffer_mmu_disable_clk_on_ts(struct kgsl_device *device,
+			struct adreno_ringbuffer *rb, unsigned int timestamp)
+{
+	if (kgsl_add_event(device, &(rb->events), timestamp,
+		adreno_ringbuffer_mmu_clk_disable_event, NULL)) {
+		KGSL_DRV_ERR(device,
+			"Failed to add IOMMU disable clk event\n");
+	}
+}
+
+/**
+ * adreno_ringbuffer_wait_callback() - Callback function for event registered
+ * on a ringbuffer timestamp
+ * @device: Device for which the the callback is valid
+ * @context: The context of the event
+ * @priv: The private parameter of the event
+ * @result: Result of the event trigger
+ */
+static void adreno_ringbuffer_wait_callback(struct kgsl_device *device,
+		struct kgsl_event_group *group,
+		void *priv, int result)
+{
+	struct adreno_ringbuffer *rb = group->priv;
+	wake_up_all(&rb->ts_expire_waitq);
+}
+
+/**
+ * adreno_ringbuffer_waittimestamp() - Wait for a RB timestamp
+ * @rb: The ringbuffer to wait on
+ * @timestamp: The timestamp to wait for
+ * @msecs: The wait timeout period
+ */
+int adreno_ringbuffer_waittimestamp(struct adreno_ringbuffer *rb,
+					unsigned int timestamp,
+					unsigned int msecs)
+{
+	struct kgsl_device *device = rb->device;
+	int ret;
+	unsigned long wait_time;
+
+	/* force a timeout from caller for the wait */
+	BUG_ON(0 == msecs);
+
+	ret = kgsl_add_event(device, &rb->events, timestamp,
+		adreno_ringbuffer_wait_callback, NULL);
+	if (ret)
+		return ret;
+
+	mutex_unlock(&device->mutex);
+
+	wait_time = msecs_to_jiffies(msecs);
+	if (0 == wait_event_timeout(rb->ts_expire_waitq,
+		!kgsl_event_pending(device, &rb->events, timestamp,
+				adreno_ringbuffer_wait_callback, NULL),
+		wait_time))
+		ret  = -ETIMEDOUT;
+
+	mutex_lock(&device->mutex);
+	/*
+	 * after wake up make sure that expected timestamp has retired
+	 * because the wakeup could have happened due to a cancel event
+	 */
+	if (!ret && !adreno_ringbuffer_check_timestamp(rb,
+		timestamp, KGSL_TIMESTAMP_RETIRED)) {
+		ret = -EAGAIN;
+	}
+
+	return ret;
+}
+
+/**
+ * adreno_ringbuffer_submit_preempt_token() - Submit a preempt token
+ * @rb: Ringbuffer in which the token is submitted
+ * @incoming_rb: The RB to which the GPU switches when this preemption
+ * token is executed.
+ *
+ * Called to make sure that an outstanding preemption request is
+ * granted.
+ */
+int adreno_ringbuffer_submit_preempt_token(struct adreno_ringbuffer *rb,
+					struct adreno_ringbuffer *incoming_rb)
+{
+	unsigned int *ringcmds, *start;
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(rb->device);
+	struct kgsl_device *device = &(adreno_dev->dev);
+	struct kgsl_iommu *iommu = device->mmu.priv;
+	struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+	int ptname;
+	struct kgsl_pagetable *pt;
+	int pt_switch_sizedwords = 0, total_sizedwords = 20;
+	unsigned link[ADRENO_RB_PREEMPT_TOKEN_DWORDS];
+	uint i;
+	uint64_t ttbr0;
+
+	if (incoming_rb->preempted_midway) {
+
+		if (adreno_is_a5xx(adreno_dev)) {
+			kgsl_sharedmem_readq(&rb->pagetable_desc, &ttbr0,
+				offsetof(struct adreno_ringbuffer_pagetable_info
+				, ttbr0));
+			kgsl_sharedmem_writeq(rb->device, &iommu->smmu_info,
+				offsetof(struct a5xx_cp_smmu_info, ttbr0),
+				ttbr0);
+		} else {
+			kgsl_sharedmem_readl(&incoming_rb->pagetable_desc,
+				&ptname, offsetof(
+				struct adreno_ringbuffer_pagetable_info,
+				current_rb_ptname));
+			pt = kgsl_mmu_get_pt_from_ptname(&(rb->device->mmu),
+				ptname);
+			/*
+			 * always expect a valid pt, else pt refcounting is
+			 * messed up or current pt tracking has a bug which
+			 * could lead to eventual disaster
+			 */
+			BUG_ON(!pt);
+			/* set the ringbuffer for incoming RB */
+			pt_switch_sizedwords =
+				adreno_iommu_set_pt_generate_cmds(incoming_rb,
+								&link[0], pt);
+			total_sizedwords += pt_switch_sizedwords;
+
+		}
+	}
+
+	/*
+	 *  Allocate total_sizedwords space in RB, this is the max space
+	 *  required.
+	 */
+	ringcmds = adreno_ringbuffer_allocspace(rb, total_sizedwords);
+
+	if (IS_ERR(ringcmds))
+		return PTR_ERR(ringcmds);
+
+	start = ringcmds;
+
+	*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+	*ringcmds++ = 0;
+
+	if (incoming_rb->preempted_midway) {
+		for (i = 0; i < pt_switch_sizedwords; i++)
+			*ringcmds++ = link[i];
+	}
+
+	*ringcmds++ = cp_register(adreno_dev, adreno_getreg(adreno_dev,
+			ADRENO_REG_CP_PREEMPT_DISABLE), 1);
+	*ringcmds++ = 0;
+
+	*ringcmds++ = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
+	*ringcmds++ = 1;
+
+	ringcmds += gpudev->preemption_token(adreno_dev, rb, ringcmds,
+				rb->device->memstore.gpuaddr +
+				KGSL_MEMSTORE_RB_OFFSET(rb, preempted));
+
+	if ((uint)(ringcmds - start) > total_sizedwords) {
+		KGSL_DRV_ERR(device, "Insufficient rb size allocated\n");
+		BUG();
+	}
+
+	/*
+	 * If we have commands less than the space reserved in RB
+	 *  adjust the wptr accordingly
+	 */
+	rb->wptr = rb->wptr - (total_sizedwords - (uint)(ringcmds - start));
+
+	/* submit just the preempt token */
+	mb();
+	kgsl_pwrscale_busy(rb->device);
+	adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR, rb->wptr);
+	return 0;
+}