diff options
| -rw-r--r-- | drivers/gpu/msm/a5xx_reg.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno.c | 36 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno.h | 27 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_a4xx.c | 64 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_a5xx.c | 80 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_dispatch.c | 47 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_iommu.c | 15 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_ringbuffer.c | 31 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_ringbuffer.h | 5 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_snapshot.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/msm/adreno_trace.h | 71 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl.c | 16 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl.h | 26 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_device.h | 1 |
14 files changed, 233 insertions, 190 deletions
diff --git a/drivers/gpu/msm/a5xx_reg.h b/drivers/gpu/msm/a5xx_reg.h index 913cedb885ad..207588844931 100644 --- a/drivers/gpu/msm/a5xx_reg.h +++ b/drivers/gpu/msm/a5xx_reg.h @@ -60,6 +60,8 @@ #define A5XX_CP_RB_BASE 0x800 #define A5XX_CP_RB_BASE_HI 0x801 #define A5XX_CP_RB_CNTL 0x802 +#define A5XX_CP_RB_RPTR_ADDR_LO 0x804 +#define A5XX_CP_RB_RPTR_ADDR_HI 0x805 #define A5XX_CP_RB_RPTR 0x806 #define A5XX_CP_RB_WPTR 0x807 #define A5XX_CP_PFP_STAT_ADDR 0x808 diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index 26e341a876e8..28980ec1e07d 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -171,6 +171,30 @@ void adreno_writereg64(struct adreno_device *adreno_dev, } /** + * adreno_get_rptr() - Get the current ringbuffer read pointer + * @rb: Pointer the ringbuffer to query + * + * Get the latest rptr + */ +unsigned int adreno_get_rptr(struct adreno_ringbuffer *rb) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + unsigned int rptr = 0; + + if (adreno_is_a3xx(adreno_dev)) + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, + &rptr); + else { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_sharedmem_readl(&device->scratch, &rptr, + SCRATCH_RPTR_OFFSET(rb->id)); + } + + return rptr; +} + +/** * adreno_of_read_property() - Adreno read property * @node: Device node * @@ -2149,8 +2173,6 @@ bool adreno_isidle(struct kgsl_device *device) if (!kgsl_state_is_awake(device)) return true; - adreno_get_rptr(ADRENO_CURRENT_RINGBUFFER(adreno_dev)); - /* * wptr is updated when we add commands to ringbuffer, add a barrier * to make sure updated wptr is compared to rptr @@ -2161,15 +2183,13 @@ bool adreno_isidle(struct kgsl_device *device) * ringbuffer is truly idle when all ringbuffers read and write * pointers are equal */ + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { - if (rb->rptr != rb->wptr) - break; + if (!adreno_rb_empty(rb)) + return false; } - if (i == adreno_dev->num_ringbuffers) - return adreno_hw_isidle(adreno_dev); - - return false; + return adreno_hw_isidle(adreno_dev); } /** diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 7ac91f203a70..22827e7ca284 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -458,6 +458,8 @@ enum adreno_regs { ADRENO_REG_CP_WFI_PEND_CTR, ADRENO_REG_CP_RB_BASE, ADRENO_REG_CP_RB_BASE_HI, + ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, ADRENO_REG_CP_RB_RPTR, ADRENO_REG_CP_RB_WPTR, ADRENO_REG_CP_CNTL, @@ -1272,24 +1274,6 @@ static inline unsigned int adreno_preempt_state( state; } -/** - * adreno_get_rptr() - Get the current ringbuffer read pointer - * @rb: Pointer the ringbuffer to query - * - * Get the current read pointer from the GPU register. - */ -static inline unsigned int -adreno_get_rptr(struct adreno_ringbuffer *rb) -{ - struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); - if (adreno_dev->cur_rb == rb && - adreno_preempt_state(adreno_dev, - ADRENO_DISPATCHER_PREEMPT_CLEAR)) - adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &(rb->rptr)); - - return rb->rptr; -} - static inline bool adreno_is_preemption_enabled( struct adreno_device *adreno_dev) { @@ -1371,6 +1355,13 @@ void adreno_readreg64(struct adreno_device *adreno_dev, void adreno_writereg64(struct adreno_device *adreno_dev, enum adreno_regs lo, enum adreno_regs hi, uint64_t val); +unsigned int adreno_get_rptr(struct adreno_ringbuffer *rb); + +static inline bool adreno_rb_empty(struct adreno_ringbuffer *rb) +{ + return (adreno_get_rptr(rb) == rb->wptr); +} + static inline bool adreno_soft_fault_detect(struct adreno_device *adreno_dev) { return adreno_dev->fast_hang_detect && diff --git a/drivers/gpu/msm/adreno_a4xx.c b/drivers/gpu/msm/adreno_a4xx.c index b1196da0cee1..29e3316907de 100644 --- a/drivers/gpu/msm/adreno_a4xx.c +++ b/drivers/gpu/msm/adreno_a4xx.c @@ -196,9 +196,10 @@ static void a4xx_preemption_start(struct adreno_device *adreno_dev, /* scratch REG9 corresponds to CP_RB_CNTL register */ kgsl_regwrite(device, A4XX_CP_SCRATCH_REG9, val); /* scratch REG10 corresponds to rptr address */ - kgsl_regwrite(device, A4XX_CP_SCRATCH_REG10, 0); + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG10, + SCRATCH_RPTR_GPU_ADDR(device, rb->id)); /* scratch REG11 corresponds to rptr */ - kgsl_regwrite(device, A4XX_CP_SCRATCH_REG11, rb->rptr); + kgsl_regwrite(device, A4XX_CP_SCRATCH_REG11, adreno_get_rptr(rb)); /* scratch REG12 corresponds to wptr */ kgsl_regwrite(device, A4XX_CP_SCRATCH_REG12, rb->wptr); /* @@ -222,7 +223,6 @@ static void a4xx_preemption_save(struct adreno_device *adreno_dev, { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - kgsl_regread(device, A4XX_CP_SCRATCH_REG18, &rb->rptr); kgsl_regread(device, A4XX_CP_SCRATCH_REG23, &rb->gpr11); } @@ -255,8 +255,7 @@ static int a4xx_preemption_pre_ibsubmit( int exec_ib = 0; cmds += a4xx_preemption_token(adreno_dev, rb, cmds, - device->memstore.gpuaddr + - KGSL_MEMSTORE_OFFSET(context->id, preempted)); + MEMSTORE_ID_GPU_ADDR(device, context->id, preempted)); if (ib) exec_ib = 1; @@ -839,6 +838,7 @@ static unsigned int a4xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { ADRENO_REG_DEFINE(ADRENO_REG_CP_WFI_PEND_CTR, A4XX_CP_WFI_PEND_CTR), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A4XX_CP_RB_BASE), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, A4XX_CP_RB_RPTR_ADDR), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A4XX_CP_RB_RPTR), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A4XX_CP_RB_WPTR), ADRENO_REG_DEFINE(ADRENO_REG_CP_CNTL, A4XX_CP_CNTL), @@ -1634,8 +1634,15 @@ static int a4xx_rb_start(struct adreno_device *adreno_dev, unsigned int start_type) { struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + struct kgsl_device *device = &adreno_dev->dev; + uint64_t addr; int ret; + addr = SCRATCH_RPTR_GPU_ADDR(device, rb->id); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, addr); + /* * The size of the ringbuffer in the hardware is the log2 * representation of the size in quadwords (sizedwords / 2). @@ -1644,8 +1651,8 @@ static int a4xx_rb_start(struct adreno_device *adreno_dev, */ adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, - (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F) | - (1 << 27)); + ((ilog2(4) << 8) & 0x1F00) | + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F)); adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); @@ -1897,7 +1904,7 @@ static int a4xx_submit_preempt_token(struct adreno_ringbuffer *rb, ringcmds += gpudev->preemption_token(adreno_dev, rb, ringcmds, device->memstore.gpuaddr + - KGSL_MEMSTORE_RB_OFFSET(rb, preempted)); + MEMSTORE_RB_OFFSET(rb, preempted)); if ((uint)(ringcmds - start) > total_sizedwords) { KGSL_DRV_ERR(device, "Insufficient rb size allocated\n"); @@ -1960,7 +1967,9 @@ static void a4xx_preempt_trig_state( KGSL_DRV_INFO(device, "Preemption completed without interrupt\n"); trace_adreno_hw_preempt_trig_to_comp(adreno_dev->cur_rb, - adreno_dev->next_rb); + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_COMPLETE); adreno_dispatcher_schedule(device); @@ -1988,9 +1997,7 @@ static void a4xx_preempt_trig_state( * commands that got submitted to current RB after triggering preemption * then submit them as those commands may have a preempt token in them */ - adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, - &adreno_dev->cur_rb->rptr); - if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) { + if (!adreno_rb_empty(adreno_dev->cur_rb)) { /* * Memory barrier before informing the * hardware of new commands @@ -2011,7 +2018,9 @@ static void a4xx_preempt_trig_state( dispatcher->preempt_token_submit = 1; adreno_dev->cur_rb->wptr_preempt_end = adreno_dev->cur_rb->wptr; trace_adreno_hw_preempt_token_submit(adreno_dev->cur_rb, - adreno_dev->next_rb); + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); } /** @@ -2035,10 +2044,6 @@ static void a4xx_preempt_clear_state( if (!kgsl_state_is_awake(device)) return; - /* keep updating the current rptr when preemption is clear */ - adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, - &(adreno_dev->cur_rb->rptr)); - highest_busy_rb = adreno_dispatcher_get_highest_busy_rb(adreno_dev); if (!highest_busy_rb) return; @@ -2056,7 +2061,7 @@ static void a4xx_preempt_clear_state( * if switching to lower priority make sure that the rptr and * wptr are equal, when the lower rb is not starved */ - if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) + if (!adreno_rb_empty(adreno_dev->cur_rb)) return; /* * switch to default context because when we switch back @@ -2095,7 +2100,9 @@ static void a4xx_preempt_clear_state( msecs_to_jiffies(ADRENO_DISPATCH_PREEMPT_TIMEOUT)); trace_adreno_hw_preempt_clear_to_trig(adreno_dev->cur_rb, - adreno_dev->next_rb); + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); /* issue PREEMPT trigger */ adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1); /* @@ -2146,6 +2153,7 @@ static void a4xx_preempt_complete_state( struct adreno_dispatcher_cmdqueue *dispatch_q; unsigned int wptr, rbbase; unsigned int val, val1; + unsigned int prevrptr; del_timer_sync(&dispatcher->preempt_timer); @@ -2176,12 +2184,15 @@ static void a4xx_preempt_complete_state( dispatch_q = &(adreno_dev->cur_rb->dispatch_q); /* new RB is the current RB */ trace_adreno_hw_preempt_comp_to_clear(adreno_dev->next_rb, - adreno_dev->cur_rb); + adreno_dev->cur_rb, + adreno_get_rptr(adreno_dev->next_rb), + adreno_get_rptr(adreno_dev->cur_rb)); adreno_dev->prev_rb = adreno_dev->cur_rb; adreno_dev->cur_rb = adreno_dev->next_rb; adreno_dev->cur_rb->preempted_midway = 0; adreno_dev->cur_rb->wptr_preempt_end = 0xFFFFFFFF; adreno_dev->next_rb = NULL; + if (adreno_disp_preempt_fair_sched) { /* starved rb is now scheduled so unhalt dispatcher */ if (ADRENO_DISPATCHER_RB_STARVE_TIMER_ELAPSED == @@ -2194,7 +2205,7 @@ static void a4xx_preempt_complete_state( * If the outgoing RB is has commands then set the * busy time for it */ - if (adreno_dev->prev_rb->rptr != adreno_dev->prev_rb->wptr) { + if (!adreno_rb_empty(adreno_dev->prev_rb)) { adreno_dev->prev_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_INIT; adreno_dev->prev_rb->sched_timer = jiffies; @@ -2205,15 +2216,16 @@ static void a4xx_preempt_complete_state( } atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_CLEAR); + + prevrptr = adreno_get_rptr(adreno_dev->prev_rb); + if (adreno_compare_prio_level(adreno_dev->prev_rb->id, adreno_dev->cur_rb->id) < 0) { - if (adreno_dev->prev_rb->wptr_preempt_end != - adreno_dev->prev_rb->rptr) + if (adreno_dev->prev_rb->wptr_preempt_end != prevrptr) adreno_dev->prev_rb->preempted_midway = 1; - } else if (adreno_dev->prev_rb->wptr_preempt_end != - adreno_dev->prev_rb->rptr) { + } else if (adreno_dev->prev_rb->wptr_preempt_end != prevrptr) BUG(); - } + /* submit wptr if required for new rb */ adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); if (adreno_dev->cur_rb->wptr != wptr) { diff --git a/drivers/gpu/msm/adreno_a5xx.c b/drivers/gpu/msm/adreno_a5xx.c index 512dcd483f45..f5ed9098fc6a 100644 --- a/drivers/gpu/msm/adreno_a5xx.c +++ b/drivers/gpu/msm/adreno_a5xx.c @@ -194,6 +194,8 @@ static void a5xx_preemption_start(struct adreno_device *adreno_dev, kgsl_sharedmem_writel(device, &rb->preemption_desc, PREEMPT_RECORD(wptr), rb->wptr); + kgsl_sharedmem_writel(device, &rb->preemption_desc, + PREEMPT_RECORD(rptr), adreno_get_rptr(rb)); kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, lower_32_bits(rb->preemption_desc.gpuaddr)); kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI, @@ -225,16 +227,8 @@ static void a5xx_preemption_start(struct adreno_device *adreno_dev, offsetof(struct a5xx_cp_smmu_info, context_idr), contextidr); } -/* - * a5xx_preemption_save() - Save the state after preemption is done - */ -static void a5xx_preemption_save(struct adreno_device *adreno_dev, - struct adreno_ringbuffer *rb) -{ - /* save the rptr from ctxrecord here */ - kgsl_sharedmem_readl(&rb->preemption_desc, &rb->rptr, - PREEMPT_RECORD(rptr)); -} +#define _CP_CNTL (((ilog2(4) << 8) & 0x1F00) | \ + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F)) #ifdef CONFIG_QCOM_KGSL_IOMMU static int a5xx_preemption_iommu_init(struct adreno_device *adreno_dev) @@ -290,7 +284,10 @@ static int a5xx_preemption_init(struct adreno_device *adreno_dev) kgsl_sharedmem_writel(device, &rb->preemption_desc, PREEMPT_RECORD(data), 0); kgsl_sharedmem_writel(device, &rb->preemption_desc, - PREEMPT_RECORD(cntl), 0x0800000C); + PREEMPT_RECORD(cntl), _CP_CNTL); + kgsl_sharedmem_writeq(device, &rb->preemption_desc, + PREEMPT_RECORD(rptr_addr), + SCRATCH_RPTR_GPU_ADDR(device, i)); kgsl_sharedmem_writel(device, &rb->preemption_desc, PREEMPT_RECORD(rptr), 0); kgsl_sharedmem_writel(device, &rb->preemption_desc, @@ -445,9 +442,7 @@ static int a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, unsigned int ctx_id = context ? context->id : 0; return a5xx_preemption_token(adreno_dev, rb, cmds, - device->memstore.gpuaddr + - KGSL_MEMSTORE_OFFSET(ctx_id, preempted)); - + MEMSTORE_ID_GPU_ADDR(device, ctx_id, preempted)); } static void a5xx_platform_setup(struct adreno_device *adreno_dev) @@ -2228,8 +2223,8 @@ static int _preemption_init( struct kgsl_device *device = KGSL_DEVICE(adreno_dev); unsigned int *cmds_orig = cmds; uint64_t gpuaddr = rb->preemption_desc.gpuaddr; - uint64_t gpuaddr_token = device->memstore.gpuaddr + - KGSL_MEMSTORE_OFFSET(0, preempted); + uint64_t gpuaddr_token = MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, preempted); /* Turn CP protection OFF */ *cmds++ = cp_type7_packet(CP_SET_PROTECTED_MODE, 1); @@ -2595,8 +2590,15 @@ static int a5xx_rb_start(struct adreno_device *adreno_dev, unsigned int start_type) { struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + struct kgsl_device *device = &adreno_dev->dev; + uint64_t addr; int ret; + addr = SCRATCH_RPTR_GPU_ADDR(device, rb->id); + + adreno_writereg64(adreno_dev, ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, addr); + /* * The size of the ringbuffer in the hardware is the log2 * representation of the size in quadwords (sizedwords / 2). @@ -2604,9 +2606,7 @@ static int a5xx_rb_start(struct adreno_device *adreno_dev, * in certain circumstances. */ - adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, - (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F) | - (1 << 27)); + adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_CNTL, _CP_CNTL); adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_BASE, rb->buffer_desc.gpuaddr); @@ -3147,6 +3147,10 @@ static unsigned int a5xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { ADRENO_REG_DEFINE(ADRENO_REG_CP_WFI_PEND_CTR, A5XX_CP_WFI_PEND_CTR), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A5XX_CP_RB_BASE), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, A5XX_CP_RB_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, + A5XX_CP_RB_RPTR_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_HI, + A5XX_CP_RB_RPTR_ADDR_HI), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A5XX_CP_RB_RPTR), ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A5XX_CP_RB_WPTR), ADRENO_REG_DEFINE(ADRENO_REG_CP_CNTL, A5XX_CP_CNTL), @@ -3805,7 +3809,9 @@ static void a5xx_preempt_trig_state( "Preemption completed without interrupt\n"); trace_adreno_hw_preempt_trig_to_comp( adreno_dev->cur_rb, - adreno_dev->next_rb); + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_COMPLETE); } else { @@ -3867,10 +3873,6 @@ static void a5xx_preempt_clear_state( if (!kgsl_state_is_awake(device)) return; - /* keep updating the current rptr when preemption is clear */ - adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, - &(adreno_dev->cur_rb->rptr)); - highest_busy_rb = adreno_dispatcher_get_highest_busy_rb(adreno_dev); if (!highest_busy_rb) return; @@ -3893,7 +3895,7 @@ static void a5xx_preempt_clear_state( * if switching to lower priority make sure that the rptr and * wptr are equal, when the lower rb is not starved */ - if (adreno_dev->cur_rb->rptr != adreno_dev->cur_rb->wptr) + if (!adreno_rb_empty(adreno_dev->cur_rb)) return; /* * switch to default context because when we switch back @@ -3912,10 +3914,6 @@ static void a5xx_preempt_clear_state( return; } - /* rptr could be updated in drawctxt switch above, update it here */ - adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, - &(adreno_dev->cur_rb->rptr)); - /* turn on IOMMU as the preemption may trigger pt switch */ kgsl_mmu_enable_clk(&device->mmu); @@ -3933,7 +3931,9 @@ static void a5xx_preempt_clear_state( msecs_to_jiffies(ADRENO_DISPATCH_PREEMPT_TIMEOUT)); trace_adreno_hw_preempt_clear_to_trig(adreno_dev->cur_rb, - adreno_dev->next_rb); + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); /* issue PREEMPT trigger */ adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1); @@ -3993,12 +3993,12 @@ static void a5xx_preempt_complete_state( return; } - a5xx_preemption_save(adreno_dev, adreno_dev->cur_rb); - dispatch_q = &(adreno_dev->cur_rb->dispatch_q); /* new RB is the current RB */ trace_adreno_hw_preempt_comp_to_clear(adreno_dev->next_rb, - adreno_dev->cur_rb); + adreno_dev->cur_rb, + adreno_get_rptr(adreno_dev->next_rb), + adreno_get_rptr(adreno_dev->cur_rb)); adreno_dev->prev_rb = adreno_dev->cur_rb; adreno_dev->cur_rb = adreno_dev->next_rb; adreno_dev->cur_rb->preempted_midway = 0; @@ -4017,7 +4017,8 @@ static void a5xx_preempt_complete_state( * If the outgoing RB is has commands then set the * busy time for it */ - if (adreno_dev->prev_rb->rptr != adreno_dev->prev_rb->wptr) { + + if (!adreno_rb_empty(adreno_dev->prev_rb)) { adreno_dev->prev_rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_INIT; adreno_dev->prev_rb->sched_timer = jiffies; @@ -4043,13 +4044,10 @@ static void a5xx_preempt_complete_state( adreno_preempt_process_dispatch_queue(adreno_dev, dispatch_q); } -static void a5xx_preemption_schedule( - struct adreno_device *adreno_dev) +static void a5xx_preemption_schedule(struct adreno_device *adreno_dev) { - struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct adreno_ringbuffer *rb; - int i = 0; + struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; if (!adreno_is_preemption_enabled(adreno_dev)) return; @@ -4062,10 +4060,6 @@ static void a5xx_preemption_schedule( */ smp_mb(); - if (KGSL_STATE_ACTIVE == device->state) - FOR_EACH_RINGBUFFER(adreno_dev, rb, i) - rb->rptr = adreno_get_rptr(rb); - switch (atomic_read(&dispatcher->preemption_state)) { case ADRENO_DISPATCHER_PREEMPT_CLEAR: a5xx_preempt_clear_state(adreno_dev); diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index 8140b726fd0f..a549d0d7f23e 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -283,7 +283,8 @@ static void _retire_marker(struct kgsl_cmdbatch *cmdbatch) /* Retire pending GPU events for the object */ kgsl_process_event_group(device, &context->events); - trace_adreno_cmdbatch_retired(cmdbatch, -1, 0, 0, drawctxt->rb); + trace_adreno_cmdbatch_retired(cmdbatch, -1, 0, 0, drawctxt->rb, + adreno_get_rptr(drawctxt->rb)); kgsl_cmdbatch_destroy(cmdbatch); } @@ -616,7 +617,8 @@ static int sendcmd(struct adreno_device *adreno_dev, nsecs = do_div(secs, 1000000000); trace_adreno_cmdbatch_submitted(cmdbatch, (int) dispatcher->inflight, - time.ticks, (unsigned long) secs, nsecs / 1000, drawctxt->rb); + time.ticks, (unsigned long) secs, nsecs / 1000, drawctxt->rb, + adreno_get_rptr(drawctxt->rb)); cmdbatch->submit_ticks = time.ticks; @@ -935,16 +937,18 @@ static int get_timestamp(struct adreno_context *drawctxt, static void adreno_dispatcher_preempt_timer(unsigned long data) { struct adreno_device *adreno_dev = (struct adreno_device *) data; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; + unsigned int cur_rptr = adreno_get_rptr(adreno_dev->cur_rb); + unsigned int next_rptr = adreno_get_rptr(adreno_dev->cur_rb); - KGSL_DRV_ERR(KGSL_DEVICE(adreno_dev), + KGSL_DRV_ERR(device, "Preemption timed out. cur_rb rptr/wptr %x/%x id %d, next_rb rptr/wptr %x/%x id %d, disp_state: %d\n", - adreno_dev->cur_rb->rptr, adreno_dev->cur_rb->wptr, - adreno_dev->cur_rb->id, adreno_dev->next_rb->rptr, - adreno_dev->next_rb->wptr, adreno_dev->next_rb->id, + cur_rptr, adreno_dev->cur_rb->wptr, adreno_dev->cur_rb->id, + next_rptr, adreno_dev->next_rb->wptr, adreno_dev->next_rb->id, atomic_read(&dispatcher->preemption_state)); adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT); - adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev)); + adreno_dispatcher_schedule(device); } /** @@ -957,9 +961,11 @@ struct adreno_ringbuffer *adreno_dispatcher_get_highest_busy_rb( { struct adreno_ringbuffer *rb, *highest_busy_rb = NULL; int i; + unsigned int rptr; FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { - if (rb->rptr != rb->wptr && !highest_busy_rb) { + rptr = adreno_get_rptr(rb); + if (rptr != rb->wptr && !highest_busy_rb) { highest_busy_rb = rb; goto done; } @@ -969,7 +975,7 @@ struct adreno_ringbuffer *adreno_dispatcher_get_highest_busy_rb( switch (rb->starve_timer_state) { case ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT: - if (rb->rptr != rb->wptr && + if (rptr != rb->wptr && adreno_dev->cur_rb != rb) { rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_INIT; @@ -991,7 +997,7 @@ struct adreno_ringbuffer *adreno_dispatcher_get_highest_busy_rb( * If the RB has not been running for the minimum * time slice then allow it to run */ - if ((rb->rptr != rb->wptr) && time_before(jiffies, + if ((rptr != rb->wptr) && time_before(jiffies, adreno_dev->cur_rb->sched_timer + msecs_to_jiffies(_dispatch_time_slice))) highest_busy_rb = rb; @@ -1437,7 +1443,7 @@ static void adreno_fault_header(struct kgsl_device *device, if (rb != NULL) pr_fault(device, cmdbatch, "gpu fault rb %d rb sw r/w %4.4x/%4.4x\n", - rb->id, rb->rptr, rb->wptr); + rb->id, rptr, rb->wptr); } else { int id = (rb != NULL) ? rb->id : -1; @@ -1448,7 +1454,7 @@ static void adreno_fault_header(struct kgsl_device *device, if (rb != NULL) dev_err(device->dev, "RB[%d] gpu fault rb sw r/w %4.4x/%4.4x\n", - rb->id, rb->rptr, rb->wptr); + rb->id, rptr, rb->wptr); } } @@ -1823,8 +1829,6 @@ static int dispatcher_do_fault(struct adreno_device *adreno_dev) if (base == rb->buffer_desc.gpuaddr) { dispatch_q = &(rb->dispatch_q); hung_rb = rb; - adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, - &hung_rb->rptr); if (adreno_dev->cur_rb != hung_rb) { adreno_dev->prev_rb = adreno_dev->cur_rb; adreno_dev->cur_rb = hung_rb; @@ -1879,12 +1883,12 @@ static int dispatcher_do_fault(struct adreno_device *adreno_dev) if (hung_rb != NULL) { kgsl_sharedmem_writel(device, &device->memstore, - KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_MAX + hung_rb->id, - soptimestamp), hung_rb->timestamp); + MEMSTORE_RB_OFFSET(hung_rb, soptimestamp), + hung_rb->timestamp); kgsl_sharedmem_writel(device, &device->memstore, - KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_MAX + hung_rb->id, - eoptimestamp), hung_rb->timestamp); + MEMSTORE_RB_OFFSET(hung_rb, eoptimestamp), + hung_rb->timestamp); /* Schedule any pending events to be run */ kgsl_process_event_group(device, &hung_rb->events); @@ -2016,7 +2020,8 @@ int adreno_dispatch_process_cmdqueue(struct adreno_device *adreno_dev, trace_adreno_cmdbatch_retired(cmdbatch, (int) dispatcher->inflight, start_ticks, - retire_ticks, ADRENO_CMDBATCH_RB(cmdbatch)); + retire_ticks, ADRENO_CMDBATCH_RB(cmdbatch), + adreno_get_rptr(drawctxt->rb)); /* Record the delta between submit and retire ticks */ drawctxt->submit_retire_ticks[drawctxt->ticks_index] = @@ -2589,7 +2594,9 @@ void adreno_dispatcher_preempt_callback(struct adreno_device *adreno_dev, return; trace_adreno_hw_preempt_trig_to_comp_int(adreno_dev->cur_rb, - adreno_dev->next_rb); + adreno_dev->next_rb, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_get_rptr(adreno_dev->next_rb)); atomic_set(&dispatcher->preemption_state, ADRENO_DISPATCHER_PREEMPT_COMPLETE); adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev)); diff --git a/drivers/gpu/msm/adreno_iommu.c b/drivers/gpu/msm/adreno_iommu.c index 2eeda01b3c4d..22dd8feea2e9 100644 --- a/drivers/gpu/msm/adreno_iommu.c +++ b/drivers/gpu/msm/adreno_iommu.c @@ -275,6 +275,7 @@ static bool _ctx_switch_use_cpu_path( struct adreno_ringbuffer *rb) { struct kgsl_mmu *mmu = KGSL_MMU(adreno_dev); + /* * If rb is current, we can use cpu path when GPU is * idle and we are switching to default pt. @@ -284,7 +285,7 @@ static bool _ctx_switch_use_cpu_path( if (adreno_dev->cur_rb == rb) return adreno_isidle(KGSL_DEVICE(adreno_dev)) && (new_pt == mmu->defaultpagetable); - else if ((rb->wptr == rb->rptr) && + else if (adreno_rb_empty(rb) && (new_pt == mmu->defaultpagetable)) return true; @@ -651,14 +652,14 @@ static unsigned int __add_curr_ctxt_cmds(struct adreno_ringbuffer *rb, *cmds++ = KGSL_CONTEXT_TO_MEM_IDENTIFIER; *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1); - cmds += cp_gpuaddr(adreno_dev, cmds, device->memstore.gpuaddr + - KGSL_MEMSTORE_RB_OFFSET(rb, current_context)); + cmds += cp_gpuaddr(adreno_dev, cmds, + MEMSTORE_RB_GPU_ADDR(device, rb, current_context)); *cmds++ = (drawctxt ? drawctxt->base.id : 0); *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1); - cmds += cp_gpuaddr(adreno_dev, cmds, device->memstore.gpuaddr + - KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, - current_context)); + cmds += cp_gpuaddr(adreno_dev, cmds, + MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, current_context)); *cmds++ = (drawctxt ? drawctxt->base.id : 0); /* Invalidate UCHE for new context */ @@ -706,7 +707,7 @@ static void _set_ctxt_cpu(struct adreno_ringbuffer *rb, } /* Update rb memstore with current context */ kgsl_sharedmem_writel(device, &device->memstore, - KGSL_MEMSTORE_RB_OFFSET(rb, current_context), + MEMSTORE_RB_OFFSET(rb, current_context), drawctxt ? drawctxt->base.id : 0); } diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index dceb8fb93461..8f0c31361854 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -279,8 +279,9 @@ int adreno_ringbuffer_start(struct adreno_device *adreno_dev, FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { kgsl_sharedmem_set(device, &(rb->buffer_desc), 0, 0xAA, KGSL_RB_SIZE); + kgsl_sharedmem_writel(device, &device->scratch, + SCRATCH_RPTR_OFFSET(rb->id), 0); rb->wptr = 0; - rb->rptr = 0; rb->wptr_preempt_end = 0xFFFFFFFF; rb->starve_timer_state = ADRENO_DISPATCHER_RB_STARVE_TIMER_UNINIT; @@ -446,7 +447,6 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, unsigned int total_sizedwords = sizedwords; unsigned int i; unsigned int context_id = 0; - uint64_t gpuaddr = device->memstore.gpuaddr; bool profile_ready; struct adreno_context *drawctxt = rb->drawctxt_active; struct kgsl_context *context = NULL; @@ -565,9 +565,7 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, if (adreno_is_preemption_enabled(adreno_dev) && gpudev->preemption_pre_ibsubmit) { - cond_addr = device->memstore.gpuaddr + - KGSL_MEMSTORE_OFFSET(context_id, - preempted); + cond_addr = MEMSTORE_ID_GPU_ADDR(device, context_id, preempted); ringcmds += gpudev->preemption_pre_ibsubmit( adreno_dev, rb, ringcmds, context, cond_addr, NULL); @@ -605,11 +603,10 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, *ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1); if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) ringcmds += cp_gpuaddr(adreno_dev, ringcmds, - gpuaddr + KGSL_MEMSTORE_OFFSET(context_id, - soptimestamp)); + MEMSTORE_ID_GPU_ADDR(device, context_id, soptimestamp)); else ringcmds += cp_gpuaddr(adreno_dev, ringcmds, - gpuaddr + KGSL_MEMSTORE_RB_OFFSET(rb, soptimestamp)); + MEMSTORE_ID_GPU_ADDR(device, context_id, soptimestamp)); *ringcmds++ = timestamp; if (secured_ctxt) @@ -660,9 +657,9 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, * off system collapse. */ *ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1); - ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr + - KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, - ref_wait_ts)); + ringcmds += cp_gpuaddr(adreno_dev, ringcmds, + MEMSTORE_ID_GPU_ADDR(device, KGSL_MEMSTORE_GLOBAL, + ref_wait_ts)); *ringcmds++ = ++_seq_cnt; /* @@ -677,16 +674,16 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, *ringcmds++ = CACHE_FLUSH_TS; if (drawctxt && !(flags & KGSL_CMD_FLAGS_INTERNAL_ISSUE)) { - ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr + - KGSL_MEMSTORE_OFFSET(context_id, eoptimestamp)); + ringcmds += cp_gpuaddr(adreno_dev, ringcmds, + MEMSTORE_ID_GPU_ADDR(device, context_id, eoptimestamp)); *ringcmds++ = timestamp; *ringcmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1); - ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr + - KGSL_MEMSTORE_RB_OFFSET(rb, eoptimestamp)); + ringcmds += cp_gpuaddr(adreno_dev, ringcmds, + MEMSTORE_RB_GPU_ADDR(device, rb, eoptimestamp)); *ringcmds++ = rb->timestamp; } else { - ringcmds += cp_gpuaddr(adreno_dev, ringcmds, gpuaddr + - KGSL_MEMSTORE_RB_OFFSET(rb, eoptimestamp)); + ringcmds += cp_gpuaddr(adreno_dev, ringcmds, + MEMSTORE_RB_GPU_ADDR(device, rb, eoptimestamp)); *ringcmds++ = timestamp; } diff --git a/drivers/gpu/msm/adreno_ringbuffer.h b/drivers/gpu/msm/adreno_ringbuffer.h index f1980fd92961..66cc7689119d 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.h +++ b/drivers/gpu/msm/adreno_ringbuffer.h @@ -78,7 +78,6 @@ struct adreno_ringbuffer_pagetable_info { * @flags: Internal control flags for the ringbuffer * @buffer_desc: Pointer to the ringbuffer memory descriptor * @wptr: Local copy of the wptr offset - * @rptr: Read pointer offset in dwords from baseaddr * @last_wptr: offset of the last H/W committed wptr * @rb_ctx: The context that represents a ringbuffer * @id: Priority level of the ringbuffer, also used as an ID @@ -106,7 +105,6 @@ struct adreno_ringbuffer { uint32_t flags; struct kgsl_memdesc buffer_desc; unsigned int wptr; - unsigned int rptr; unsigned int last_wptr; int id; unsigned int fault_detect_ts; @@ -127,9 +125,6 @@ struct adreno_ringbuffer { /* Returns the current ringbuffer */ #define ADRENO_CURRENT_RINGBUFFER(a) ((a)->cur_rb) -#define KGSL_MEMSTORE_RB_OFFSET(rb, field) \ - KGSL_MEMSTORE_OFFSET((rb->id + KGSL_MEMSTORE_MAX), field) - int cp_secure_mode(struct adreno_device *adreno_dev, uint *cmds, int set); int adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv, diff --git a/drivers/gpu/msm/adreno_snapshot.c b/drivers/gpu/msm/adreno_snapshot.c index ca61d36a1384..6bdf73164902 100644 --- a/drivers/gpu/msm/adreno_snapshot.c +++ b/drivers/gpu/msm/adreno_snapshot.c @@ -467,7 +467,7 @@ static size_t snapshot_rb(struct kgsl_device *device, u8 *buf, header->start = 0; header->end = KGSL_RB_DWORDS; header->wptr = rb->wptr; - header->rptr = rb->rptr; + header->rptr = adreno_get_rptr(rb); header->rbsize = KGSL_RB_DWORDS; header->count = KGSL_RB_DWORDS; adreno_rb_readtimestamp(adreno_dev, rb, KGSL_TIMESTAMP_QUEUED, diff --git a/drivers/gpu/msm/adreno_trace.h b/drivers/gpu/msm/adreno_trace.h index 5f1bbb9a83b3..0109a4c0a5e7 100644 --- a/drivers/gpu/msm/adreno_trace.h +++ b/drivers/gpu/msm/adreno_trace.h @@ -55,8 +55,8 @@ TRACE_EVENT(adreno_cmdbatch_queued, TRACE_EVENT(adreno_cmdbatch_submitted, TP_PROTO(struct kgsl_cmdbatch *cmdbatch, int inflight, uint64_t ticks, unsigned long secs, unsigned long usecs, - struct adreno_ringbuffer *rb), - TP_ARGS(cmdbatch, inflight, ticks, secs, usecs, rb), + struct adreno_ringbuffer *rb, unsigned int rptr), + TP_ARGS(cmdbatch, inflight, ticks, secs, usecs, rb, rptr), TP_STRUCT__entry( __field(unsigned int, id) __field(unsigned int, timestamp) @@ -81,7 +81,7 @@ TRACE_EVENT(adreno_cmdbatch_submitted, __entry->usecs = usecs; __entry->prio = cmdbatch->context->priority; __entry->rb_id = rb->id; - __entry->rptr = rb->rptr; + __entry->rptr = rptr; __entry->wptr = rb->wptr; __entry->q_inflight = rb->dispatch_q.inflight; ), @@ -100,8 +100,8 @@ TRACE_EVENT(adreno_cmdbatch_submitted, TRACE_EVENT(adreno_cmdbatch_retired, TP_PROTO(struct kgsl_cmdbatch *cmdbatch, int inflight, uint64_t start, uint64_t retire, - struct adreno_ringbuffer *rb), - TP_ARGS(cmdbatch, inflight, start, retire, rb), + struct adreno_ringbuffer *rb, unsigned int rptr), + TP_ARGS(cmdbatch, inflight, start, retire, rb, rptr), TP_STRUCT__entry( __field(unsigned int, id) __field(unsigned int, timestamp) @@ -126,7 +126,7 @@ TRACE_EVENT(adreno_cmdbatch_retired, __entry->retire = retire; __entry->prio = cmdbatch->context->priority; __entry->rb_id = rb->id; - __entry->rptr = rb->rptr; + __entry->rptr = rptr; __entry->wptr = rb->wptr; __entry->q_inflight = rb->dispatch_q.inflight; ), @@ -427,8 +427,9 @@ TRACE_EVENT(kgsl_a5xx_irq_status, DECLARE_EVENT_CLASS(adreno_hw_preempt_template, TP_PROTO(struct adreno_ringbuffer *cur_rb, - struct adreno_ringbuffer *new_rb), - TP_ARGS(cur_rb, new_rb), + struct adreno_ringbuffer *new_rb, + unsigned int cur_rptr, unsigned int new_rptr), + TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr), TP_STRUCT__entry(__field(int, cur_level) __field(int, new_level) __field(unsigned int, cur_rptr) @@ -440,8 +441,8 @@ DECLARE_EVENT_CLASS(adreno_hw_preempt_template, ), TP_fast_assign(__entry->cur_level = cur_rb->id; __entry->new_level = new_rb->id; - __entry->cur_rptr = cur_rb->rptr; - __entry->new_rptr = new_rb->rptr; + __entry->cur_rptr = cur_rptr; + __entry->new_rptr = new_rptr; __entry->cur_wptr = cur_rb->wptr; __entry->new_wptr = new_rb->wptr; __entry->cur_rbbase = cur_rb->buffer_desc.gpuaddr; @@ -458,26 +459,30 @@ DECLARE_EVENT_CLASS(adreno_hw_preempt_template, DEFINE_EVENT(adreno_hw_preempt_template, adreno_hw_preempt_clear_to_trig, TP_PROTO(struct adreno_ringbuffer *cur_rb, - struct adreno_ringbuffer *new_rb), - TP_ARGS(cur_rb, new_rb) + struct adreno_ringbuffer *new_rb, + unsigned int cur_rptr, unsigned int new_rptr), + TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr) ); DEFINE_EVENT(adreno_hw_preempt_template, adreno_hw_preempt_trig_to_comp, TP_PROTO(struct adreno_ringbuffer *cur_rb, - struct adreno_ringbuffer *new_rb), - TP_ARGS(cur_rb, new_rb) + struct adreno_ringbuffer *new_rb, + unsigned int cur_rptr, unsigned int new_rptr), + TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr) ); DEFINE_EVENT(adreno_hw_preempt_template, adreno_hw_preempt_trig_to_comp_int, TP_PROTO(struct adreno_ringbuffer *cur_rb, - struct adreno_ringbuffer *new_rb), - TP_ARGS(cur_rb, new_rb) + struct adreno_ringbuffer *new_rb, + unsigned int cur_rptr, unsigned int new_rptr), + TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr) ); TRACE_EVENT(adreno_hw_preempt_comp_to_clear, TP_PROTO(struct adreno_ringbuffer *cur_rb, - struct adreno_ringbuffer *new_rb), - TP_ARGS(cur_rb, new_rb), + struct adreno_ringbuffer *new_rb, + unsigned int cur_rptr, unsigned int new_rptr), + TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr), TP_STRUCT__entry(__field(int, cur_level) __field(int, new_level) __field(unsigned int, cur_rptr) @@ -490,8 +495,8 @@ TRACE_EVENT(adreno_hw_preempt_comp_to_clear, ), TP_fast_assign(__entry->cur_level = cur_rb->id; __entry->new_level = new_rb->id; - __entry->cur_rptr = cur_rb->rptr; - __entry->new_rptr = new_rb->rptr; + __entry->cur_rptr = cur_rptr; + __entry->new_rptr = new_rptr; __entry->cur_wptr = cur_rb->wptr; __entry->new_wptr_end = new_rb->wptr_preempt_end; __entry->new_wptr = new_rb->wptr; @@ -509,8 +514,9 @@ TRACE_EVENT(adreno_hw_preempt_comp_to_clear, TRACE_EVENT(adreno_hw_preempt_token_submit, TP_PROTO(struct adreno_ringbuffer *cur_rb, - struct adreno_ringbuffer *new_rb), - TP_ARGS(cur_rb, new_rb), + struct adreno_ringbuffer *new_rb, + unsigned int cur_rptr, unsigned int new_rptr), + TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr), TP_STRUCT__entry(__field(int, cur_level) __field(int, new_level) __field(unsigned int, cur_rptr) @@ -523,8 +529,8 @@ TRACE_EVENT(adreno_hw_preempt_token_submit, ), TP_fast_assign(__entry->cur_level = cur_rb->id; __entry->new_level = new_rb->id; - __entry->cur_rptr = cur_rb->rptr; - __entry->new_rptr = new_rb->rptr; + __entry->cur_rptr = cur_rptr; + __entry->new_rptr = new_rptr; __entry->cur_wptr = cur_rb->wptr; __entry->cur_wptr_end = cur_rb->wptr_preempt_end; __entry->new_wptr = new_rb->wptr; @@ -541,23 +547,6 @@ TRACE_EVENT(adreno_hw_preempt_token_submit, ) ); -TRACE_EVENT(adreno_rb_starve, - TP_PROTO(struct adreno_ringbuffer *rb), - TP_ARGS(rb), - TP_STRUCT__entry(__field(int, id) - __field(unsigned int, rptr) - __field(unsigned int, wptr) - ), - TP_fast_assign(__entry->id = rb->id; - __entry->rptr = rb->rptr; - __entry->wptr = rb->wptr; - ), - TP_printk( - "rb %d r/w %x/%x starved", __entry->id, __entry->rptr, - __entry->wptr - ) -); - #endif /* _ADRENO_TRACE_H */ /* This part must be outside protection */ diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index 2563591f376e..7902480972f8 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -1150,6 +1150,8 @@ static int kgsl_open_device(struct kgsl_device *device) atomic_inc(&device->active_cnt); kgsl_sharedmem_set(device, &device->memstore, 0, 0, device->memstore.size); + kgsl_sharedmem_set(device, &device->scratch, 0, 0, + device->scratch.size); result = device->ftbl->init(device); if (result) @@ -3910,11 +3912,13 @@ int kgsl_device_platform_probe(struct kgsl_device *device) status = kgsl_allocate_global(device, &device->memstore, KGSL_MEMSTORE_SIZE, 0, 0); - if (status != 0) { - KGSL_DRV_ERR(device, "kgsl_allocate_global failed %d\n", - status); + if (status != 0) goto error_close_mmu; - } + + status = kgsl_allocate_global(device, &device->scratch, + PAGE_SIZE, 0, 0); + if (status != 0) + goto error_free_memstore; /* * The default request type PM_QOS_REQ_ALL_CORES is @@ -3964,6 +3968,8 @@ int kgsl_device_platform_probe(struct kgsl_device *device) return 0; +error_free_memstore: + kgsl_free_global(device, &device->memstore); error_close_mmu: kgsl_mmu_close(device); error_pwrctrl_close: @@ -3990,6 +3996,8 @@ void kgsl_device_platform_remove(struct kgsl_device *device) idr_destroy(&device->context_idr); + kgsl_free_global(device, &device->scratch); + kgsl_free_global(device, &device->memstore); kgsl_mmu_close(device); diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h index dfe83be799b3..ad1a8f1decdd 100644 --- a/drivers/gpu/msm/kgsl.h +++ b/drivers/gpu/msm/kgsl.h @@ -37,6 +37,32 @@ #define KGSL_MEMSTORE_MAX (KGSL_MEMSTORE_SIZE / \ sizeof(struct kgsl_devmemstore) - 1 - KGSL_PRIORITY_MAX_RB_LEVELS) +#define MEMSTORE_RB_OFFSET(rb, field) \ + KGSL_MEMSTORE_OFFSET(((rb)->id + KGSL_MEMSTORE_MAX), field) + +#define MEMSTORE_ID_GPU_ADDR(dev, iter, field) \ + ((dev)->memstore.gpuaddr + KGSL_MEMSTORE_OFFSET(iter, field)) + +#define MEMSTORE_RB_GPU_ADDR(dev, rb, field) \ + ((dev)->memstore.gpuaddr + \ + KGSL_MEMSTORE_OFFSET(((rb)->id + KGSL_MEMSTORE_MAX), field)) + +/* + * SCRATCH MEMORY: The scratch memory is one page worth of data that + * is mapped into the GPU. This allows for some 'shared' data between + * the GPU and CPU. For example, it will be used by the GPU to write + * each updated RPTR for each RB. + * + * Used Data: + * Offset: Length(bytes): What + * 0x0: 4 * KGSL_PRIORITY_MAX_RB_LEVELS: RB0 RPTR + */ + +/* Shadow global helpers */ +#define SCRATCH_RPTR_OFFSET(id) ((id) * sizeof(unsigned int)) +#define SCRATCH_RPTR_GPU_ADDR(dev, id) \ + ((dev)->scratch.gpuaddr + SCRATCH_RPTR_OFFSET(id)) + /* Timestamp window used to detect rollovers (half of integer range) */ #define KGSL_TIMESTAMP_WINDOW 0x80000000 diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h index c3fb2b81fcbd..4159a5fe375f 100644 --- a/drivers/gpu/msm/kgsl_device.h +++ b/drivers/gpu/msm/kgsl_device.h @@ -227,6 +227,7 @@ struct kgsl_device { /* GPU shader memory size */ unsigned int shader_mem_len; struct kgsl_memdesc memstore; + struct kgsl_memdesc scratch; const char *iomemname; const char *shadermemname; |