drm/msm: Use mmu notifiers to track SVM range invalidations

SVM buffer objects share the same virtual address on
both the CPU and GPU. Register for notifications when
SVM address ranges are unmapped on the CPU. When such
a notification is received, unmap the corresponding
SVM objects from the SMMU, after waiting on the most
recent fence that uses them. The notifier struct is
reference counted starting with the creation of the
first SVM bo in the process and is released when the
last SVM bo is freed.

Change-Id: I01f590d21fd1d146f5324539e5041f03653f858a
Signed-off-by: Sushmita Susheelendra <ssusheel@codeaurora.org>

7 files changed, 269 insertions, 6 deletions

diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 5838545468f8..dbc198b00792 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -12,6 +12,8 @@ config DRM_MSM
 	select QCOM_SCM
 	select BACKLIGHT_CLASS_DEVICE
 	select MSM_EXT_DISPLAY
+	select MMU_NOTIFIER
+	select INTERVAL_TREE
 	default y
 	help
 	  DRM/KMS driver for MSM/snapdragon.
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 92eafa083855..74f298d7328d 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -391,6 +391,8 @@ static int msm_load(struct drm_device *dev, unsigned long flags)
 	INIT_LIST_HEAD(&priv->vblank_ctrl.event_list);
 	init_kthread_work(&priv->vblank_ctrl.work, vblank_ctrl_worker);
 	spin_lock_init(&priv->vblank_ctrl.lock);
+	hash_init(priv->mn_hash);
+	mutex_init(&priv->mn_lock);
 
 	drm_mode_config_init(dev);
 
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index ade0d91d9aee..4083e990b2c8 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -34,6 +34,7 @@
 #include <linux/of_graph.h>
 #include <linux/of_device.h>
 #include <linux/sde_io_util.h>
+#include <linux/hashtable.h>
 #include <asm/sizes.h>
 #include <linux/kthread.h>
 
@@ -325,6 +326,11 @@ struct msm_drm_private {
 	unsigned int num_connectors;
 	struct drm_connector *connectors[MAX_CONNECTORS];
 
+	/* hash to store mm_struct to msm_mmu_notifier mappings */
+	DECLARE_HASHTABLE(mn_hash, 7);
+	/* protects mn_hash and the msm_mmu_notifier for the process */
+	struct mutex mn_lock;
+
 	/* Properties */
 	struct drm_property *plane_property[PLANE_PROP_COUNT];
 	struct drm_property *crtc_property[CRTC_PROP_COUNT];
@@ -404,7 +410,7 @@ void msm_update_fence(struct drm_device *dev, uint32_t fence);
 
 void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
 		struct msm_gem_vma *vma, struct sg_table *sgt,
-		void *priv);
+		void *priv, bool invalidated);
 int msm_gem_map_vma(struct msm_gem_address_space *aspace,
 		struct msm_gem_vma *vma, struct sg_table *sgt,
 		void *priv, unsigned int flags);
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 6cff8ef77d0d..ddf808e87828 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -25,6 +25,129 @@
 #include "msm_gpu.h"
 #include "msm_mmu.h"
 
+static void msm_gem_mn_free(struct kref *refcount)
+{
+	struct msm_mmu_notifier *msm_mn = container_of(refcount,
+			struct msm_mmu_notifier, refcount);
+
+	mmu_notifier_unregister(&msm_mn->mn, msm_mn->mm);
+	hash_del(&msm_mn->node);
+
+	kfree(msm_mn);
+}
+
+static int msm_gem_mn_get(struct msm_mmu_notifier *msm_mn)
+{
+	if (msm_mn)
+		return kref_get_unless_zero(&msm_mn->refcount);
+	return 0;
+}
+
+static void msm_gem_mn_put(struct msm_mmu_notifier *msm_mn)
+{
+	if (msm_mn) {
+		struct msm_drm_private *msm_dev = msm_mn->msm_dev;
+
+		mutex_lock(&msm_dev->mn_lock);
+		kref_put(&msm_mn->refcount, msm_gem_mn_free);
+		mutex_unlock(&msm_dev->mn_lock);
+	}
+}
+
+void msm_mn_invalidate_range_start(struct mmu_notifier *mn,
+		struct mm_struct *mm, unsigned long start, unsigned long end);
+
+static const struct mmu_notifier_ops msm_mn_ops = {
+	.invalidate_range_start = msm_mn_invalidate_range_start,
+};
+
+static struct msm_mmu_notifier *
+msm_gem_mn_find(struct msm_drm_private *msm_dev, struct mm_struct *mm,
+		struct msm_gem_address_space *aspace)
+{
+	struct msm_mmu_notifier *msm_mn;
+	int ret = 0;
+
+	mutex_lock(&msm_dev->mn_lock);
+	hash_for_each_possible(msm_dev->mn_hash, msm_mn, node,
+			(unsigned long) mm) {
+		if (msm_mn->mm == mm) {
+			if (!msm_gem_mn_get(msm_mn)) {
+				ret = -EINVAL;
+				goto fail;
+			}
+			mutex_unlock(&msm_dev->mn_lock);
+			return msm_mn;
+		}
+	}
+
+	msm_mn = kzalloc(sizeof(*msm_mn), GFP_KERNEL);
+	if (!msm_mn) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	msm_mn->mm = current->mm;
+	msm_mn->mn.ops = &msm_mn_ops;
+	ret = mmu_notifier_register(&msm_mn->mn, msm_mn->mm);
+	if (ret) {
+		kfree(msm_mn);
+		goto fail;
+	}
+
+	msm_mn->svm_tree = RB_ROOT;
+	spin_lock_init(&msm_mn->svm_tree_lock);
+	kref_init(&msm_mn->refcount);
+	msm_mn->msm_dev = msm_dev;
+
+	/* Insert the msm_mn into the hash */
+	hash_add(msm_dev->mn_hash, &msm_mn->node, (unsigned long) msm_mn->mm);
+	mutex_unlock(&msm_dev->mn_lock);
+
+	return msm_mn;
+
+fail:
+	mutex_unlock(&msm_dev->mn_lock);
+	return ERR_PTR(ret);
+}
+
+static int msm_gem_mn_register(struct msm_gem_svm_object *msm_svm_obj,
+		struct msm_gem_address_space *aspace)
+{
+	struct drm_gem_object *obj = &msm_svm_obj->msm_obj_base.base;
+	struct msm_drm_private *msm_dev = obj->dev->dev_private;
+	struct msm_mmu_notifier *msm_mn;
+
+	msm_svm_obj->mm = current->mm;
+	msm_svm_obj->svm_node.start = msm_svm_obj->hostptr;
+	msm_svm_obj->svm_node.last = msm_svm_obj->hostptr + obj->size - 1;
+
+	msm_mn = msm_gem_mn_find(msm_dev, msm_svm_obj->mm, aspace);
+	if (IS_ERR(msm_mn))
+		return PTR_ERR(msm_mn);
+
+	msm_svm_obj->msm_mn = msm_mn;
+
+	spin_lock(&msm_mn->svm_tree_lock);
+	interval_tree_insert(&msm_svm_obj->svm_node, &msm_mn->svm_tree);
+	spin_unlock(&msm_mn->svm_tree_lock);
+
+	return 0;
+}
+
+static void msm_gem_mn_unregister(struct msm_gem_svm_object *msm_svm_obj)
+{
+	struct msm_mmu_notifier *msm_mn = msm_svm_obj->msm_mn;
+
+	/* invalid: bo already unregistered */
+	if (!msm_mn || msm_svm_obj->invalid)
+		return;
+
+	spin_lock(&msm_mn->svm_tree_lock);
+	interval_tree_remove(&msm_svm_obj->svm_node, &msm_mn->svm_tree);
+	spin_unlock(&msm_mn->svm_tree_lock);
+}
+
 static int protect_pages(struct msm_gem_object *msm_obj)
 {
 	int perm = PERM_READ | PERM_WRITE;
@@ -354,14 +477,21 @@ static void
 put_iova(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+	struct msm_gem_svm_object *msm_svm_obj;
 	struct msm_gem_vma *domain, *tmp;
+	bool invalid = false;
 
 	WARN_ON(!mutex_is_locked(&msm_obj->lock));
 
+	if (msm_obj->flags & MSM_BO_SVM) {
+		msm_svm_obj = to_msm_svm_obj(msm_obj);
+		invalid = msm_svm_obj->invalid;
+	}
+
 	list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) {
 		if (iommu_present(&platform_bus_type)) {
 			msm_gem_unmap_vma(domain->aspace, domain,
-				msm_obj->sgt, get_dmabuf_ptr(obj));
+				msm_obj->sgt, get_dmabuf_ptr(obj), invalid);
 		}
 
 		obj_remove_domain(domain);
@@ -676,8 +806,14 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 
 	list_del(&msm_obj->mm_list);
 
-	mutex_lock(&msm_obj->lock);
+	/* Unregister SVM object from mmu notifications */
+	if (msm_obj->flags & MSM_BO_SVM) {
+		msm_gem_mn_unregister(msm_svm_obj);
+		msm_gem_mn_put(msm_svm_obj->msm_mn);
+		msm_svm_obj->msm_mn = NULL;
+	}
 
+	mutex_lock(&msm_obj->lock);
 	put_iova(obj);
 
 	if (obj->import_attach) {
@@ -947,6 +1083,11 @@ struct drm_gem_object *msm_gem_svm_new(struct drm_device *dev,
 
 	msm_svm_obj = to_msm_svm_obj(msm_obj);
 	msm_svm_obj->hostptr = hostptr;
+	msm_svm_obj->invalid = false;
+
+	ret = msm_gem_mn_register(msm_svm_obj, aspace);
+	if (ret)
+		goto fail;
 
 	/*
 	 * Get physical pages and map into smmu in the ioctl itself.
@@ -1038,7 +1179,8 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 	/* OR the passed in flags */
 	msm_obj->flags |= flags;
 
-	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages,
+			NULL, npages);
 	if (ret) {
 		mutex_unlock(&msm_obj->lock);
 		goto fail;
@@ -1053,3 +1195,82 @@ fail:
 
 	return ERR_PTR(ret);
 }
+
+/* Timeout in ms, long enough so we are sure the GPU is hung */
+#define SVM_OBJ_WAIT_TIMEOUT 10000
+static void invalidate_svm_object(struct msm_gem_svm_object *msm_svm_obj)
+{
+	struct msm_gem_object *msm_obj = &msm_svm_obj->msm_obj_base;
+	struct drm_device *dev = msm_obj->base.dev;
+	struct msm_gem_vma *domain, *tmp;
+	uint32_t fence;
+	int ret;
+
+	if (is_active(msm_obj)) {
+		ktime_t timeout = ktime_add_ms(ktime_get(),
+				SVM_OBJ_WAIT_TIMEOUT);
+
+		/* Get the most recent fence that touches the object */
+		fence = msm_gem_fence(msm_obj, MSM_PREP_READ | MSM_PREP_WRITE);
+
+		/* Wait for the fence to retire */
+		ret = msm_wait_fence(dev, fence, &timeout, true);
+		if (ret)
+			/* The GPU could be hung! Not much we can do */
+			dev_err(dev->dev, "drm: Error (%d) waiting for svm object: 0x%llx",
+					ret, msm_svm_obj->hostptr);
+	}
+
+	/* GPU is done, unmap object from SMMU */
+	mutex_lock(&msm_obj->lock);
+	list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) {
+		struct msm_gem_address_space *aspace = domain->aspace;
+
+		if (domain->iova)
+			aspace->mmu->funcs->unmap(aspace->mmu,
+					domain->iova, msm_obj->sgt,
+					get_dmabuf_ptr(&msm_obj->base));
+	}
+	/* Let go of the physical pages */
+	put_pages(&msm_obj->base);
+	mutex_unlock(&msm_obj->lock);
+}
+
+void msm_mn_invalidate_range_start(struct mmu_notifier *mn,
+		struct mm_struct *mm, unsigned long start, unsigned long end)
+{
+	struct msm_mmu_notifier *msm_mn =
+		container_of(mn, struct msm_mmu_notifier, mn);
+	struct interval_tree_node *itn = NULL;
+	struct msm_gem_svm_object *msm_svm_obj;
+	struct drm_gem_object *obj;
+	LIST_HEAD(inv_list);
+
+	if (!msm_gem_mn_get(msm_mn))
+		return;
+
+	spin_lock(&msm_mn->svm_tree_lock);
+	itn = interval_tree_iter_first(&msm_mn->svm_tree, start, end - 1);
+	while (itn) {
+		msm_svm_obj = container_of(itn,
+				struct msm_gem_svm_object, svm_node);
+		obj = &msm_svm_obj->msm_obj_base.base;
+
+		if (kref_get_unless_zero(&obj->refcount))
+			list_add(&msm_svm_obj->lnode, &inv_list);
+
+		itn = interval_tree_iter_next(itn, start, end - 1);
+	}
+	spin_unlock(&msm_mn->svm_tree_lock);
+
+	list_for_each_entry(msm_svm_obj, &inv_list, lnode) {
+		obj = &msm_svm_obj->msm_obj_base.base;
+		/* Unregister SVM object from mmu notifications */
+		msm_gem_mn_unregister(msm_svm_obj);
+		msm_svm_obj->invalid = true;
+		invalidate_svm_object(msm_svm_obj);
+		drm_gem_object_unreference_unlocked(obj);
+	}
+
+	msm_gem_mn_put(msm_mn);
+}
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index f3060915ac20..04e6c658b5f3 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -20,6 +20,8 @@
 
 #include <linux/kref.h>
 #include <linux/reservation.h>
+#include <linux/mmu_notifier.h>
+#include <linux/interval_tree.h>
 #include "msm_drv.h"
 
 /* Additional internal-use only BO flags: */
@@ -86,9 +88,25 @@ struct msm_gem_object {
 };
 #define to_msm_bo(x) container_of(x, struct msm_gem_object, base)
 
+struct msm_mmu_notifier {
+	struct mmu_notifier mn;
+	struct mm_struct *mm; /* mm_struct owning the mmu notifier mn */
+	struct hlist_node node;
+	struct rb_root svm_tree; /* interval tree holding all svm bos */
+	spinlock_t svm_tree_lock; /* Protects svm_tree*/
+	struct msm_drm_private *msm_dev;
+	struct kref refcount;
+};
+
 struct msm_gem_svm_object {
 	struct msm_gem_object msm_obj_base;
 	uint64_t hostptr;
+	struct mm_struct *mm; /* mm_struct the svm bo belongs to */
+	struct interval_tree_node svm_node;
+	struct msm_mmu_notifier *msm_mn;
+	struct list_head lnode;
+	/* bo has been unmapped on CPU, cannot be part of GPU submits */
+	bool invalid;
 };
 
 #define to_msm_svm_obj(x) \
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 612e8e77b782..c22c89ae1fcf 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -213,6 +213,18 @@ retry:
 			submit->bos[i].flags |= BO_LOCKED;
 		}
 
+		/*
+		 * An invalid SVM object is part of
+		 * this submit's buffer list, fail.
+		 */
+		if (msm_obj->flags & MSM_BO_SVM) {
+			struct msm_gem_svm_object *msm_svm_obj =
+				to_msm_svm_obj(msm_obj);
+			if (msm_svm_obj->invalid) {
+				ret = -EINVAL;
+				goto fail;
+			}
+		}
 
 		/* if locking succeeded, pin bo: */
 		ret = msm_gem_get_iova(&msm_obj->base, aspace, &iova);
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index 3bc17e642f7b..95be430ea5d8 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -166,12 +166,14 @@ int msm_gem_map_vma(struct msm_gem_address_space *aspace,
 }
 
 void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, struct sg_table *sgt, void *priv)
+		struct msm_gem_vma *vma, struct sg_table *sgt,
+		void *priv, bool invalidated)
 {
 	if (!aspace || !vma->iova)
 		return;
 
-	aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, sgt, priv);
+	if (!invalidated)
+		aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, sgt, priv);
 
 	msm_gem_release_iova(aspace, vma);