Merge "drm/msm: Use mmu notifiers to track SVM range invalidations"

8 files changed, 688 insertions, 81 deletions

diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index 04de6fd88f8c..6b81035b51a1 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -46,6 +46,7 @@
 #include <linux/slab.h>
 #include <linux/seq_file.h>
 #include <linux/export.h>
+#include <linux/interval_tree_generic.h>
 
 /**
  * DOC: Overview
@@ -103,6 +104,72 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
 						u64 end,
 						enum drm_mm_search_flags flags);
 
+#define START(node) ((node)->start)
+#define LAST(node)  ((node)->start + (node)->size - 1)
+
+INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
+		     u64, __subtree_last,
+		     START, LAST, static inline, drm_mm_interval_tree)
+
+struct drm_mm_node *
+drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last)
+{
+	return drm_mm_interval_tree_iter_first(&mm->interval_tree,
+					       start, last);
+}
+EXPORT_SYMBOL(drm_mm_interval_first);
+
+struct drm_mm_node *
+drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last)
+{
+	return drm_mm_interval_tree_iter_next(node, start, last);
+}
+EXPORT_SYMBOL(drm_mm_interval_next);
+
+static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
+					  struct drm_mm_node *node)
+{
+	struct drm_mm *mm = hole_node->mm;
+	struct rb_node **link, *rb;
+	struct drm_mm_node *parent;
+
+	node->__subtree_last = LAST(node);
+
+	if (hole_node->allocated) {
+		rb = &hole_node->rb;
+		while (rb) {
+			parent = rb_entry(rb, struct drm_mm_node, rb);
+			if (parent->__subtree_last >= node->__subtree_last)
+				break;
+
+			parent->__subtree_last = node->__subtree_last;
+			rb = rb_parent(rb);
+		}
+
+		rb = &hole_node->rb;
+		link = &hole_node->rb.rb_right;
+	} else {
+		rb = NULL;
+		link = &mm->interval_tree.rb_node;
+	}
+
+	while (*link) {
+		rb = *link;
+		parent = rb_entry(rb, struct drm_mm_node, rb);
+		if (parent->__subtree_last < node->__subtree_last)
+			parent->__subtree_last = node->__subtree_last;
+		if (node->start < parent->start)
+			link = &parent->rb.rb_left;
+		else
+			link = &parent->rb.rb_right;
+	}
+
+	rb_link_node(&node->rb, rb, link);
+	rb_insert_augmented(&node->rb,
+			    &mm->interval_tree,
+			    &drm_mm_interval_tree_augment);
+}
+
 static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
 				 struct drm_mm_node *node,
 				 u64 size, unsigned alignment,
@@ -150,9 +217,10 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
 	node->color = color;
 	node->allocated = 1;
 
-	INIT_LIST_HEAD(&node->hole_stack);
 	list_add(&node->node_list, &hole_node->node_list);
 
+	drm_mm_interval_tree_add_node(hole_node, node);
+
 	BUG_ON(node->start + node->size > adj_end);
 
 	node->hole_follows = 0;
@@ -178,39 +246,52 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
  */
 int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 {
-	struct drm_mm_node *hole;
 	u64 end = node->start + node->size;
-	u64 hole_start;
-	u64 hole_end;
+	struct drm_mm_node *hole;
+	u64 hole_start, hole_end;
 
-	BUG_ON(node == NULL);
+	if (WARN_ON(node->size == 0))
+		return -EINVAL;
 
 	/* Find the relevant hole to add our node to */
-	drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
-		if (hole_start > node->start || hole_end < end)
-			continue;
+	hole = drm_mm_interval_tree_iter_first(&mm->interval_tree,
+					       node->start, ~(u64)0);
+	if (hole) {
+		if (hole->start < end)
+			return -ENOSPC;
+	} else {
+		hole = list_entry(&mm->head_node.node_list,
+				  typeof(*hole), node_list);
+	}
 
-		node->mm = mm;
-		node->allocated = 1;
+	hole = list_last_entry(&hole->node_list, typeof(*hole), node_list);
+	if (!hole->hole_follows)
+		return -ENOSPC;
 
-		INIT_LIST_HEAD(&node->hole_stack);
-		list_add(&node->node_list, &hole->node_list);
+	hole_start = __drm_mm_hole_node_start(hole);
+	hole_end = __drm_mm_hole_node_end(hole);
+	if (hole_start > node->start || hole_end < end)
+		return -ENOSPC;
 
-		if (node->start == hole_start) {
-			hole->hole_follows = 0;
-			list_del_init(&hole->hole_stack);
-		}
+	node->mm = mm;
+	node->allocated = 1;
 
-		node->hole_follows = 0;
-		if (end != hole_end) {
-			list_add(&node->hole_stack, &mm->hole_stack);
-			node->hole_follows = 1;
-		}
+	list_add(&node->node_list, &hole->node_list);
 
-		return 0;
+	drm_mm_interval_tree_add_node(hole, node);
+
+	if (node->start == hole_start) {
+		hole->hole_follows = 0;
+		list_del(&hole->hole_stack);
+	}
+
+	node->hole_follows = 0;
+	if (end != hole_end) {
+		list_add(&node->hole_stack, &mm->hole_stack);
+		node->hole_follows = 1;
 	}
 
-	return -ENOSPC;
+	return 0;
 }
 EXPORT_SYMBOL(drm_mm_reserve_node);
 
@@ -237,6 +318,9 @@ int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
 {
 	struct drm_mm_node *hole_node;
 
+	if (WARN_ON(size == 0))
+		return -EINVAL;
+
 	hole_node = drm_mm_search_free_generic(mm, size, alignment,
 					       color, sflags);
 	if (!hole_node)
@@ -297,9 +381,10 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
 	node->color = color;
 	node->allocated = 1;
 
-	INIT_LIST_HEAD(&node->hole_stack);
 	list_add(&node->node_list, &hole_node->node_list);
 
+	drm_mm_interval_tree_add_node(hole_node, node);
+
 	BUG_ON(node->start < start);
 	BUG_ON(node->start < adj_start);
 	BUG_ON(node->start + node->size > adj_end);
@@ -338,6 +423,9 @@ int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, struct drm_mm_node *n
 {
 	struct drm_mm_node *hole_node;
 
+	if (WARN_ON(size == 0))
+		return -EINVAL;
+
 	hole_node = drm_mm_search_free_in_range_generic(mm,
 							size, alignment, color,
 							start, end, sflags);
@@ -388,6 +476,7 @@ void drm_mm_remove_node(struct drm_mm_node *node)
 	} else
 		list_move(&prev_node->hole_stack, &mm->hole_stack);
 
+	drm_mm_interval_tree_remove(node, &mm->interval_tree);
 	list_del(&node->node_list);
 	node->allocated = 0;
 }
@@ -514,11 +603,13 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
 {
 	list_replace(&old->node_list, &new->node_list);
 	list_replace(&old->hole_stack, &new->hole_stack);
+	rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree);
 	new->hole_follows = old->hole_follows;
 	new->mm = old->mm;
 	new->start = old->start;
 	new->size = old->size;
 	new->color = old->color;
+	new->__subtree_last = old->__subtree_last;
 
 	old->allocated = 0;
 	new->allocated = 1;
@@ -746,7 +837,6 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size)
 
 	/* Clever trick to avoid a special case in the free hole tracking. */
 	INIT_LIST_HEAD(&mm->head_node.node_list);
-	INIT_LIST_HEAD(&mm->head_node.hole_stack);
 	mm->head_node.hole_follows = 1;
 	mm->head_node.scanned_block = 0;
 	mm->head_node.scanned_prev_free = 0;
@@ -756,6 +846,8 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size)
 	mm->head_node.size = start - mm->head_node.start;
 	list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
 
+	mm->interval_tree = RB_ROOT;
+
 	mm->color_adjust = NULL;
 }
 EXPORT_SYMBOL(drm_mm_init);
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 5838545468f8..dbc198b00792 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -12,6 +12,8 @@ config DRM_MSM
 	select QCOM_SCM
 	select BACKLIGHT_CLASS_DEVICE
 	select MSM_EXT_DISPLAY
+	select MMU_NOTIFIER
+	select INTERVAL_TREE
 	default y
 	help
 	  DRM/KMS driver for MSM/snapdragon.
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index b9503564cdd6..74f298d7328d 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -391,6 +391,8 @@ static int msm_load(struct drm_device *dev, unsigned long flags)
 	INIT_LIST_HEAD(&priv->vblank_ctrl.event_list);
 	init_kthread_work(&priv->vblank_ctrl.work, vblank_ctrl_worker);
 	spin_lock_init(&priv->vblank_ctrl.lock);
+	hash_init(priv->mn_hash);
+	mutex_init(&priv->mn_lock);
 
 	drm_mode_config_init(dev);
 
@@ -559,7 +561,8 @@ static struct msm_file_private *setup_pagetable(struct msm_drm_private *priv)
 		return ERR_PTR(-ENOMEM);
 
 	ctx->aspace = msm_gem_address_space_create_instance(
-		priv->gpu->aspace->mmu, "gpu", 0x100000000, 0x1ffffffff);
+		priv->gpu->aspace->mmu, "gpu", 0x100000000ULL,
+		TASK_SIZE_64 - 1);
 
 	if (IS_ERR(ctx->aspace)) {
 		int ret = PTR_ERR(ctx->aspace);
@@ -1141,6 +1144,20 @@ static int msm_ioctl_gem_new(struct drm_device *dev, void *data,
 			args->flags, &args->handle);
 }
 
+static int msm_ioctl_gem_svm_new(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct drm_msm_gem_svm_new *args = data;
+
+	if (args->flags & ~MSM_BO_FLAGS) {
+		DRM_ERROR("invalid flags: %08x\n", args->flags);
+		return -EINVAL;
+	}
+
+	return msm_gem_svm_new_handle(dev, file, args->hostptr, args->size,
+			args->flags, &args->handle);
+}
+
 static inline ktime_t to_ktime(struct drm_msm_timespec timeout)
 {
 	return ktime_set(timeout.tv_sec, timeout.tv_nsec);
@@ -1193,6 +1210,7 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
 {
 	struct drm_msm_gem_info *args = data;
 	struct drm_gem_object *obj;
+	struct msm_gem_object *msm_obj;
 	struct msm_file_private *ctx = file->driver_priv;
 	int ret = 0;
 
@@ -1203,10 +1221,10 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
 	if (!obj)
 		return -ENOENT;
 
+	msm_obj = to_msm_bo(obj);
 	if (args->flags & MSM_INFO_IOVA) {
 		struct msm_gem_address_space *aspace = NULL;
 		struct msm_drm_private *priv = dev->dev_private;
-		struct msm_gem_object *msm_obj = to_msm_bo(obj);
 		uint64_t iova;
 
 		if (msm_obj->flags & MSM_BO_SECURE && priv->gpu)
@@ -1223,6 +1241,14 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
 		if (!ret)
 			args->offset = iova;
 	} else {
+		if (msm_obj->flags & MSM_BO_SVM) {
+			/*
+			 * Offset for an SVM object is not needed as they are
+			 * already mmap'ed before the SVM ioctl is invoked.
+			 */
+			ret = -EACCES;
+			goto out;
+		}
 		args->offset = msm_gem_mmap_offset(obj);
 	}
 
@@ -1700,6 +1726,8 @@ static const struct drm_ioctl_desc msm_ioctls[] = {
 			  DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(MSM_GEM_SYNC, msm_ioctl_gem_sync,
 			  DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(MSM_GEM_SVM_NEW, msm_ioctl_gem_svm_new,
+			  DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
 static const struct vm_operations_struct vm_ops = {
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 54a3568ca11f..4083e990b2c8 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -34,6 +34,7 @@
 #include <linux/of_graph.h>
 #include <linux/of_device.h>
 #include <linux/sde_io_util.h>
+#include <linux/hashtable.h>
 #include <asm/sizes.h>
 #include <linux/kthread.h>
 
@@ -325,6 +326,11 @@ struct msm_drm_private {
 	unsigned int num_connectors;
 	struct drm_connector *connectors[MAX_CONNECTORS];
 
+	/* hash to store mm_struct to msm_mmu_notifier mappings */
+	DECLARE_HASHTABLE(mn_hash, 7);
+	/* protects mn_hash and the msm_mmu_notifier for the process */
+	struct mutex mn_lock;
+
 	/* Properties */
 	struct drm_property *plane_property[PLANE_PROP_COUNT];
 	struct drm_property *crtc_property[CRTC_PROP_COUNT];
@@ -404,10 +410,15 @@ void msm_update_fence(struct drm_device *dev, uint32_t fence);
 
 void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
 		struct msm_gem_vma *vma, struct sg_table *sgt,
-		void *priv);
+		void *priv, bool invalidated);
 int msm_gem_map_vma(struct msm_gem_address_space *aspace,
 		struct msm_gem_vma *vma, struct sg_table *sgt,
 		void *priv, unsigned int flags);
+int msm_gem_reserve_iova(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *domain,
+		uint64_t hostptr, uint64_t size);
+void msm_gem_release_iova(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma);
 
 void msm_gem_address_space_put(struct msm_gem_address_space *aspace);
 
@@ -471,6 +482,12 @@ struct drm_gem_object *msm_gem_new_locked(struct drm_device *dev,
 struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 		uint32_t size, struct sg_table *sgt, u32 flags);
 void msm_gem_sync(struct drm_gem_object *obj, u32 op);
+int msm_gem_svm_new_handle(struct drm_device *dev, struct drm_file *file,
+		uint64_t hostptr, uint64_t size,
+		uint32_t flags, uint32_t *handle);
+struct drm_gem_object *msm_gem_svm_new(struct drm_device *dev,
+		struct drm_file *file, uint64_t hostptr,
+		uint64_t size, uint32_t flags);
 
 int msm_framebuffer_prepare(struct drm_framebuffer *fb,
 		struct msm_gem_address_space *aspace);
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 4b1a586d474d..6bb29c62378d 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -25,6 +25,129 @@
 #include "msm_gpu.h"
 #include "msm_mmu.h"
 
+static void msm_gem_mn_free(struct kref *refcount)
+{
+	struct msm_mmu_notifier *msm_mn = container_of(refcount,
+			struct msm_mmu_notifier, refcount);
+
+	mmu_notifier_unregister(&msm_mn->mn, msm_mn->mm);
+	hash_del(&msm_mn->node);
+
+	kfree(msm_mn);
+}
+
+static int msm_gem_mn_get(struct msm_mmu_notifier *msm_mn)
+{
+	if (msm_mn)
+		return kref_get_unless_zero(&msm_mn->refcount);
+	return 0;
+}
+
+static void msm_gem_mn_put(struct msm_mmu_notifier *msm_mn)
+{
+	if (msm_mn) {
+		struct msm_drm_private *msm_dev = msm_mn->msm_dev;
+
+		mutex_lock(&msm_dev->mn_lock);
+		kref_put(&msm_mn->refcount, msm_gem_mn_free);
+		mutex_unlock(&msm_dev->mn_lock);
+	}
+}
+
+void msm_mn_invalidate_range_start(struct mmu_notifier *mn,
+		struct mm_struct *mm, unsigned long start, unsigned long end);
+
+static const struct mmu_notifier_ops msm_mn_ops = {
+	.invalidate_range_start = msm_mn_invalidate_range_start,
+};
+
+static struct msm_mmu_notifier *
+msm_gem_mn_find(struct msm_drm_private *msm_dev, struct mm_struct *mm,
+		struct msm_gem_address_space *aspace)
+{
+	struct msm_mmu_notifier *msm_mn;
+	int ret = 0;
+
+	mutex_lock(&msm_dev->mn_lock);
+	hash_for_each_possible(msm_dev->mn_hash, msm_mn, node,
+			(unsigned long) mm) {
+		if (msm_mn->mm == mm) {
+			if (!msm_gem_mn_get(msm_mn)) {
+				ret = -EINVAL;
+				goto fail;
+			}
+			mutex_unlock(&msm_dev->mn_lock);
+			return msm_mn;
+		}
+	}
+
+	msm_mn = kzalloc(sizeof(*msm_mn), GFP_KERNEL);
+	if (!msm_mn) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	msm_mn->mm = current->mm;
+	msm_mn->mn.ops = &msm_mn_ops;
+	ret = mmu_notifier_register(&msm_mn->mn, msm_mn->mm);
+	if (ret) {
+		kfree(msm_mn);
+		goto fail;
+	}
+
+	msm_mn->svm_tree = RB_ROOT;
+	spin_lock_init(&msm_mn->svm_tree_lock);
+	kref_init(&msm_mn->refcount);
+	msm_mn->msm_dev = msm_dev;
+
+	/* Insert the msm_mn into the hash */
+	hash_add(msm_dev->mn_hash, &msm_mn->node, (unsigned long) msm_mn->mm);
+	mutex_unlock(&msm_dev->mn_lock);
+
+	return msm_mn;
+
+fail:
+	mutex_unlock(&msm_dev->mn_lock);
+	return ERR_PTR(ret);
+}
+
+static int msm_gem_mn_register(struct msm_gem_svm_object *msm_svm_obj,
+		struct msm_gem_address_space *aspace)
+{
+	struct drm_gem_object *obj = &msm_svm_obj->msm_obj_base.base;
+	struct msm_drm_private *msm_dev = obj->dev->dev_private;
+	struct msm_mmu_notifier *msm_mn;
+
+	msm_svm_obj->mm = current->mm;
+	msm_svm_obj->svm_node.start = msm_svm_obj->hostptr;
+	msm_svm_obj->svm_node.last = msm_svm_obj->hostptr + obj->size - 1;
+
+	msm_mn = msm_gem_mn_find(msm_dev, msm_svm_obj->mm, aspace);
+	if (IS_ERR(msm_mn))
+		return PTR_ERR(msm_mn);
+
+	msm_svm_obj->msm_mn = msm_mn;
+
+	spin_lock(&msm_mn->svm_tree_lock);
+	interval_tree_insert(&msm_svm_obj->svm_node, &msm_mn->svm_tree);
+	spin_unlock(&msm_mn->svm_tree_lock);
+
+	return 0;
+}
+
+static void msm_gem_mn_unregister(struct msm_gem_svm_object *msm_svm_obj)
+{
+	struct msm_mmu_notifier *msm_mn = msm_svm_obj->msm_mn;
+
+	/* invalid: bo already unregistered */
+	if (!msm_mn || msm_svm_obj->invalid)
+		return;
+
+	spin_lock(&msm_mn->svm_tree_lock);
+	interval_tree_remove(&msm_svm_obj->svm_node, &msm_mn->svm_tree);
+	spin_unlock(&msm_mn->svm_tree_lock);
+}
+
 static int protect_pages(struct msm_gem_object *msm_obj)
 {
 	int perm = PERM_READ | PERM_WRITE;
@@ -175,10 +298,19 @@ static void put_pages(struct drm_gem_object *obj)
 			sg_free_table(msm_obj->sgt);
 		kfree(msm_obj->sgt);
 
-		if (use_pages(obj))
-			drm_gem_put_pages(obj, msm_obj->pages, true, false);
-		else
+		if (use_pages(obj)) {
+			if (msm_obj->flags & MSM_BO_SVM) {
+				int npages = obj->size >> PAGE_SHIFT;
+
+				release_pages(msm_obj->pages, npages, 0);
+				kfree(msm_obj->pages);
+			} else {
+				drm_gem_put_pages(obj, msm_obj->pages,
+						true, false);
+			}
+		} else {
 			put_pages_vram(obj);
+		}
 
 		msm_obj->pages = NULL;
 	}
@@ -205,8 +337,8 @@ int msm_gem_mmap_obj(struct drm_gem_object *obj,
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	/* We can't mmap secure objects */
-	if (msm_obj->flags & MSM_BO_SECURE) {
+	/* We can't mmap secure objects or SVM objects */
+	if (msm_obj->flags & (MSM_BO_SECURE | MSM_BO_SVM)) {
 		drm_gem_vm_close(vma);
 		return -EACCES;
 	}
@@ -348,14 +480,21 @@ static void
 put_iova(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+	struct msm_gem_svm_object *msm_svm_obj;
 	struct msm_gem_vma *domain, *tmp;
+	bool invalid = false;
 
 	WARN_ON(!mutex_is_locked(&msm_obj->lock));
 
+	if (msm_obj->flags & MSM_BO_SVM) {
+		msm_svm_obj = to_msm_svm_obj(msm_obj);
+		invalid = msm_svm_obj->invalid;
+	}
+
 	list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) {
 		if (iommu_present(&platform_bus_type)) {
 			msm_gem_unmap_vma(domain->aspace, domain,
-				msm_obj->sgt, get_dmabuf_ptr(obj));
+				msm_obj->sgt, get_dmabuf_ptr(obj), invalid);
 		}
 
 		obj_remove_domain(domain);
@@ -658,15 +797,26 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 {
 	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+	struct msm_gem_svm_object *msm_svm_obj = NULL;
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
 	/* object should not be on active list: */
 	WARN_ON(is_active(msm_obj));
+
+	if (msm_obj->flags & MSM_BO_SVM)
+		msm_svm_obj = to_msm_svm_obj(msm_obj);
+
 	list_del(&msm_obj->mm_list);
 
-	mutex_lock(&msm_obj->lock);
+	/* Unregister SVM object from mmu notifications */
+	if (msm_obj->flags & MSM_BO_SVM) {
+		msm_gem_mn_unregister(msm_svm_obj);
+		msm_gem_mn_put(msm_svm_obj->msm_mn);
+		msm_svm_obj->msm_mn = NULL;
+	}
 
+	mutex_lock(&msm_obj->lock);
 	put_iova(obj);
 
 	if (obj->import_attach) {
@@ -691,7 +841,10 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 	drm_gem_object_release(obj);
 	mutex_unlock(&msm_obj->lock);
 
-	kfree(msm_obj);
+	if (msm_obj->flags & MSM_BO_SVM)
+		kfree(msm_svm_obj);
+	else
+		kfree(msm_obj);
 }
 
 /* convenience method to construct a GEM buffer object, and userspace handle */
@@ -714,26 +867,32 @@ int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
 	return ret;
 }
 
-static inline void msm_gem_add_to_inactive_list(struct msm_gem_object *msm_obj,
-		struct drm_device *dev, bool struct_mutex_locked)
+/* convenience method to construct an SVM buffer object, and userspace handle */
+int msm_gem_svm_new_handle(struct drm_device *dev, struct drm_file *file,
+		uint64_t hostptr, uint64_t size,
+		uint32_t flags, uint32_t *handle)
 {
-	struct msm_drm_private *priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	int ret;
 
-	if (struct_mutex_locked) {
-		list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
-	} else {
-		mutex_lock(&dev->struct_mutex);
-		list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
-		mutex_unlock(&dev->struct_mutex);
-	}
+	obj = msm_gem_svm_new(dev, file, hostptr, size, flags);
+
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	ret = drm_gem_handle_create(file, obj, handle);
+
+	/* drop reference from allocate - handle holds it now */
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ret;
 }
 
-static int msm_gem_new_impl(struct drm_device *dev,
-		uint32_t size, uint32_t flags, struct drm_gem_object **obj,
-		bool struct_mutex_locked)
+static int msm_gem_obj_init(struct drm_device *dev,
+		uint32_t size, uint32_t flags,
+		struct msm_gem_object *msm_obj, bool struct_mutex_locked)
 {
 	struct msm_drm_private *priv = dev->dev_private;
-	struct msm_gem_object *msm_obj;
 	bool use_vram = false;
 
 	switch (flags & MSM_BO_CACHE_MASK) {
@@ -755,10 +914,6 @@ static int msm_gem_new_impl(struct drm_device *dev,
 	if (WARN_ON(use_vram && !priv->vram.size))
 		return -EINVAL;
 
-	msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);
-	if (!msm_obj)
-		return -ENOMEM;
-
 	mutex_init(&msm_obj->lock);
 
 	if (use_vram) {
@@ -773,20 +928,44 @@ static int msm_gem_new_impl(struct drm_device *dev,
 	msm_obj->resv = &msm_obj->_resv;
 	reservation_object_init(msm_obj->resv);
 
+	INIT_LIST_HEAD(&msm_obj->mm_list);
 	INIT_LIST_HEAD(&msm_obj->submit_entry);
 	INIT_LIST_HEAD(&msm_obj->domains);
 
-	msm_gem_add_to_inactive_list(msm_obj, dev, struct_mutex_locked);
-
-	*obj = &msm_obj->base;
+	if (struct_mutex_locked) {
+		list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
+	} else {
+		mutex_lock(&dev->struct_mutex);
+		list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
+		mutex_unlock(&dev->struct_mutex);
+	}
 
 	return 0;
 }
 
+static struct drm_gem_object *msm_gem_new_impl(struct drm_device *dev,
+		uint32_t size, uint32_t flags, bool struct_mutex_locked)
+{
+	struct msm_gem_object *msm_obj;
+	int ret;
+
+	msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);
+	if (!msm_obj)
+		return ERR_PTR(-ENOMEM);
+
+	ret = msm_gem_obj_init(dev, size, flags, msm_obj, struct_mutex_locked);
+	if (ret) {
+		kfree(msm_obj);
+		return ERR_PTR(ret);
+	}
+
+	return &msm_obj->base;
+}
+
 static struct drm_gem_object *_msm_gem_new(struct drm_device *dev,
 		uint32_t size, uint32_t flags, bool struct_mutex_locked)
 {
-	struct drm_gem_object *obj = NULL;
+	struct drm_gem_object *obj;
 	int ret;
 
 	size = PAGE_ALIGN(size);
@@ -798,9 +977,9 @@ static struct drm_gem_object *_msm_gem_new(struct drm_device *dev,
 	if (!size)
 		return ERR_PTR(-EINVAL);
 
-	ret = msm_gem_new_impl(dev, size, flags, &obj, struct_mutex_locked);
-	if (ret)
-		goto fail;
+	obj = msm_gem_new_impl(dev, size, flags, struct_mutex_locked);
+	if (IS_ERR(obj))
+		return obj;
 
 	if (use_pages(obj)) {
 		ret = drm_gem_object_init(dev, obj, size);
@@ -813,8 +992,7 @@ static struct drm_gem_object *_msm_gem_new(struct drm_device *dev,
 	return obj;
 
 fail:
-	if (obj)
-		drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_unreference_unlocked(obj);
 
 	return ERR_PTR(ret);
 }
@@ -831,6 +1009,143 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev,
 	return _msm_gem_new(dev, size, flags, false);
 }
 
+static struct drm_gem_object *msm_svm_gem_new_impl(struct drm_device *dev,
+		uint32_t size, uint32_t flags)
+{
+	struct msm_gem_svm_object *msm_svm_obj;
+	struct msm_gem_object *msm_obj;
+	int ret;
+
+	msm_svm_obj = kzalloc(sizeof(*msm_svm_obj), GFP_KERNEL);
+	if (!msm_svm_obj)
+		return ERR_PTR(-ENOMEM);
+
+	msm_obj = &msm_svm_obj->msm_obj_base;
+
+	ret = msm_gem_obj_init(dev, size, flags | MSM_BO_SVM, msm_obj, false);
+	if (ret) {
+		kfree(msm_svm_obj);
+		return ERR_PTR(ret);
+	}
+
+	return &msm_obj->base;
+}
+
+/* convenience method to construct an SVM GEM bo, and userspace handle */
+struct drm_gem_object *msm_gem_svm_new(struct drm_device *dev,
+		struct drm_file *file, uint64_t hostptr,
+		uint64_t size, uint32_t flags)
+{
+	struct drm_gem_object *obj;
+	struct msm_file_private *ctx = file->driver_priv;
+	struct msm_gem_address_space *aspace = ctx->aspace;
+	struct msm_gem_object *msm_obj;
+	struct msm_gem_svm_object *msm_svm_obj;
+	struct msm_gem_vma *domain = NULL;
+	struct page **p;
+	int npages;
+	int num_pinned = 0;
+	int write;
+	int ret;
+
+	/* if we don't have IOMMU, don't bother pretending we can import: */
+	if (!iommu_present(&platform_bus_type)) {
+		dev_err_once(dev->dev, "cannot import without IOMMU\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* hostptr and size must be page-aligned */
+	if (offset_in_page(hostptr | size))
+		return ERR_PTR(-EINVAL);
+
+	/* Only CPU cached SVM objects are allowed */
+	if ((flags & MSM_BO_CACHE_MASK) != MSM_BO_CACHED)
+		return ERR_PTR(-EINVAL);
+
+	/* Allocate and initialize a new msm_gem_object */
+	obj = msm_svm_gem_new_impl(dev, size, flags);
+	if (IS_ERR(obj))
+		return obj;
+
+	drm_gem_private_object_init(dev, obj, size);
+
+	msm_obj = to_msm_bo(obj);
+	domain = obj_add_domain(&msm_obj->base, aspace);
+	if (IS_ERR(domain)) {
+		drm_gem_object_unreference_unlocked(obj);
+		return ERR_CAST(domain);
+	}
+
+	/* Reserve iova if not already in use, else fail */
+	ret = msm_gem_reserve_iova(aspace, domain, hostptr, size);
+	if (ret) {
+		obj_remove_domain(domain);
+		drm_gem_object_unreference_unlocked(obj);
+		return ERR_PTR(ret);
+	}
+
+	msm_svm_obj = to_msm_svm_obj(msm_obj);
+	msm_svm_obj->hostptr = hostptr;
+	msm_svm_obj->invalid = false;
+
+	ret = msm_gem_mn_register(msm_svm_obj, aspace);
+	if (ret)
+		goto fail;
+
+	/*
+	 * Get physical pages and map into smmu in the ioctl itself.
+	 * The driver handles iova allocation, physical page allocation and
+	 * SMMU map all in one go. If we break this, then we have to maintain
+	 * state to tell if physical pages allocation/map needs to happen.
+	 * For SVM, iova reservation needs to happen in the ioctl itself,
+	 * so do the rest right here as well.
+	 */
+	npages = size >> PAGE_SHIFT;
+	p = kcalloc(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!p) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	write = (msm_obj->flags & MSM_BO_GPU_READONLY) ? 0 : 1;
+	/* This may hold mm->mmap_sem */
+	num_pinned = get_user_pages_fast(hostptr, npages, write, p);
+	if (num_pinned != npages) {
+		ret = -EINVAL;
+		goto free_pages;
+	}
+
+	msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
+	if (IS_ERR(msm_obj->sgt)) {
+		ret = PTR_ERR(msm_obj->sgt);
+		goto free_pages;
+	}
+
+	msm_obj->pages = p;
+
+	ret = aspace->mmu->funcs->map(aspace->mmu, domain->iova,
+			msm_obj->sgt, msm_obj->flags, get_dmabuf_ptr(obj));
+	if (ret)
+		goto free_pages;
+
+	kref_get(&aspace->kref);
+
+	return obj;
+
+free_pages:
+	release_pages(p, num_pinned, 0);
+	kfree(p);
+
+fail:
+	if (domain)
+		msm_gem_release_iova(aspace, domain);
+
+	obj_remove_domain(domain);
+	drm_gem_object_unreference_unlocked(obj);
+
+	return ERR_PTR(ret);
+}
+
 struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 		uint32_t size, struct sg_table *sgt, u32 flags)
 {
@@ -846,10 +1161,9 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 
 	size = PAGE_ALIGN(size);
 
-	ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj, false);
-
-	if (ret)
-		goto fail;
+	obj = msm_gem_new_impl(dev, size, MSM_BO_WC, false);
+	if (IS_ERR(obj))
+		return obj;
 
 	drm_gem_private_object_init(dev, obj, size);
 
@@ -868,7 +1182,8 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 	/* OR the passed in flags */
 	msm_obj->flags |= flags;
 
-	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages,
+			NULL, npages);
 	if (ret) {
 		mutex_unlock(&msm_obj->lock);
 		goto fail;
@@ -879,8 +1194,86 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 	return obj;
 
 fail:
-	if (obj)
-		drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_unreference_unlocked(obj);
 
 	return ERR_PTR(ret);
 }
+
+/* Timeout in ms, long enough so we are sure the GPU is hung */
+#define SVM_OBJ_WAIT_TIMEOUT 10000
+static void invalidate_svm_object(struct msm_gem_svm_object *msm_svm_obj)
+{
+	struct msm_gem_object *msm_obj = &msm_svm_obj->msm_obj_base;
+	struct drm_device *dev = msm_obj->base.dev;
+	struct msm_gem_vma *domain, *tmp;
+	uint32_t fence;
+	int ret;
+
+	if (is_active(msm_obj)) {
+		ktime_t timeout = ktime_add_ms(ktime_get(),
+				SVM_OBJ_WAIT_TIMEOUT);
+
+		/* Get the most recent fence that touches the object */
+		fence = msm_gem_fence(msm_obj, MSM_PREP_READ | MSM_PREP_WRITE);
+
+		/* Wait for the fence to retire */
+		ret = msm_wait_fence(dev, fence, &timeout, true);
+		if (ret)
+			/* The GPU could be hung! Not much we can do */
+			dev_err(dev->dev, "drm: Error (%d) waiting for svm object: 0x%llx",
+					ret, msm_svm_obj->hostptr);
+	}
+
+	/* GPU is done, unmap object from SMMU */
+	mutex_lock(&msm_obj->lock);
+	list_for_each_entry_safe(domain, tmp, &msm_obj->domains, list) {
+		struct msm_gem_address_space *aspace = domain->aspace;
+
+		if (domain->iova)
+			aspace->mmu->funcs->unmap(aspace->mmu,
+					domain->iova, msm_obj->sgt,
+					get_dmabuf_ptr(&msm_obj->base));
+	}
+	/* Let go of the physical pages */
+	put_pages(&msm_obj->base);
+	mutex_unlock(&msm_obj->lock);
+}
+
+void msm_mn_invalidate_range_start(struct mmu_notifier *mn,
+		struct mm_struct *mm, unsigned long start, unsigned long end)
+{
+	struct msm_mmu_notifier *msm_mn =
+		container_of(mn, struct msm_mmu_notifier, mn);
+	struct interval_tree_node *itn = NULL;
+	struct msm_gem_svm_object *msm_svm_obj;
+	struct drm_gem_object *obj;
+	LIST_HEAD(inv_list);
+
+	if (!msm_gem_mn_get(msm_mn))
+		return;
+
+	spin_lock(&msm_mn->svm_tree_lock);
+	itn = interval_tree_iter_first(&msm_mn->svm_tree, start, end - 1);
+	while (itn) {
+		msm_svm_obj = container_of(itn,
+				struct msm_gem_svm_object, svm_node);
+		obj = &msm_svm_obj->msm_obj_base.base;
+
+		if (kref_get_unless_zero(&obj->refcount))
+			list_add(&msm_svm_obj->lnode, &inv_list);
+
+		itn = interval_tree_iter_next(itn, start, end - 1);
+	}
+	spin_unlock(&msm_mn->svm_tree_lock);
+
+	list_for_each_entry(msm_svm_obj, &inv_list, lnode) {
+		obj = &msm_svm_obj->msm_obj_base.base;
+		/* Unregister SVM object from mmu notifications */
+		msm_gem_mn_unregister(msm_svm_obj);
+		msm_svm_obj->invalid = true;
+		invalidate_svm_object(msm_svm_obj);
+		drm_gem_object_unreference_unlocked(obj);
+	}
+
+	msm_gem_mn_put(msm_mn);
+}
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index eb850952f1f5..04e6c658b5f3 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -20,11 +20,14 @@
 
 #include <linux/kref.h>
 #include <linux/reservation.h>
+#include <linux/mmu_notifier.h>
+#include <linux/interval_tree.h>
 #include "msm_drv.h"
 
 /* Additional internal-use only BO flags: */
 #define MSM_BO_STOLEN        0x10000000    /* try to use stolen/splash memory */
 #define MSM_BO_LOCKED        0x20000000    /* Pages have been securely locked */
+#define MSM_BO_SVM           0x40000000    /* bo is SVM */
 
 struct msm_gem_address_space {
 	const char *name;
@@ -85,6 +88,32 @@ struct msm_gem_object {
 };
 #define to_msm_bo(x) container_of(x, struct msm_gem_object, base)
 
+struct msm_mmu_notifier {
+	struct mmu_notifier mn;
+	struct mm_struct *mm; /* mm_struct owning the mmu notifier mn */
+	struct hlist_node node;
+	struct rb_root svm_tree; /* interval tree holding all svm bos */
+	spinlock_t svm_tree_lock; /* Protects svm_tree*/
+	struct msm_drm_private *msm_dev;
+	struct kref refcount;
+};
+
+struct msm_gem_svm_object {
+	struct msm_gem_object msm_obj_base;
+	uint64_t hostptr;
+	struct mm_struct *mm; /* mm_struct the svm bo belongs to */
+	struct interval_tree_node svm_node;
+	struct msm_mmu_notifier *msm_mn;
+	struct list_head lnode;
+	/* bo has been unmapped on CPU, cannot be part of GPU submits */
+	bool invalid;
+};
+
+#define to_msm_svm_obj(x) \
+	((struct msm_gem_svm_object *) \
+	 container_of(x, struct msm_gem_svm_object, msm_obj_base))
+
+
 static inline bool is_active(struct msm_gem_object *msm_obj)
 {
 	return msm_obj->gpu != NULL;
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index c9b3151180dc..c861bfd77537 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -217,6 +217,18 @@ retry:
 			submit->bos[i].flags |= BO_LOCKED;
 		}
 
+		/*
+		 * An invalid SVM object is part of
+		 * this submit's buffer list, fail.
+		 */
+		if (msm_obj->flags & MSM_BO_SVM) {
+			struct msm_gem_svm_object *msm_svm_obj =
+				to_msm_svm_obj(msm_obj);
+			if (msm_svm_obj->invalid) {
+				ret = -EINVAL;
+				goto fail;
+			}
+		}
 
 		/* if locking succeeded, pin bo: */
 		ret = msm_gem_get_iova(&msm_obj->base, aspace, &iova);
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index 12e3c0f7c101..95be430ea5d8 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -98,6 +98,45 @@ static int allocate_iova(struct msm_gem_address_space *aspace,
 	return ret;
 }
 
+int msm_gem_reserve_iova(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma,
+		uint64_t hostptr, uint64_t size)
+{
+	struct drm_mm *mm = &aspace->mm;
+	uint64_t start = hostptr >> PAGE_SHIFT;
+	uint64_t last = (hostptr + size - 1) >> PAGE_SHIFT;
+	int ret;
+
+	spin_lock(&aspace->lock);
+
+	if (drm_mm_interval_first(mm, start, last)) {
+		/* iova already in use, fail */
+		spin_unlock(&aspace->lock);
+		return -EADDRINUSE;
+	}
+
+	vma->node.start = hostptr >> PAGE_SHIFT;
+	vma->node.size = size >> PAGE_SHIFT;
+	vma->node.color = 0;
+
+	ret = drm_mm_reserve_node(mm, &vma->node);
+	if (!ret)
+		vma->iova = hostptr;
+
+	spin_unlock(&aspace->lock);
+
+	return ret;
+}
+
+void msm_gem_release_iova(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma)
+{
+	spin_lock(&aspace->lock);
+	if (drm_mm_node_allocated(&vma->node))
+		drm_mm_remove_node(&vma->node);
+	spin_unlock(&aspace->lock);
+}
+
 int msm_gem_map_vma(struct msm_gem_address_space *aspace,
 		struct msm_gem_vma *vma, struct sg_table *sgt,
 		void *priv, unsigned int flags)
@@ -116,11 +155,7 @@ int msm_gem_map_vma(struct msm_gem_address_space *aspace,
 		flags, priv);
 
 	if (ret) {
-		spin_lock(&aspace->lock);
-		if (drm_mm_node_allocated(&vma->node))
-			drm_mm_remove_node(&vma->node);
-		spin_unlock(&aspace->lock);
-
+		msm_gem_release_iova(aspace, vma);
 		return ret;
 	}
 
@@ -131,17 +166,16 @@ int msm_gem_map_vma(struct msm_gem_address_space *aspace,
 }
 
 void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, struct sg_table *sgt, void *priv)
+		struct msm_gem_vma *vma, struct sg_table *sgt,
+		void *priv, bool invalidated)
 {
 	if (!aspace || !vma->iova)
 		return;
 
-	aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, sgt, priv);
+	if (!invalidated)
+		aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, sgt, priv);
 
-	spin_lock(&aspace->lock);
-	if (drm_mm_node_allocated(&vma->node))
-		drm_mm_remove_node(&vma->node);
-	spin_unlock(&aspace->lock);
+	msm_gem_release_iova(aspace, vma);
 
 	vma->iova = 0;