diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index f7181a4..12de42c 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -353,6 +353,7 @@
 	unsigned long *rmap;
 	unsigned long *dirty_bitmap;
 	unsigned long userspace_addr;
+	int user_alloc;
 };
 
 struct kvm {
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 453e98e..ab2c77c 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -713,6 +713,7 @@
 
 		memset(new.rmap, 0, npages * sizeof(*new.rmap));
 
+		new.user_alloc = user_alloc;
 		if (user_alloc)
 			new.userspace_addr = mem->userspace_addr;
 		else {
@@ -727,6 +728,19 @@
 			if (IS_ERR((void *)new.userspace_addr))
 				goto out_unlock;
 		}
+	} else {
+		if (!old.user_alloc && old.rmap) {
+			int ret;
+
+			down_write(&current->mm->mmap_sem);
+			ret = do_munmap(current->mm, old.userspace_addr,
+					old.npages * PAGE_SIZE);
+			up_write(&current->mm->mmap_sem);
+			if (ret < 0)
+				printk(KERN_WARNING
+				       "kvm_vm_ioctl_set_memory_region: "
+				       "failed to munmap memory\n");
+		}
 	}
 
 	/* Allocate page dirty bitmap if needed */