diff --git a/git-merge-one-file.sh b/git-merge-one-file.sh
index b08597d..b285990 100755
--- a/git-merge-one-file.sh
+++ b/git-merge-one-file.sh
@@ -25,7 +25,12 @@
 		echo "Removing $4"
 	fi
 	if test -f "$4"; then
-		rm -f -- "$4"
+		rm -f -- "$4" &&
+		dn="$4" &&
+		while dn=$(expr "$dn" : '\(.*\)/') && rmdir "$dn" 2>/dev/null
+		do
+			:;
+		done
 	fi &&
 		exec git-update-index --remove -- "$4"
 	;;