t/t5326-multi-pack-bitmaps.sh - jrn/git - Git at Google

 #!/bin/sh

 test_description='exercise basic multi-pack bitmap functionality'
 . ./test-lib.sh
 . "${TEST_DIRECTORY}/lib-bitmap.sh"

 # We'll be writing our own midx and bitmaps, so avoid getting confused by the
 # automatic ones.
 GIT_TEST_MULTI_PACK_INDEX=0
 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0

 objdir=.git/objects
 midx=$objdir/pack/multi-pack-index

 # midx_pack_source <obj>
 midx_pack_source () {
 	test-tool read-midx --show-objects .git/objects | grep "^$1 " | cut -f2
 }

 setup_bitmap_history

 test_expect_success 'enable core.multiPackIndex' '
 	git config core.multiPackIndex true
 '

 test_expect_success 'create single-pack midx with bitmaps' '
 	git repack -ad &&
 	git multi-pack-index write --bitmap &&
 	test_path_is_file $midx &&
 	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 	test_path_is_file $midx-$(midx_checksum $objdir).rev
 '

 basic_bitmap_tests

 test_expect_success 'create new additional packs' '
 	for i in $(test_seq 1 16)
 	do
 		test_commit "$i" &&
 		git repack -d || return 1
 	done &&

 	git checkout -b other2 HEAD~8 &&
 	for i in $(test_seq 1 8)
 	do
 		test_commit "side-$i" &&
 		git repack -d || return 1
 	done &&
 	git checkout second
 '

 test_expect_success 'create multi-pack midx with bitmaps' '
 	git multi-pack-index write --bitmap &&

 	ls $objdir/pack/pack-*.pack >packs &&
 	test_line_count = 25 packs &&

 	test_path_is_file $midx &&
 	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 	test_path_is_file $midx-$(midx_checksum $objdir).rev
 '

 basic_bitmap_tests

 test_expect_success '--no-bitmap is respected when bitmaps exist' '
 	git multi-pack-index write --bitmap &&

 	test_commit respect--no-bitmap &&
 	git repack -d &&

 	test_path_is_file $midx &&
 	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 	test_path_is_file $midx-$(midx_checksum $objdir).rev &&

 	git multi-pack-index write --no-bitmap &&

 	test_path_is_file $midx &&
 	test_path_is_missing $midx-$(midx_checksum $objdir).bitmap &&
 	test_path_is_missing $midx-$(midx_checksum $objdir).rev
 '

 test_expect_success 'setup midx with base from later pack' '
 	# Write a and b so that "a" is a delta on top of base "b", since Git
 	# prefers to delete contents out of a base rather than add to a shorter
 	# object.
 	test_seq 1 128 >a &&
 	test_seq 1 130 >b &&

 	git add a b &&
 	git commit -m "initial commit" &&

 	a=$(git rev-parse HEAD:a) &&
 	b=$(git rev-parse HEAD:b) &&

 	# In the first pack, "a" is stored as a delta to "b".
 	p1=$(git pack-objects .git/objects/pack/pack <<-EOF
 	$a
 	$b
 	EOF
 	) &&

 	# In the second pack, "a" is missing, and "b" is not a delta nor base to
 	# any other object.
 	p2=$(git pack-objects .git/objects/pack/pack <<-EOF
 	$b
 	$(git rev-parse HEAD)
 	$(git rev-parse HEAD^{tree})
 	EOF
 	) &&

 	git prune-packed &&
 	# Use the second pack as the preferred source, so that "b" occurs
 	# earlier in the MIDX object order, rendering "a" unusable for pack
 	# reuse.
 	git multi-pack-index write --bitmap --preferred-pack=pack-$p2.idx &&

 	have_delta $a $b &&
 	test $(midx_pack_source $a) != $(midx_pack_source $b)
 '

 rev_list_tests 'full bitmap with backwards delta'

 test_expect_success 'clone with bitmaps enabled' '
 	git clone --no-local --bare . clone-reverse-delta.git &&
 	test_when_finished "rm -fr clone-reverse-delta.git" &&

 	git rev-parse HEAD >expect &&
 	git --git-dir=clone-reverse-delta.git rev-parse HEAD >actual &&
 	test_cmp expect actual
 '

 bitmap_reuse_tests() {
 	from=$1
 	to=$2

 	test_expect_success "setup pack reuse tests ($from -> $to)" '
 		rm -fr repo &&
 		git init repo &&
 		(
 			cd repo &&
 			test_commit_bulk 16 &&
 			git tag old-tip &&

 			git config core.multiPackIndex true &&
 			if test "MIDX" = "$from"
 			then
 				git repack -Ad &&
 				git multi-pack-index write --bitmap
 			else
 				git repack -Adb
 			fi
 		)
 	'

 	test_expect_success "build bitmap from existing ($from -> $to)" '
 		(
 			cd repo &&
 			test_commit_bulk --id=further 16 &&
 			git tag new-tip &&

 			if test "MIDX" = "$to"
 			then
 				git repack -d &&
 				git multi-pack-index write --bitmap
 			else
 				git repack -Adb
 			fi
 		)
 	'

 	test_expect_success "verify resulting bitmaps ($from -> $to)" '
 		(
 			cd repo &&
 			git for-each-ref &&
 			git rev-list --test-bitmap refs/tags/old-tip &&
 			git rev-list --test-bitmap refs/tags/new-tip
 		)
 	'
 }

 bitmap_reuse_tests 'pack' 'MIDX'
 bitmap_reuse_tests 'MIDX' 'pack'
 bitmap_reuse_tests 'MIDX' 'MIDX'

 test_expect_success 'missing object closure fails gracefully' '
 	rm -fr repo &&
 	git init repo &&
 	test_when_finished "rm -fr repo" &&
 	(
 		cd repo &&

 		test_commit loose &&
 		test_commit packed &&

 		# Do not pass "--revs"; we want a pack without the "loose"
 		# commit.
 		git pack-objects $objdir/pack/pack <<-EOF &&
 		$(git rev-parse packed)
 		EOF

 		test_must_fail git multi-pack-index write --bitmap 2>err &&
 		grep "doesn.t have full closure" err &&
 		test_path_is_missing $midx
 	)
 '

 test_expect_success 'setup partial bitmaps' '
 	test_commit packed &&
 	git repack &&
 	test_commit loose &&
 	git multi-pack-index write --bitmap 2>err &&
 	test_path_is_file $midx &&
 	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 	test_path_is_file $midx-$(midx_checksum $objdir).rev
 '

 basic_bitmap_tests HEAD~

 test_expect_success 'removing a MIDX clears stale bitmaps' '
 	rm -fr repo &&
 	git init repo &&
 	test_when_finished "rm -fr repo" &&
 	(
 		cd repo &&
 		test_commit base &&
 		git repack &&
 		git multi-pack-index write --bitmap &&

 		# Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
 		stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
 		stale_rev=$midx-$(midx_checksum $objdir).rev &&
 		rm $midx &&

 		# Then write a new MIDX.
 		test_commit new &&
 		git repack &&
 		git multi-pack-index write --bitmap &&

 		test_path_is_file $midx &&
 		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 		test_path_is_file $midx-$(midx_checksum $objdir).rev &&
 		test_path_is_missing $stale_bitmap &&
 		test_path_is_missing $stale_rev
 	)
 '

 test_expect_success 'pack.preferBitmapTips' '
 	git init repo &&
 	test_when_finished "rm -fr repo" &&
 	(
 		cd repo &&

 		test_commit_bulk --message="%s" 103 &&

 		git log --format="%H" >commits.raw &&
 		sort <commits.raw >commits &&

 		git log --format="create refs/tags/%s %H" HEAD >refs &&
 		git update-ref --stdin <refs &&

 		git multi-pack-index write --bitmap &&
 		test_path_is_file $midx &&
 		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 		test_path_is_file $midx-$(midx_checksum $objdir).rev &&

 		test-tool bitmap list-commits | sort >bitmaps &&
 		comm -13 bitmaps commits >before &&
 		test_line_count = 1 before &&

 		perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
 			<before | git update-ref --stdin &&

 		rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 		rm -fr $midx-$(midx_checksum $objdir).rev &&
 		rm -fr $midx &&

 		git -c pack.preferBitmapTips=refs/tags/include \
 			multi-pack-index write --bitmap &&
 		test-tool bitmap list-commits | sort >bitmaps &&
 		comm -13 bitmaps commits >after &&

 		! test_cmp before after
 	)
 '

 test_expect_success 'writing a bitmap with --refs-snapshot' '
 	git init repo &&
 	test_when_finished "rm -fr repo" &&
 	(
 		cd repo &&

 		test_commit one &&
 		test_commit two &&

 		git rev-parse one >snapshot &&

 		git repack -ad &&

 		# First, write a MIDX which see both refs/tags/one and
 		# refs/tags/two (causing both of those commits to receive
 		# bitmaps).
 		git multi-pack-index write --bitmap &&

 		test_path_is_file $midx &&
 		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

 		test-tool bitmap list-commits | sort >bitmaps &&
 		grep "$(git rev-parse one)" bitmaps &&
 		grep "$(git rev-parse two)" bitmaps &&

 		rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 		rm -fr $midx-$(midx_checksum $objdir).rev &&
 		rm -fr $midx &&

 		# Then again, but with a refs snapshot which only sees
 		# refs/tags/one.
 		git multi-pack-index write --bitmap --refs-snapshot=snapshot &&

 		test_path_is_file $midx &&
 		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

 		test-tool bitmap list-commits | sort >bitmaps &&
 		grep "$(git rev-parse one)" bitmaps &&
 		! grep "$(git rev-parse two)" bitmaps
 	)
 '

 test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
 	git init repo &&
 	test_when_finished "rm -fr repo" &&
 	(
 		cd repo &&

 		test_commit_bulk --message="%s" 103 &&

 		git log --format="%H" >commits.raw &&
 		sort <commits.raw >commits &&

 		git log --format="create refs/tags/%s %H" HEAD >refs &&
 		git update-ref --stdin <refs &&

 		git multi-pack-index write --bitmap &&
 		test_path_is_file $midx &&
 		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

 		test-tool bitmap list-commits | sort >bitmaps &&
 		comm -13 bitmaps commits >before &&
 		test_line_count = 1 before &&

 		(
 			grep -vf before commits.raw &&
 			# mark missing commits as preferred
 			sed "s/^/+/" before
 		) >snapshot &&

 		rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 		rm -fr $midx-$(midx_checksum $objdir).rev &&
 		rm -fr $midx &&

 		git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
 		test-tool bitmap list-commits | sort >bitmaps &&
 		comm -13 bitmaps commits >after &&

 		! test_cmp before after
 	)
 '

 test_expect_success 'hash-cache values are propagated from pack bitmaps' '
 	rm -fr repo &&
 	git init repo &&
 	test_when_finished "rm -fr repo" &&
 	(
 		cd repo &&

 		test_commit base &&
 		test_commit base2 &&
 		git repack -adb &&

 		test-tool bitmap dump-hashes >pack.raw &&
 		test_file_not_empty pack.raw &&
 		sort pack.raw >pack.hashes &&

 		test_commit new &&
 		git repack &&
 		git multi-pack-index write --bitmap &&

 		test-tool bitmap dump-hashes >midx.raw &&
 		sort midx.raw >midx.hashes &&

 		# ensure that every namehash in the pack bitmap can be found in
 		# the midx bitmap (i.e., that there are no oid-namehash pairs
 		# unique to the pack bitmap).
 		comm -23 pack.hashes midx.hashes >dropped.hashes &&
 		test_must_be_empty dropped.hashes
 	)
 '

 test_done
	#!/bin/sh

	test_description='exercise basic multi-pack bitmap functionality'
	. ./test-lib.sh
	. "${TEST_DIRECTORY}/lib-bitmap.sh"

	# We'll be writing our own midx and bitmaps, so avoid getting confused by the
	# automatic ones.
	GIT_TEST_MULTI_PACK_INDEX=0
	GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0

	objdir=.git/objects
	midx=$objdir/pack/multi-pack-index

	# midx_pack_source <obj>
	midx_pack_source () {
	test-tool read-midx --show-objects .git/objects \| grep "^$1 " \| cut -f2
	}

	setup_bitmap_history

	test_expect_success 'enable core.multiPackIndex' '
	git config core.multiPackIndex true
	'

	test_expect_success 'create single-pack midx with bitmaps' '
	git repack -ad &&
	git multi-pack-index write --bitmap &&
	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_file $midx-$(midx_checksum $objdir).rev
	'

	basic_bitmap_tests

	test_expect_success 'create new additional packs' '
	for i in $(test_seq 1 16)
	do
	test_commit "$i" &&
	git repack -d \|\| return 1
	done &&

	git checkout -b other2 HEAD~8 &&
	for i in $(test_seq 1 8)
	do
	test_commit "side-$i" &&
	git repack -d \|\| return 1
	done &&
	git checkout second
	'

	test_expect_success 'create multi-pack midx with bitmaps' '
	git multi-pack-index write --bitmap &&

	ls $objdir/pack/pack-*.pack >packs &&
	test_line_count = 25 packs &&

	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_file $midx-$(midx_checksum $objdir).rev
	'

	basic_bitmap_tests

	test_expect_success '--no-bitmap is respected when bitmaps exist' '
	git multi-pack-index write --bitmap &&

	test_commit respect--no-bitmap &&
	git repack -d &&

	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_file $midx-$(midx_checksum $objdir).rev &&

	git multi-pack-index write --no-bitmap &&

	test_path_is_file $midx &&
	test_path_is_missing $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_missing $midx-$(midx_checksum $objdir).rev
	'

	test_expect_success 'setup midx with base from later pack' '
	# Write a and b so that "a" is a delta on top of base "b", since Git
	# prefers to delete contents out of a base rather than add to a shorter
	# object.
	test_seq 1 128 >a &&
	test_seq 1 130 >b &&

	git add a b &&
	git commit -m "initial commit" &&

	a=$(git rev-parse HEAD:a) &&
	b=$(git rev-parse HEAD:b) &&

	# In the first pack, "a" is stored as a delta to "b".
	p1=$(git pack-objects .git/objects/pack/pack <<-EOF
	$a
	$b
	EOF
	) &&

	# In the second pack, "a" is missing, and "b" is not a delta nor base to
	# any other object.
	p2=$(git pack-objects .git/objects/pack/pack <<-EOF
	$b
	$(git rev-parse HEAD)
	$(git rev-parse HEAD^{tree})
	EOF
	) &&

	git prune-packed &&
	# Use the second pack as the preferred source, so that "b" occurs
	# earlier in the MIDX object order, rendering "a" unusable for pack
	# reuse.
	git multi-pack-index write --bitmap --preferred-pack=pack-$p2.idx &&

	have_delta $a $b &&
	test $(midx_pack_source $a) != $(midx_pack_source $b)
	'

	rev_list_tests 'full bitmap with backwards delta'

	test_expect_success 'clone with bitmaps enabled' '
	git clone --no-local --bare . clone-reverse-delta.git &&
	test_when_finished "rm -fr clone-reverse-delta.git" &&

	git rev-parse HEAD >expect &&
	git --git-dir=clone-reverse-delta.git rev-parse HEAD >actual &&
	test_cmp expect actual
	'

	bitmap_reuse_tests() {
	from=$1
	to=$2

	test_expect_success "setup pack reuse tests ($from -> $to)" '
	rm -fr repo &&
	git init repo &&
	(
	cd repo &&
	test_commit_bulk 16 &&
	git tag old-tip &&

	git config core.multiPackIndex true &&
	if test "MIDX" = "$from"
	then
	git repack -Ad &&
	git multi-pack-index write --bitmap
	else
	git repack -Adb
	fi
	)
	'

	test_expect_success "build bitmap from existing ($from -> $to)" '
	(
	cd repo &&
	test_commit_bulk --id=further 16 &&
	git tag new-tip &&

	if test "MIDX" = "$to"
	then
	git repack -d &&
	git multi-pack-index write --bitmap
	else
	git repack -Adb
	fi
	)
	'

	test_expect_success "verify resulting bitmaps ($from -> $to)" '
	(
	cd repo &&
	git for-each-ref &&
	git rev-list --test-bitmap refs/tags/old-tip &&
	git rev-list --test-bitmap refs/tags/new-tip
	)
	'
	}

	bitmap_reuse_tests 'pack' 'MIDX'
	bitmap_reuse_tests 'MIDX' 'pack'
	bitmap_reuse_tests 'MIDX' 'MIDX'

	test_expect_success 'missing object closure fails gracefully' '
	rm -fr repo &&
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
	cd repo &&

	test_commit loose &&
	test_commit packed &&

	# Do not pass "--revs"; we want a pack without the "loose"
	# commit.
	git pack-objects $objdir/pack/pack <<-EOF &&
	$(git rev-parse packed)
	EOF

	test_must_fail git multi-pack-index write --bitmap 2>err &&
	grep "doesn.t have full closure" err &&
	test_path_is_missing $midx
	)
	'

	test_expect_success 'setup partial bitmaps' '
	test_commit packed &&
	git repack &&
	test_commit loose &&
	git multi-pack-index write --bitmap 2>err &&
	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_file $midx-$(midx_checksum $objdir).rev
	'

	basic_bitmap_tests HEAD~

	test_expect_success 'removing a MIDX clears stale bitmaps' '
	rm -fr repo &&
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
	cd repo &&
	test_commit base &&
	git repack &&
	git multi-pack-index write --bitmap &&

	# Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
	stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
	stale_rev=$midx-$(midx_checksum $objdir).rev &&
	rm $midx &&

	# Then write a new MIDX.
	test_commit new &&
	git repack &&
	git multi-pack-index write --bitmap &&

	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_file $midx-$(midx_checksum $objdir).rev &&
	test_path_is_missing $stale_bitmap &&
	test_path_is_missing $stale_rev
	)
	'

	test_expect_success 'pack.preferBitmapTips' '
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
	cd repo &&

	test_commit_bulk --message="%s" 103 &&

	git log --format="%H" >commits.raw &&
	sort <commits.raw >commits &&

	git log --format="create refs/tags/%s %H" HEAD >refs &&
	git update-ref --stdin <refs &&

	git multi-pack-index write --bitmap &&
	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	test_path_is_file $midx-$(midx_checksum $objdir).rev &&

	test-tool bitmap list-commits \| sort >bitmaps &&
	comm -13 bitmaps commits >before &&
	test_line_count = 1 before &&

	perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
	<before \| git update-ref --stdin &&

	rm -fr $midx-$(midx_checksum $objdir).bitmap &&
	rm -fr $midx-$(midx_checksum $objdir).rev &&
	rm -fr $midx &&

	git -c pack.preferBitmapTips=refs/tags/include \
	multi-pack-index write --bitmap &&
	test-tool bitmap list-commits \| sort >bitmaps &&
	comm -13 bitmaps commits >after &&

	! test_cmp before after
	)
	'

	test_expect_success 'writing a bitmap with --refs-snapshot' '
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
	cd repo &&

	test_commit one &&
	test_commit two &&

	git rev-parse one >snapshot &&

	git repack -ad &&

	# First, write a MIDX which see both refs/tags/one and
	# refs/tags/two (causing both of those commits to receive
	# bitmaps).
	git multi-pack-index write --bitmap &&

	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

	test-tool bitmap list-commits \| sort >bitmaps &&
	grep "$(git rev-parse one)" bitmaps &&
	grep "$(git rev-parse two)" bitmaps &&

	rm -fr $midx-$(midx_checksum $objdir).bitmap &&
	rm -fr $midx-$(midx_checksum $objdir).rev &&
	rm -fr $midx &&

	# Then again, but with a refs snapshot which only sees
	# refs/tags/one.
	git multi-pack-index write --bitmap --refs-snapshot=snapshot &&

	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

	test-tool bitmap list-commits \| sort >bitmaps &&
	grep "$(git rev-parse one)" bitmaps &&
	! grep "$(git rev-parse two)" bitmaps
	)
	'

	test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
	cd repo &&

	test_commit_bulk --message="%s" 103 &&

	git log --format="%H" >commits.raw &&
	sort <commits.raw >commits &&

	git log --format="create refs/tags/%s %H" HEAD >refs &&
	git update-ref --stdin <refs &&

	git multi-pack-index write --bitmap &&
	test_path_is_file $midx &&
	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

	test-tool bitmap list-commits \| sort >bitmaps &&
	comm -13 bitmaps commits >before &&
	test_line_count = 1 before &&

	(
	grep -vf before commits.raw &&
	# mark missing commits as preferred
	sed "s/^/+/" before
	) >snapshot &&

	rm -fr $midx-$(midx_checksum $objdir).bitmap &&
	rm -fr $midx-$(midx_checksum $objdir).rev &&
	rm -fr $midx &&

	git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
	test-tool bitmap list-commits \| sort >bitmaps &&
	comm -13 bitmaps commits >after &&

	! test_cmp before after
	)
	'

	test_expect_success 'hash-cache values are propagated from pack bitmaps' '
	rm -fr repo &&
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
	cd repo &&

	test_commit base &&
	test_commit base2 &&
	git repack -adb &&

	test-tool bitmap dump-hashes >pack.raw &&
	test_file_not_empty pack.raw &&
	sort pack.raw >pack.hashes &&

	test_commit new &&
	git repack &&
	git multi-pack-index write --bitmap &&

	test-tool bitmap dump-hashes >midx.raw &&
	sort midx.raw >midx.hashes &&

	# ensure that every namehash in the pack bitmap can be found in
	# the midx bitmap (i.e., that there are no oid-namehash pairs
	# unique to the pack bitmap).
	comm -23 pack.hashes midx.hashes >dropped.hashes &&
	test_must_be_empty dropped.hashes
	)
	'

	test_done