Derrick Stolee | a340773 | 2018-07-12 15:39:21 -0400 | [diff] [blame] | 1 | #!/bin/sh |
| 2 | |
| 3 | test_description='multi-pack-indexes' |
| 4 | . ./test-lib.sh |
| 5 | |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 6 | objdir=.git/objects |
| 7 | |
Derrick Stolee | 4d80560 | 2018-07-12 15:39:23 -0400 | [diff] [blame] | 8 | midx_read_expect () { |
Derrick Stolee | 396f257 | 2018-07-12 15:39:26 -0400 | [diff] [blame] | 9 | NUM_PACKS=$1 |
Derrick Stolee | d7cacf2 | 2018-07-12 15:39:31 -0400 | [diff] [blame] | 10 | NUM_OBJECTS=$2 |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 11 | NUM_CHUNKS=$3 |
| 12 | OBJECT_DIR=$4 |
| 13 | EXTRA_CHUNKS="$5" |
Derrick Stolee | 3227565 | 2018-07-12 15:39:28 -0400 | [diff] [blame] | 14 | { |
| 15 | cat <<-EOF && |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 16 | header: 4d494458 1 $NUM_CHUNKS $NUM_PACKS |
| 17 | chunks: pack-names oid-fanout oid-lookup object-offsets$EXTRA_CHUNKS |
Derrick Stolee | d7cacf2 | 2018-07-12 15:39:31 -0400 | [diff] [blame] | 18 | num_objects: $NUM_OBJECTS |
Derrick Stolee | 3227565 | 2018-07-12 15:39:28 -0400 | [diff] [blame] | 19 | packs: |
| 20 | EOF |
| 21 | if test $NUM_PACKS -ge 1 |
| 22 | then |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 23 | ls $OBJECT_DIR/pack/ | grep idx | sort |
Derrick Stolee | 3227565 | 2018-07-12 15:39:28 -0400 | [diff] [blame] | 24 | fi && |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 25 | printf "object-dir: $OBJECT_DIR\n" |
Derrick Stolee | 3227565 | 2018-07-12 15:39:28 -0400 | [diff] [blame] | 26 | } >expect && |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 27 | test-tool read-midx $OBJECT_DIR >actual && |
Derrick Stolee | 4d80560 | 2018-07-12 15:39:23 -0400 | [diff] [blame] | 28 | test_cmp expect actual |
| 29 | } |
| 30 | |
brian m. carlson | 3c5e65c | 2019-12-21 19:49:26 +0000 | [diff] [blame^] | 31 | test_expect_success 'setup' ' |
| 32 | test_oid_init && |
| 33 | test_oid_cache <<-EOF |
| 34 | idxoff sha1:2999 |
| 35 | idxoff sha256:3739 |
| 36 | |
| 37 | packnameoff sha1:652 |
| 38 | packnameoff sha256:940 |
| 39 | |
| 40 | fanoutoff sha1:1 |
| 41 | fanoutoff sha256:3 |
| 42 | EOF |
| 43 | ' |
| 44 | |
Derrick Stolee | a340773 | 2018-07-12 15:39:21 -0400 | [diff] [blame] | 45 | test_expect_success 'write midx with no packs' ' |
Derrick Stolee | fc59e74 | 2018-07-12 15:39:22 -0400 | [diff] [blame] | 46 | test_when_finished rm -f pack/multi-pack-index && |
| 47 | git multi-pack-index --object-dir=. write && |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 48 | midx_read_expect 0 0 4 . |
Derrick Stolee | a340773 | 2018-07-12 15:39:21 -0400 | [diff] [blame] | 49 | ' |
| 50 | |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 51 | generate_objects () { |
| 52 | i=$1 |
| 53 | iii=$(printf '%03i' $i) |
| 54 | { |
| 55 | test-tool genrandom "bar" 200 && |
| 56 | test-tool genrandom "baz $iii" 50 |
| 57 | } >wide_delta_$iii && |
| 58 | { |
| 59 | test-tool genrandom "foo"$i 100 && |
| 60 | test-tool genrandom "foo"$(( $i + 1 )) 100 && |
| 61 | test-tool genrandom "foo"$(( $i + 2 )) 100 |
| 62 | } >deep_delta_$iii && |
| 63 | { |
| 64 | echo $iii && |
| 65 | test-tool genrandom "$iii" 8192 |
| 66 | } >file_$iii && |
| 67 | git update-index --add file_$iii deep_delta_$iii wide_delta_$iii |
| 68 | } |
| 69 | |
| 70 | commit_and_list_objects () { |
| 71 | { |
| 72 | echo 101 && |
| 73 | test-tool genrandom 100 8192; |
| 74 | } >file_101 && |
| 75 | git update-index --add file_101 && |
| 76 | tree=$(git write-tree) && |
| 77 | commit=$(git commit-tree $tree -p HEAD</dev/null) && |
| 78 | { |
| 79 | echo $tree && |
| 80 | git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/" |
| 81 | } >obj-list && |
| 82 | git reset --hard $commit |
| 83 | } |
| 84 | |
| 85 | test_expect_success 'create objects' ' |
| 86 | test_commit initial && |
| 87 | for i in $(test_seq 1 5) |
| 88 | do |
| 89 | generate_objects $i |
| 90 | done && |
| 91 | commit_and_list_objects |
| 92 | ' |
| 93 | |
| 94 | test_expect_success 'write midx with one v1 pack' ' |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 95 | pack=$(git pack-objects --index-version=1 $objdir/pack/test <obj-list) && |
| 96 | test_when_finished rm $objdir/pack/test-$pack.pack \ |
| 97 | $objdir/pack/test-$pack.idx $objdir/pack/multi-pack-index && |
| 98 | git multi-pack-index --object-dir=$objdir write && |
| 99 | midx_read_expect 1 18 4 $objdir |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 100 | ' |
| 101 | |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 102 | midx_git_two_modes () { |
Jeff King | b4a1439 | 2019-04-05 14:04:56 -0400 | [diff] [blame] | 103 | git -c core.multiPackIndex=false $1 >expect && |
| 104 | git -c core.multiPackIndex=true $1 >actual && |
Derrick Stolee | e9ab2ed | 2018-08-20 16:52:06 +0000 | [diff] [blame] | 105 | if [ "$2" = "sorted" ] |
| 106 | then |
Jeff King | b4a1439 | 2019-04-05 14:04:56 -0400 | [diff] [blame] | 107 | sort <expect >expect.sorted && |
| 108 | mv expect.sorted expect && |
| 109 | sort <actual >actual.sorted && |
| 110 | mv actual.sorted actual |
Derrick Stolee | e9ab2ed | 2018-08-20 16:52:06 +0000 | [diff] [blame] | 111 | fi && |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 112 | test_cmp expect actual |
| 113 | } |
| 114 | |
| 115 | compare_results_with_midx () { |
| 116 | MSG=$1 |
| 117 | test_expect_success "check normal git operations: $MSG" ' |
| 118 | midx_git_two_modes "rev-list --objects --all" && |
Derrick Stolee | e9ab2ed | 2018-08-20 16:52:06 +0000 | [diff] [blame] | 119 | midx_git_two_modes "log --raw" && |
| 120 | midx_git_two_modes "count-objects --verbose" && |
Jeff King | 5670ad9 | 2019-04-05 14:05:03 -0400 | [diff] [blame] | 121 | midx_git_two_modes "cat-file --batch-all-objects --batch-check" && |
| 122 | midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 123 | ' |
| 124 | } |
| 125 | |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 126 | test_expect_success 'write midx with one v2 pack' ' |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 127 | git pack-objects --index-version=2,0x40 $objdir/pack/test <obj-list && |
| 128 | git multi-pack-index --object-dir=$objdir write && |
| 129 | midx_read_expect 1 18 4 $objdir |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 130 | ' |
| 131 | |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 132 | compare_results_with_midx "one v2 pack" |
| 133 | |
Jeff King | fc78915 | 2019-04-05 14:06:22 -0400 | [diff] [blame] | 134 | test_expect_success 'corrupt idx not opened' ' |
| 135 | idx=$(test-tool read-midx $objdir | grep "\.idx\$") && |
| 136 | mv $objdir/pack/$idx backup-$idx && |
| 137 | test_when_finished "mv backup-\$idx \$objdir/pack/\$idx" && |
| 138 | |
| 139 | # This is the minimum size for a sha-1 based .idx; this lets |
| 140 | # us pass perfunctory tests, but anything that actually opens and reads |
| 141 | # the idx file will complain. |
| 142 | test_copy_bytes 1064 <backup-$idx >$objdir/pack/$idx && |
| 143 | |
| 144 | git -c core.multiPackIndex=true rev-list --objects --all 2>err && |
| 145 | test_must_be_empty err |
| 146 | ' |
| 147 | |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 148 | test_expect_success 'add more objects' ' |
| 149 | for i in $(test_seq 6 10) |
| 150 | do |
| 151 | generate_objects $i |
| 152 | done && |
| 153 | commit_and_list_objects |
| 154 | ' |
| 155 | |
| 156 | test_expect_success 'write midx with two packs' ' |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 157 | git pack-objects --index-version=1 $objdir/pack/test-2 <obj-list && |
| 158 | git multi-pack-index --object-dir=$objdir write && |
| 159 | midx_read_expect 2 34 4 $objdir |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 160 | ' |
| 161 | |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 162 | compare_results_with_midx "two packs" |
| 163 | |
William Baker | 680cba2 | 2019-10-21 18:40:03 +0000 | [diff] [blame] | 164 | test_expect_success 'write progress off for redirected stderr' ' |
| 165 | git multi-pack-index --object-dir=$objdir write 2>err && |
| 166 | test_line_count = 0 err |
| 167 | ' |
| 168 | |
| 169 | test_expect_success 'write force progress on for stderr' ' |
| 170 | git multi-pack-index --object-dir=$objdir --progress write 2>err && |
| 171 | test_file_not_empty err |
| 172 | ' |
| 173 | |
| 174 | test_expect_success 'write with the --no-progress option' ' |
| 175 | git multi-pack-index --object-dir=$objdir --no-progress write 2>err && |
| 176 | test_line_count = 0 err |
| 177 | ' |
| 178 | |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 179 | test_expect_success 'add more packs' ' |
| 180 | for j in $(test_seq 11 20) |
| 181 | do |
| 182 | generate_objects $j && |
| 183 | commit_and_list_objects && |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 184 | git pack-objects --index-version=2 $objdir/pack/test-pack <obj-list |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 185 | done |
| 186 | ' |
| 187 | |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 188 | compare_results_with_midx "mixed mode (two packs + extra)" |
| 189 | |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 190 | test_expect_success 'write midx with twelve packs' ' |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 191 | git multi-pack-index --object-dir=$objdir write && |
| 192 | midx_read_expect 12 74 4 $objdir |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 193 | ' |
| 194 | |
Derrick Stolee | c4d2522 | 2018-07-12 15:39:33 -0400 | [diff] [blame] | 195 | compare_results_with_midx "twelve packs" |
| 196 | |
Derrick Stolee | 56ee7ff | 2018-09-13 11:02:13 -0700 | [diff] [blame] | 197 | test_expect_success 'verify multi-pack-index success' ' |
| 198 | git multi-pack-index verify --object-dir=$objdir |
| 199 | ' |
| 200 | |
William Baker | 680cba2 | 2019-10-21 18:40:03 +0000 | [diff] [blame] | 201 | test_expect_success 'verify progress off for redirected stderr' ' |
| 202 | git multi-pack-index verify --object-dir=$objdir 2>err && |
| 203 | test_line_count = 0 err |
| 204 | ' |
| 205 | |
| 206 | test_expect_success 'verify force progress on for stderr' ' |
| 207 | git multi-pack-index verify --object-dir=$objdir --progress 2>err && |
| 208 | test_file_not_empty err |
| 209 | ' |
| 210 | |
| 211 | test_expect_success 'verify with the --no-progress option' ' |
| 212 | git multi-pack-index verify --object-dir=$objdir --no-progress 2>err && |
| 213 | test_line_count = 0 err |
| 214 | ' |
| 215 | |
Derrick Stolee | 53ad040 | 2018-09-13 11:02:15 -0700 | [diff] [blame] | 216 | # usage: corrupt_midx_and_verify <pos> <data> <objdir> <string> |
| 217 | corrupt_midx_and_verify() { |
| 218 | POS=$1 && |
| 219 | DATA="${2:-\0}" && |
| 220 | OBJDIR=$3 && |
| 221 | GREPSTR="$4" && |
Derrick Stolee | 66ec039 | 2018-09-13 11:02:27 -0700 | [diff] [blame] | 222 | COMMAND="$5" && |
| 223 | if test -z "$COMMAND" |
| 224 | then |
| 225 | COMMAND="git multi-pack-index verify --object-dir=$OBJDIR" |
| 226 | fi && |
Derrick Stolee | 53ad040 | 2018-09-13 11:02:15 -0700 | [diff] [blame] | 227 | FILE=$OBJDIR/pack/multi-pack-index && |
| 228 | chmod a+w $FILE && |
| 229 | test_when_finished mv midx-backup $FILE && |
| 230 | cp $FILE midx-backup && |
| 231 | printf "$DATA" | dd of="$FILE" bs=1 seek="$POS" conv=notrunc && |
Derrick Stolee | 66ec039 | 2018-09-13 11:02:27 -0700 | [diff] [blame] | 232 | test_must_fail $COMMAND 2>test_err && |
Derrick Stolee | 53ad040 | 2018-09-13 11:02:15 -0700 | [diff] [blame] | 233 | grep -v "^+" test_err >err && |
| 234 | test_i18ngrep "$GREPSTR" err |
| 235 | } |
| 236 | |
| 237 | test_expect_success 'verify bad signature' ' |
| 238 | corrupt_midx_and_verify 0 "\00" $objdir \ |
| 239 | "multi-pack-index signature" |
| 240 | ' |
| 241 | |
brian m. carlson | 3c5e65c | 2019-12-21 19:49:26 +0000 | [diff] [blame^] | 242 | HASH_LEN=$(test_oid rawsz) |
Derrick Stolee | cc6af73 | 2018-09-13 11:02:25 -0700 | [diff] [blame] | 243 | NUM_OBJECTS=74 |
Derrick Stolee | 53ad040 | 2018-09-13 11:02:15 -0700 | [diff] [blame] | 244 | MIDX_BYTE_VERSION=4 |
| 245 | MIDX_BYTE_OID_VERSION=5 |
| 246 | MIDX_BYTE_CHUNK_COUNT=6 |
Derrick Stolee | d3f8e21 | 2018-09-13 11:02:16 -0700 | [diff] [blame] | 247 | MIDX_HEADER_SIZE=12 |
| 248 | MIDX_BYTE_CHUNK_ID=$MIDX_HEADER_SIZE |
| 249 | MIDX_BYTE_CHUNK_OFFSET=$(($MIDX_HEADER_SIZE + 4)) |
Derrick Stolee | 8e72a3c | 2018-09-13 11:02:18 -0700 | [diff] [blame] | 250 | MIDX_NUM_CHUNKS=5 |
| 251 | MIDX_CHUNK_LOOKUP_WIDTH=12 |
| 252 | MIDX_OFFSET_PACKNAMES=$(($MIDX_HEADER_SIZE + \ |
| 253 | $MIDX_NUM_CHUNKS * $MIDX_CHUNK_LOOKUP_WIDTH)) |
| 254 | MIDX_BYTE_PACKNAME_ORDER=$(($MIDX_OFFSET_PACKNAMES + 2)) |
brian m. carlson | 3c5e65c | 2019-12-21 19:49:26 +0000 | [diff] [blame^] | 255 | MIDX_OFFSET_OID_FANOUT=$(($MIDX_OFFSET_PACKNAMES + $(test_oid packnameoff))) |
Derrick Stolee | 2f23d3f | 2018-09-13 11:02:20 -0700 | [diff] [blame] | 256 | MIDX_OID_FANOUT_WIDTH=4 |
brian m. carlson | 3c5e65c | 2019-12-21 19:49:26 +0000 | [diff] [blame^] | 257 | MIDX_BYTE_OID_FANOUT_ORDER=$((MIDX_OFFSET_OID_FANOUT + 250 * $MIDX_OID_FANOUT_WIDTH + $(test_oid fanoutoff))) |
Derrick Stolee | 55c5648 | 2018-09-13 11:02:22 -0700 | [diff] [blame] | 258 | MIDX_OFFSET_OID_LOOKUP=$(($MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH)) |
| 259 | MIDX_BYTE_OID_LOOKUP=$(($MIDX_OFFSET_OID_LOOKUP + 16 * $HASH_LEN)) |
Derrick Stolee | cc6af73 | 2018-09-13 11:02:25 -0700 | [diff] [blame] | 260 | MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN)) |
| 261 | MIDX_OFFSET_WIDTH=8 |
| 262 | MIDX_BYTE_PACK_INT_ID=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 2)) |
| 263 | MIDX_BYTE_OFFSET=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 6)) |
Derrick Stolee | 53ad040 | 2018-09-13 11:02:15 -0700 | [diff] [blame] | 264 | |
| 265 | test_expect_success 'verify bad version' ' |
| 266 | corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ |
| 267 | "multi-pack-index version" |
| 268 | ' |
| 269 | |
| 270 | test_expect_success 'verify bad OID version' ' |
| 271 | corrupt_midx_and_verify $MIDX_BYTE_OID_VERSION "\02" $objdir \ |
| 272 | "hash version" |
| 273 | ' |
| 274 | |
| 275 | test_expect_success 'verify truncated chunk count' ' |
| 276 | corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\01" $objdir \ |
| 277 | "missing required" |
| 278 | ' |
| 279 | |
| 280 | test_expect_success 'verify extended chunk count' ' |
| 281 | corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\07" $objdir \ |
| 282 | "terminating multi-pack-index chunk id appears earlier than expected" |
| 283 | ' |
| 284 | |
Derrick Stolee | d3f8e21 | 2018-09-13 11:02:16 -0700 | [diff] [blame] | 285 | test_expect_success 'verify missing required chunk' ' |
| 286 | corrupt_midx_and_verify $MIDX_BYTE_CHUNK_ID "\01" $objdir \ |
| 287 | "missing required" |
| 288 | ' |
| 289 | |
| 290 | test_expect_success 'verify invalid chunk offset' ' |
| 291 | corrupt_midx_and_verify $MIDX_BYTE_CHUNK_OFFSET "\01" $objdir \ |
| 292 | "invalid chunk offset (too large)" |
| 293 | ' |
| 294 | |
Derrick Stolee | 8e72a3c | 2018-09-13 11:02:18 -0700 | [diff] [blame] | 295 | test_expect_success 'verify packnames out of order' ' |
| 296 | corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "z" $objdir \ |
| 297 | "pack names out of order" |
| 298 | ' |
| 299 | |
Derrick Stolee | d4bf1d8 | 2018-09-13 11:02:19 -0700 | [diff] [blame] | 300 | test_expect_success 'verify packnames out of order' ' |
| 301 | corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "a" $objdir \ |
| 302 | "failed to load pack" |
| 303 | ' |
| 304 | |
Derrick Stolee | 2f23d3f | 2018-09-13 11:02:20 -0700 | [diff] [blame] | 305 | test_expect_success 'verify oid fanout out of order' ' |
| 306 | corrupt_midx_and_verify $MIDX_BYTE_OID_FANOUT_ORDER "\01" $objdir \ |
| 307 | "oid fanout out of order" |
| 308 | ' |
| 309 | |
Derrick Stolee | 55c5648 | 2018-09-13 11:02:22 -0700 | [diff] [blame] | 310 | test_expect_success 'verify oid lookup out of order' ' |
| 311 | corrupt_midx_and_verify $MIDX_BYTE_OID_LOOKUP "\00" $objdir \ |
| 312 | "oid lookup out of order" |
| 313 | ' |
| 314 | |
Derrick Stolee | cc6af73 | 2018-09-13 11:02:25 -0700 | [diff] [blame] | 315 | test_expect_success 'verify incorrect pack-int-id' ' |
| 316 | corrupt_midx_and_verify $MIDX_BYTE_PACK_INT_ID "\07" $objdir \ |
| 317 | "bad pack-int-id" |
| 318 | ' |
| 319 | |
| 320 | test_expect_success 'verify incorrect offset' ' |
brian m. carlson | 235d3cd | 2019-12-21 19:49:25 +0000 | [diff] [blame] | 321 | corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\377" $objdir \ |
Derrick Stolee | cc6af73 | 2018-09-13 11:02:25 -0700 | [diff] [blame] | 322 | "incorrect object offset" |
| 323 | ' |
| 324 | |
Derrick Stolee | 66ec039 | 2018-09-13 11:02:27 -0700 | [diff] [blame] | 325 | test_expect_success 'git-fsck incorrect offset' ' |
brian m. carlson | 235d3cd | 2019-12-21 19:49:25 +0000 | [diff] [blame] | 326 | corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\377" $objdir \ |
Derrick Stolee | 66ec039 | 2018-09-13 11:02:27 -0700 | [diff] [blame] | 327 | "incorrect object offset" \ |
| 328 | "git -c core.multipackindex=true fsck" |
| 329 | ' |
| 330 | |
William Baker | 680cba2 | 2019-10-21 18:40:03 +0000 | [diff] [blame] | 331 | test_expect_success 'repack progress off for redirected stderr' ' |
| 332 | git multi-pack-index --object-dir=$objdir repack 2>err && |
| 333 | test_line_count = 0 err |
| 334 | ' |
| 335 | |
| 336 | test_expect_success 'repack force progress on for stderr' ' |
| 337 | git multi-pack-index --object-dir=$objdir --progress repack 2>err && |
| 338 | test_file_not_empty err |
| 339 | ' |
| 340 | |
| 341 | test_expect_success 'repack with the --no-progress option' ' |
| 342 | git multi-pack-index --object-dir=$objdir --no-progress repack 2>err && |
| 343 | test_line_count = 0 err |
| 344 | ' |
| 345 | |
Derrick Stolee | 525e18c | 2018-07-12 15:39:40 -0400 | [diff] [blame] | 346 | test_expect_success 'repack removes multi-pack-index' ' |
| 347 | test_path_is_file $objdir/pack/multi-pack-index && |
Derrick Stolee | 0465a50 | 2018-10-12 10:34:20 -0700 | [diff] [blame] | 348 | GIT_TEST_MULTI_PACK_INDEX=0 git repack -adf && |
Derrick Stolee | 525e18c | 2018-07-12 15:39:40 -0400 | [diff] [blame] | 349 | test_path_is_missing $objdir/pack/multi-pack-index |
| 350 | ' |
| 351 | |
| 352 | compare_results_with_midx "after repack" |
| 353 | |
Derrick Stolee | e9ab2ed | 2018-08-20 16:52:06 +0000 | [diff] [blame] | 354 | test_expect_success 'multi-pack-index and pack-bitmap' ' |
| 355 | git -c repack.writeBitmaps=true repack -ad && |
| 356 | git multi-pack-index write && |
| 357 | git rev-list --test-bitmap HEAD |
| 358 | ' |
| 359 | |
Derrick Stolee | 29e2016 | 2018-08-20 16:52:00 +0000 | [diff] [blame] | 360 | test_expect_success 'multi-pack-index and alternates' ' |
| 361 | git init --bare alt.git && |
| 362 | echo $(pwd)/alt.git/objects >.git/objects/info/alternates && |
| 363 | echo content1 >file1 && |
| 364 | altblob=$(GIT_DIR=alt.git git hash-object -w file1) && |
| 365 | git cat-file blob $altblob && |
| 366 | git rev-list --all |
| 367 | ' |
| 368 | |
| 369 | compare_results_with_midx "with alternate (local midx)" |
| 370 | |
| 371 | test_expect_success 'multi-pack-index in an alternate' ' |
Derrick Stolee | 6a22d52 | 2018-08-20 16:52:08 +0000 | [diff] [blame] | 372 | mv .git/objects/pack/* alt.git/objects/pack && |
| 373 | test_commit add_local_objects && |
| 374 | git repack --local && |
| 375 | git multi-pack-index write && |
| 376 | midx_read_expect 1 3 4 $objdir && |
| 377 | git reset --hard HEAD~1 && |
| 378 | rm -f .git/objects/pack/* |
Derrick Stolee | 29e2016 | 2018-08-20 16:52:00 +0000 | [diff] [blame] | 379 | ' |
| 380 | |
| 381 | compare_results_with_midx "with alternate (remote midx)" |
| 382 | |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 383 | # usage: corrupt_data <file> <pos> [<data>] |
| 384 | corrupt_data () { |
| 385 | file=$1 |
| 386 | pos=$2 |
| 387 | data="${3:-\0}" |
| 388 | printf "$data" | dd of="$file" bs=1 seek="$pos" conv=notrunc |
| 389 | } |
| 390 | |
| 391 | # Force 64-bit offsets by manipulating the idx file. |
| 392 | # This makes the IDX file _incorrect_ so be careful to clean up after! |
| 393 | test_expect_success 'force some 64-bit offsets with pack-objects' ' |
| 394 | mkdir objects64 && |
| 395 | mkdir objects64/pack && |
| 396 | for i in $(test_seq 1 11) |
| 397 | do |
| 398 | generate_objects 11 |
| 399 | done && |
| 400 | commit_and_list_objects && |
| 401 | pack64=$(git pack-objects --index-version=2,0x40 objects64/pack/test-64 <obj-list) && |
| 402 | idx64=objects64/pack/test-64-$pack64.idx && |
| 403 | chmod u+w $idx64 && |
brian m. carlson | 3c5e65c | 2019-12-21 19:49:26 +0000 | [diff] [blame^] | 404 | corrupt_data $idx64 $(test_oid idxoff) "\02" && |
Derrick Stolee | 662148c | 2018-07-12 15:39:32 -0400 | [diff] [blame] | 405 | midx64=$(git multi-pack-index --object-dir=objects64 write) && |
| 406 | midx_read_expect 1 63 5 objects64 " large-offsets" |
Derrick Stolee | 2c38133 | 2018-07-12 15:39:24 -0400 | [diff] [blame] | 407 | ' |
| 408 | |
Derrick Stolee | 56ee7ff | 2018-09-13 11:02:13 -0700 | [diff] [blame] | 409 | test_expect_success 'verify multi-pack-index with 64-bit offsets' ' |
| 410 | git multi-pack-index verify --object-dir=objects64 |
| 411 | ' |
| 412 | |
Derrick Stolee | cc6af73 | 2018-09-13 11:02:25 -0700 | [diff] [blame] | 413 | NUM_OBJECTS=63 |
| 414 | MIDX_OFFSET_OID_FANOUT=$((MIDX_OFFSET_PACKNAMES + 54)) |
| 415 | MIDX_OFFSET_OID_LOOKUP=$((MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH)) |
| 416 | MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN)) |
| 417 | MIDX_OFFSET_LARGE_OFFSETS=$(($MIDX_OFFSET_OBJECT_OFFSETS + $NUM_OBJECTS * $MIDX_OFFSET_WIDTH)) |
| 418 | MIDX_BYTE_LARGE_OFFSET=$(($MIDX_OFFSET_LARGE_OFFSETS + 3)) |
| 419 | |
| 420 | test_expect_success 'verify incorrect 64-bit offset' ' |
| 421 | corrupt_midx_and_verify $MIDX_BYTE_LARGE_OFFSET "\07" objects64 \ |
| 422 | "incorrect object offset" |
| 423 | ' |
| 424 | |
Derrick Stolee | cff9711 | 2019-06-10 16:35:23 -0700 | [diff] [blame] | 425 | test_expect_success 'setup expire tests' ' |
| 426 | mkdir dup && |
| 427 | ( |
| 428 | cd dup && |
| 429 | git init && |
| 430 | test-tool genrandom "data" 4096 >large_file.txt && |
| 431 | git update-index --add large_file.txt && |
| 432 | for i in $(test_seq 1 20) |
| 433 | do |
| 434 | test_commit $i |
| 435 | done && |
| 436 | git branch A HEAD && |
| 437 | git branch B HEAD~8 && |
| 438 | git branch C HEAD~13 && |
| 439 | git branch D HEAD~16 && |
| 440 | git branch E HEAD~18 && |
| 441 | git pack-objects --revs .git/objects/pack/pack-A <<-EOF && |
| 442 | refs/heads/A |
| 443 | ^refs/heads/B |
| 444 | EOF |
| 445 | git pack-objects --revs .git/objects/pack/pack-B <<-EOF && |
| 446 | refs/heads/B |
| 447 | ^refs/heads/C |
| 448 | EOF |
| 449 | git pack-objects --revs .git/objects/pack/pack-C <<-EOF && |
| 450 | refs/heads/C |
| 451 | ^refs/heads/D |
| 452 | EOF |
| 453 | git pack-objects --revs .git/objects/pack/pack-D <<-EOF && |
| 454 | refs/heads/D |
| 455 | ^refs/heads/E |
| 456 | EOF |
| 457 | git pack-objects --revs .git/objects/pack/pack-E <<-EOF && |
| 458 | refs/heads/E |
| 459 | EOF |
Derrick Stolee | 2af890b | 2019-06-10 16:35:26 -0700 | [diff] [blame] | 460 | git multi-pack-index write && |
| 461 | cp -r .git/objects/pack .git/objects/pack-backup |
Derrick Stolee | cff9711 | 2019-06-10 16:35:23 -0700 | [diff] [blame] | 462 | ) |
| 463 | ' |
| 464 | |
| 465 | test_expect_success 'expire does not remove any packs' ' |
| 466 | ( |
| 467 | cd dup && |
| 468 | ls .git/objects/pack >expect && |
| 469 | git multi-pack-index expire && |
| 470 | ls .git/objects/pack >actual && |
| 471 | test_cmp expect actual |
| 472 | ) |
| 473 | ' |
| 474 | |
William Baker | 680cba2 | 2019-10-21 18:40:03 +0000 | [diff] [blame] | 475 | test_expect_success 'expire progress off for redirected stderr' ' |
| 476 | ( |
| 477 | cd dup && |
| 478 | git multi-pack-index expire 2>err && |
| 479 | test_line_count = 0 err |
| 480 | ) |
| 481 | ' |
| 482 | |
| 483 | test_expect_success 'expire force progress on for stderr' ' |
| 484 | ( |
| 485 | cd dup && |
| 486 | git multi-pack-index --progress expire 2>err && |
| 487 | test_file_not_empty err |
| 488 | ) |
| 489 | ' |
| 490 | |
| 491 | test_expect_success 'expire with the --no-progress option' ' |
| 492 | ( |
| 493 | cd dup && |
| 494 | git multi-pack-index --no-progress expire 2>err && |
| 495 | test_line_count = 0 err |
| 496 | ) |
| 497 | ' |
| 498 | |
Derrick Stolee | 19575c7 | 2019-06-10 16:35:25 -0700 | [diff] [blame] | 499 | test_expect_success 'expire removes unreferenced packs' ' |
| 500 | ( |
| 501 | cd dup && |
| 502 | git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && |
| 503 | refs/heads/A |
| 504 | ^refs/heads/C |
| 505 | EOF |
| 506 | git multi-pack-index write && |
| 507 | ls .git/objects/pack | grep -v -e pack-[AB] >expect && |
| 508 | git multi-pack-index expire && |
| 509 | ls .git/objects/pack >actual && |
| 510 | test_cmp expect actual && |
| 511 | ls .git/objects/pack/ | grep idx >expect-idx && |
| 512 | test-tool read-midx .git/objects | grep idx >actual-midx && |
| 513 | test_cmp expect-idx actual-midx && |
| 514 | git multi-pack-index verify && |
| 515 | git fsck |
| 516 | ) |
| 517 | ' |
| 518 | |
Derrick Stolee | 2af890b | 2019-06-10 16:35:26 -0700 | [diff] [blame] | 519 | test_expect_success 'repack with minimum size does not alter existing packs' ' |
| 520 | ( |
| 521 | cd dup && |
| 522 | rm -rf .git/objects/pack && |
| 523 | mv .git/objects/pack-backup .git/objects/pack && |
| 524 | touch -m -t 201901010000 .git/objects/pack/pack-D* && |
| 525 | touch -m -t 201901010001 .git/objects/pack/pack-C* && |
| 526 | touch -m -t 201901010002 .git/objects/pack/pack-B* && |
| 527 | touch -m -t 201901010003 .git/objects/pack/pack-A* && |
| 528 | ls .git/objects/pack >expect && |
Derrick Stolee | 3612c23 | 2019-07-01 06:16:19 -0700 | [diff] [blame] | 529 | MINSIZE=$(test-tool path-utils file-size .git/objects/pack/*pack | sort -n | head -n 1) && |
Derrick Stolee | 2af890b | 2019-06-10 16:35:26 -0700 | [diff] [blame] | 530 | git multi-pack-index repack --batch-size=$MINSIZE && |
| 531 | ls .git/objects/pack >actual && |
| 532 | test_cmp expect actual |
| 533 | ) |
| 534 | ' |
| 535 | |
Derrick Stolee | ce1e4a1 | 2019-06-10 16:35:27 -0700 | [diff] [blame] | 536 | test_expect_success 'repack creates a new pack' ' |
| 537 | ( |
| 538 | cd dup && |
| 539 | ls .git/objects/pack/*idx >idx-list && |
| 540 | test_line_count = 5 idx-list && |
Derrick Stolee | 3612c23 | 2019-07-01 06:16:19 -0700 | [diff] [blame] | 541 | THIRD_SMALLEST_SIZE=$(test-tool path-utils file-size .git/objects/pack/*pack | sort -n | head -n 3 | tail -n 1) && |
Derrick Stolee | ce1e4a1 | 2019-06-10 16:35:27 -0700 | [diff] [blame] | 542 | BATCH_SIZE=$(($THIRD_SMALLEST_SIZE + 1)) && |
| 543 | git multi-pack-index repack --batch-size=$BATCH_SIZE && |
| 544 | ls .git/objects/pack/*idx >idx-list && |
| 545 | test_line_count = 6 idx-list && |
| 546 | test-tool read-midx .git/objects | grep idx >midx-list && |
| 547 | test_line_count = 6 midx-list |
| 548 | ) |
| 549 | ' |
| 550 | |
| 551 | test_expect_success 'expire removes repacked packs' ' |
| 552 | ( |
| 553 | cd dup && |
| 554 | ls -al .git/objects/pack/*pack && |
| 555 | ls -S .git/objects/pack/*pack | head -n 4 >expect && |
| 556 | git multi-pack-index expire && |
| 557 | ls -S .git/objects/pack/*pack >actual && |
| 558 | test_cmp expect actual && |
| 559 | test-tool read-midx .git/objects | grep idx >midx-list && |
| 560 | test_line_count = 4 midx-list |
| 561 | ) |
| 562 | ' |
| 563 | |
Derrick Stolee | d274331 | 2019-06-10 16:35:27 -0700 | [diff] [blame] | 564 | test_expect_success 'expire works when adding new packs' ' |
| 565 | ( |
| 566 | cd dup && |
| 567 | git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && |
| 568 | refs/heads/A |
| 569 | ^refs/heads/B |
| 570 | EOF |
| 571 | git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && |
| 572 | refs/heads/B |
| 573 | ^refs/heads/C |
| 574 | EOF |
| 575 | git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && |
| 576 | refs/heads/C |
| 577 | ^refs/heads/D |
| 578 | EOF |
| 579 | git multi-pack-index write && |
| 580 | git pack-objects --revs .git/objects/pack/a-pack <<-EOF && |
| 581 | refs/heads/D |
| 582 | ^refs/heads/E |
| 583 | EOF |
| 584 | git multi-pack-index write && |
| 585 | git pack-objects --revs .git/objects/pack/z-pack <<-EOF && |
| 586 | refs/heads/E |
| 587 | EOF |
| 588 | git multi-pack-index expire && |
| 589 | ls .git/objects/pack/ | grep idx >expect && |
| 590 | test-tool read-midx .git/objects | grep idx >actual && |
| 591 | test_cmp expect actual && |
| 592 | git multi-pack-index verify |
| 593 | ) |
| 594 | ' |
| 595 | |
Derrick Stolee | 10bfa3f | 2019-06-10 16:35:28 -0700 | [diff] [blame] | 596 | test_expect_success 'expire respects .keep files' ' |
| 597 | ( |
| 598 | cd dup && |
| 599 | git pack-objects --revs .git/objects/pack/pack-all <<-EOF && |
| 600 | refs/heads/A |
| 601 | EOF |
| 602 | git multi-pack-index write && |
| 603 | PACKA=$(ls .git/objects/pack/a-pack*\.pack | sed s/\.pack\$//) && |
| 604 | touch $PACKA.keep && |
| 605 | git multi-pack-index expire && |
| 606 | ls -S .git/objects/pack/a-pack* | grep $PACKA >a-pack-files && |
| 607 | test_line_count = 3 a-pack-files && |
| 608 | test-tool read-midx .git/objects | grep idx >midx-list && |
| 609 | test_line_count = 2 midx-list |
| 610 | ) |
| 611 | ' |
| 612 | |
Derrick Stolee | b526d8c | 2019-06-10 16:35:29 -0700 | [diff] [blame] | 613 | test_expect_success 'repack --batch-size=0 repacks everything' ' |
| 614 | ( |
| 615 | cd dup && |
| 616 | rm .git/objects/pack/*.keep && |
| 617 | ls .git/objects/pack/*idx >idx-list && |
| 618 | test_line_count = 2 idx-list && |
| 619 | git multi-pack-index repack --batch-size=0 && |
| 620 | ls .git/objects/pack/*idx >idx-list && |
| 621 | test_line_count = 3 idx-list && |
| 622 | test-tool read-midx .git/objects | grep idx >midx-list && |
| 623 | test_line_count = 3 midx-list && |
| 624 | git multi-pack-index expire && |
| 625 | ls -al .git/objects/pack/*idx >idx-list && |
| 626 | test_line_count = 1 idx-list && |
| 627 | git multi-pack-index repack --batch-size=0 && |
| 628 | ls -al .git/objects/pack/*idx >new-idx-list && |
| 629 | test_cmp idx-list new-idx-list |
| 630 | ) |
| 631 | ' |
Derrick Stolee | 10bfa3f | 2019-06-10 16:35:28 -0700 | [diff] [blame] | 632 | |
Derrick Stolee | a340773 | 2018-07-12 15:39:21 -0400 | [diff] [blame] | 633 | test_done |