Bigfile: teach "git add" to send a large file straight to a pack

When adding a new content to the repository, we have always slurped
the blob in its entirety in-core first, and computed the object name
and compressed it into a loose object file.  Handling large binary
files (e.g.  video and audio asset for games) has been problematic
because of this design.

At the middle level of "git add" callchain is an internal API
index_fd() that takes an open file descriptor to read from the
working tree file being added with its size. Teach it to call out to
fast-import when adding a large blob.

The write-out codepath in entry.c::write_entry() should be taught to
stream, instead of reading everything in core. This should not be so
hard to implement, especially if we limit ourselves only to loose
object files and non-delta representation in packfiles.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
2 files changed
tree: 6672594f53e8688c10ccff922d8d52c16385ed36
  1. block-sha1/
  2. builtin/
  3. compat/
  4. contrib/
  5. Documentation/
  6. git-gui/
  7. git_remote_helpers/
  8. gitk-git/
  9. gitweb/
  10. perl/
  11. po/
  12. ppc/
  13. t/
  14. templates/
  15. vcs-svn/
  16. xdiff/
  17. .gitattributes
  18. .gitignore
  19. .mailmap
  20. abspath.c
  21. aclocal.m4
  22. advice.c
  23. advice.h
  24. alias.c
  25. alloc.c
  26. archive-tar.c
  27. archive-zip.c
  28. archive.c
  29. archive.h
  30. attr.c
  31. attr.h
  32. base85.c
  33. bisect.c
  34. bisect.h
  35. blob.c
  36. blob.h
  37. branch.c
  38. branch.h
  39. builtin.h
  40. bundle.c
  41. bundle.h
  42. cache-tree.c
  43. cache-tree.h
  44. cache.h
  45. check-builtins.sh
  46. check-racy.c
  47. check_bindir
  48. color.c
  49. color.h
  50. combine-diff.c
  51. command-list.txt
  52. commit.c
  53. commit.h
  54. config.c
  55. config.mak.in
  56. configure.ac
  57. connect.c
  58. convert.c
  59. copy.c
  60. COPYING
  61. csum-file.c
  62. csum-file.h
  63. ctype.c
  64. daemon.c
  65. date.c
  66. decorate.c
  67. decorate.h
  68. delta.h
  69. diff-delta.c
  70. diff-lib.c
  71. diff-no-index.c
  72. diff.c
  73. diff.h
  74. diffcore-break.c
  75. diffcore-delta.c
  76. diffcore-order.c
  77. diffcore-pickaxe.c
  78. diffcore-rename.c
  79. diffcore.h
  80. dir.c
  81. dir.h
  82. editor.c
  83. entry.c
  84. environment.c
  85. exec_cmd.c
  86. exec_cmd.h
  87. fast-import.c
  88. fetch-pack.h
  89. fixup-builtins
  90. fsck.c
  91. fsck.h
  92. generate-cmdlist.sh
  93. gettext.c
  94. gettext.h
  95. git-add--interactive.perl
  96. git-am.sh
  97. git-archimport.perl
  98. git-bisect.sh
  99. git-compat-util.h
  100. git-cvsexportcommit.perl
  101. git-cvsimport.perl
  102. git-cvsserver.perl
  103. git-difftool--helper.sh
  104. git-difftool.perl
  105. git-filter-branch.sh
  106. git-instaweb.sh
  107. git-lost-found.sh
  108. git-merge-octopus.sh
  109. git-merge-one-file.sh
  110. git-merge-resolve.sh
  111. git-mergetool--lib.sh
  112. git-mergetool.sh
  113. git-parse-remote.sh
  114. git-pull.sh
  115. git-quiltimport.sh
  116. git-rebase--interactive.sh
  117. git-rebase.sh
  118. git-relink.perl
  119. git-remote-testgit.py
  120. git-repack.sh
  121. git-request-pull.sh
  122. git-send-email.perl
  123. git-sh-setup.sh
  124. git-stash.sh
  125. git-submodule.sh
  126. git-svn.perl
  127. GIT-VERSION-GEN
  128. git-web--browse.sh
  129. git.c
  130. git.spec.in
  131. graph.c
  132. graph.h
  133. grep.c
  134. grep.h
  135. hash.c
  136. hash.h
  137. help.c
  138. help.h
  139. hex.c
  140. http-backend.c
  141. http-fetch.c
  142. http-push.c
  143. http-walker.c
  144. http.c
  145. http.h
  146. ident.c
  147. imap-send.c
  148. INSTALL
  149. levenshtein.c
  150. levenshtein.h
  151. list-objects.c
  152. list-objects.h
  153. ll-merge.c
  154. ll-merge.h
  155. lockfile.c
  156. log-tree.c
  157. log-tree.h
  158. mailmap.c
  159. mailmap.h
  160. Makefile
  161. match-trees.c
  162. merge-file.c
  163. merge-recursive.c
  164. merge-recursive.h
  165. name-hash.c
  166. notes-cache.c
  167. notes-cache.h
  168. notes-merge.c
  169. notes-merge.h
  170. notes.c
  171. notes.h
  172. object.c
  173. object.h
  174. pack-check.c
  175. pack-refs.c
  176. pack-refs.h
  177. pack-revindex.c
  178. pack-revindex.h
  179. pack-write.c
  180. pack.h
  181. pager.c
  182. parse-options.c
  183. parse-options.h
  184. patch-delta.c
  185. patch-ids.c
  186. patch-ids.h
  187. path.c
  188. pkt-line.c
  189. pkt-line.h
  190. preload-index.c
  191. pretty.c
  192. progress.c
  193. progress.h
  194. quote.c
  195. quote.h
  196. reachable.c
  197. reachable.h
  198. read-cache.c
  199. README
  200. reflog-walk.c
  201. reflog-walk.h
  202. refs.c
  203. refs.h
  204. remote-curl.c
  205. remote.c
  206. remote.h
  207. replace_object.c
  208. rerere.c
  209. rerere.h
  210. resolve-undo.c
  211. resolve-undo.h
  212. revision.c
  213. revision.h
  214. run-command.c
  215. run-command.h
  216. send-pack.h
  217. server-info.c
  218. setup.c
  219. sha1-lookup.c
  220. sha1-lookup.h
  221. sha1_file.c
  222. sha1_name.c
  223. shallow.c
  224. shell.c
  225. shortlog.h
  226. show-index.c
  227. sideband.c
  228. sideband.h
  229. sigchain.c
  230. sigchain.h
  231. strbuf.c
  232. strbuf.h
  233. string-list.c
  234. string-list.h
  235. submodule.c
  236. submodule.h
  237. symlinks.c
  238. tag.c
  239. tag.h
  240. tar.h
  241. test-chmtime.c
  242. test-ctype.c
  243. test-date.c
  244. test-delta.c
  245. test-dump-cache-tree.c
  246. test-genrandom.c
  247. test-index-version.c
  248. test-line-buffer.c
  249. test-match-trees.c
  250. test-mktemp.c
  251. test-obj-pool.c
  252. test-parse-options.c
  253. test-path-utils.c
  254. test-run-command.c
  255. test-sha1.c
  256. test-sha1.sh
  257. test-sigchain.c
  258. test-string-pool.c
  259. test-subprocess.c
  260. test-svn-fe.c
  261. test-treap.c
  262. thread-utils.c
  263. thread-utils.h
  264. trace.c
  265. transport-helper.c
  266. transport.c
  267. transport.h
  268. tree-diff.c
  269. tree-walk.c
  270. tree-walk.h
  271. tree.c
  272. tree.h
  273. unimplemented.sh
  274. unpack-trees.c
  275. unpack-trees.h
  276. upload-pack.c
  277. url.c
  278. url.h
  279. usage.c
  280. userdiff.c
  281. userdiff.h
  282. utf8.c
  283. utf8.h
  284. walker.c
  285. walker.h
  286. wrap-for-bin.sh
  287. wrapper.c
  288. write_or_die.c
  289. ws.c
  290. wt-status.c
  291. wt-status.h
  292. xdiff-interface.c
  293. xdiff-interface.h
  294. zlib.c