2 * Copyright (c) 2011, Google Inc.
15 typedef int (*open_istream_fn
)(struct git_istream
*,
17 const unsigned char *,
19 typedef int (*close_istream_fn
)(struct git_istream
*);
20 typedef ssize_t (*read_istream_fn
)(struct git_istream
*, char *, size_t);
23 close_istream_fn close
;
27 #define open_method_decl(name) \
28 int open_istream_ ##name \
29 (struct git_istream *st, struct object_info *oi, \
30 const unsigned char *sha1, \
31 enum object_type *type)
33 #define close_method_decl(name) \
34 int close_istream_ ##name \
35 (struct git_istream *st)
37 #define read_method_decl(name) \
38 ssize_t read_istream_ ##name \
39 (struct git_istream *st, char *buf, size_t sz)
41 /* forward declaration */
42 static open_method_decl(incore
);
43 static open_method_decl(loose
);
44 static open_method_decl(pack_non_delta
);
45 static struct git_istream
*attach_stream_filter(struct git_istream
*st
,
46 struct stream_filter
*filter
);
49 static open_istream_fn open_istream_tbl
[] = {
52 open_istream_pack_non_delta
,
55 #define FILTER_BUFFER (1024*16)
57 struct filtered_istream
{
58 struct git_istream
*upstream
;
59 struct stream_filter
*filter
;
60 char ibuf
[FILTER_BUFFER
];
61 char obuf
[FILTER_BUFFER
];
68 const struct stream_vtbl
*vtbl
;
69 unsigned long size
; /* inflated size of full object */
71 enum { z_unused
, z_used
, z_done
, z_error
} z_state
;
75 char *buf
; /* from read_object() */
76 unsigned long read_ptr
;
81 unsigned long mapsize
;
88 struct packed_git
*pack
;
92 struct filtered_istream filtered
;
96 int close_istream(struct git_istream
*st
)
98 int r
= st
->vtbl
->close(st
);
103 ssize_t
read_istream(struct git_istream
*st
, void *buf
, size_t sz
)
105 return st
->vtbl
->read(st
, buf
, sz
);
108 static enum input_source
istream_source(const unsigned char *sha1
,
109 enum object_type
*type
,
110 struct object_info
*oi
)
114 struct object_id oid
;
116 hashcpy(oid
.hash
, sha1
);
120 status
= oid_object_info_extended(&oid
, oi
, 0);
124 switch (oi
->whence
) {
128 if (!oi
->u
.packed
.is_delta
&& big_file_threshold
< size
)
129 return pack_non_delta
;
136 struct git_istream
*open_istream(const struct object_id
*oid
,
137 enum object_type
*type
,
139 struct stream_filter
*filter
)
141 struct git_istream
*st
;
142 struct object_info oi
= OBJECT_INFO_INIT
;
143 const unsigned char *real
= lookup_replace_object(oid
->hash
);
144 enum input_source src
= istream_source(real
, type
, &oi
);
149 st
= xmalloc(sizeof(*st
));
150 if (open_istream_tbl
[src
](st
, &oi
, real
, type
)) {
151 if (open_istream_incore(st
, &oi
, real
, type
)) {
157 /* Add "&& !is_null_stream_filter(filter)" for performance */
158 struct git_istream
*nst
= attach_stream_filter(st
, filter
);
171 /*****************************************************************
175 *****************************************************************/
177 static void close_deflated_stream(struct git_istream
*st
)
179 if (st
->z_state
== z_used
)
180 git_inflate_end(&st
->z
);
184 /*****************************************************************
188 *****************************************************************/
190 static close_method_decl(filtered
)
192 free_stream_filter(st
->u
.filtered
.filter
);
193 return close_istream(st
->u
.filtered
.upstream
);
196 static read_method_decl(filtered
)
198 struct filtered_istream
*fs
= &(st
->u
.filtered
);
202 /* do we already have filtered output? */
203 if (fs
->o_ptr
< fs
->o_end
) {
204 size_t to_move
= fs
->o_end
- fs
->o_ptr
;
207 memcpy(buf
+ filled
, fs
->obuf
+ fs
->o_ptr
, to_move
);
208 fs
->o_ptr
+= to_move
;
213 fs
->o_end
= fs
->o_ptr
= 0;
215 /* do we have anything to feed the filter with? */
216 if (fs
->i_ptr
< fs
->i_end
) {
217 size_t to_feed
= fs
->i_end
- fs
->i_ptr
;
218 size_t to_receive
= FILTER_BUFFER
;
219 if (stream_filter(fs
->filter
,
220 fs
->ibuf
+ fs
->i_ptr
, &to_feed
,
221 fs
->obuf
, &to_receive
))
223 fs
->i_ptr
= fs
->i_end
- to_feed
;
224 fs
->o_end
= FILTER_BUFFER
- to_receive
;
228 /* tell the filter to drain upon no more input */
229 if (fs
->input_finished
) {
230 size_t to_receive
= FILTER_BUFFER
;
231 if (stream_filter(fs
->filter
,
233 fs
->obuf
, &to_receive
))
235 fs
->o_end
= FILTER_BUFFER
- to_receive
;
240 fs
->i_end
= fs
->i_ptr
= 0;
242 /* refill the input from the upstream */
243 if (!fs
->input_finished
) {
244 fs
->i_end
= read_istream(fs
->upstream
, fs
->ibuf
, FILTER_BUFFER
);
250 fs
->input_finished
= 1;
255 static struct stream_vtbl filtered_vtbl
= {
256 close_istream_filtered
,
257 read_istream_filtered
,
260 static struct git_istream
*attach_stream_filter(struct git_istream
*st
,
261 struct stream_filter
*filter
)
263 struct git_istream
*ifs
= xmalloc(sizeof(*ifs
));
264 struct filtered_istream
*fs
= &(ifs
->u
.filtered
);
266 ifs
->vtbl
= &filtered_vtbl
;
269 fs
->i_end
= fs
->i_ptr
= 0;
270 fs
->o_end
= fs
->o_ptr
= 0;
271 fs
->input_finished
= 0;
272 ifs
->size
= -1; /* unknown */
276 /*****************************************************************
278 * Loose object stream
280 *****************************************************************/
282 static read_method_decl(loose
)
284 size_t total_read
= 0;
286 switch (st
->z_state
) {
295 if (st
->u
.loose
.hdr_used
< st
->u
.loose
.hdr_avail
) {
296 size_t to_copy
= st
->u
.loose
.hdr_avail
- st
->u
.loose
.hdr_used
;
299 memcpy(buf
, st
->u
.loose
.hdr
+ st
->u
.loose
.hdr_used
, to_copy
);
300 st
->u
.loose
.hdr_used
+= to_copy
;
301 total_read
+= to_copy
;
304 while (total_read
< sz
) {
307 st
->z
.next_out
= (unsigned char *)buf
+ total_read
;
308 st
->z
.avail_out
= sz
- total_read
;
309 status
= git_inflate(&st
->z
, Z_FINISH
);
311 total_read
= st
->z
.next_out
- (unsigned char *)buf
;
313 if (status
== Z_STREAM_END
) {
314 git_inflate_end(&st
->z
);
315 st
->z_state
= z_done
;
318 if (status
!= Z_OK
&& (status
!= Z_BUF_ERROR
|| total_read
< sz
)) {
319 git_inflate_end(&st
->z
);
320 st
->z_state
= z_error
;
327 static close_method_decl(loose
)
329 close_deflated_stream(st
);
330 munmap(st
->u
.loose
.mapped
, st
->u
.loose
.mapsize
);
334 static struct stream_vtbl loose_vtbl
= {
339 static open_method_decl(loose
)
341 st
->u
.loose
.mapped
= map_sha1_file(sha1
, &st
->u
.loose
.mapsize
);
342 if (!st
->u
.loose
.mapped
)
344 if ((unpack_sha1_header(&st
->z
,
348 sizeof(st
->u
.loose
.hdr
)) < 0) ||
349 (parse_sha1_header(st
->u
.loose
.hdr
, &st
->size
) < 0)) {
350 git_inflate_end(&st
->z
);
351 munmap(st
->u
.loose
.mapped
, st
->u
.loose
.mapsize
);
355 st
->u
.loose
.hdr_used
= strlen(st
->u
.loose
.hdr
) + 1;
356 st
->u
.loose
.hdr_avail
= st
->z
.total_out
;
357 st
->z_state
= z_used
;
359 st
->vtbl
= &loose_vtbl
;
364 /*****************************************************************
366 * Non-delta packed object stream
368 *****************************************************************/
370 static read_method_decl(pack_non_delta
)
372 size_t total_read
= 0;
374 switch (st
->z_state
) {
376 memset(&st
->z
, 0, sizeof(st
->z
));
377 git_inflate_init(&st
->z
);
378 st
->z_state
= z_used
;
388 while (total_read
< sz
) {
390 struct pack_window
*window
= NULL
;
391 unsigned char *mapped
;
393 mapped
= use_pack(st
->u
.in_pack
.pack
, &window
,
394 st
->u
.in_pack
.pos
, &st
->z
.avail_in
);
396 st
->z
.next_out
= (unsigned char *)buf
+ total_read
;
397 st
->z
.avail_out
= sz
- total_read
;
398 st
->z
.next_in
= mapped
;
399 status
= git_inflate(&st
->z
, Z_FINISH
);
401 st
->u
.in_pack
.pos
+= st
->z
.next_in
- mapped
;
402 total_read
= st
->z
.next_out
- (unsigned char *)buf
;
405 if (status
== Z_STREAM_END
) {
406 git_inflate_end(&st
->z
);
407 st
->z_state
= z_done
;
410 if (status
!= Z_OK
&& status
!= Z_BUF_ERROR
) {
411 git_inflate_end(&st
->z
);
412 st
->z_state
= z_error
;
419 static close_method_decl(pack_non_delta
)
421 close_deflated_stream(st
);
425 static struct stream_vtbl pack_non_delta_vtbl
= {
426 close_istream_pack_non_delta
,
427 read_istream_pack_non_delta
,
430 static open_method_decl(pack_non_delta
)
432 struct pack_window
*window
;
433 enum object_type in_pack_type
;
435 st
->u
.in_pack
.pack
= oi
->u
.packed
.pack
;
436 st
->u
.in_pack
.pos
= oi
->u
.packed
.offset
;
439 in_pack_type
= unpack_object_header(st
->u
.in_pack
.pack
,
444 switch (in_pack_type
) {
446 return -1; /* we do not do deltas for now */
453 st
->z_state
= z_unused
;
454 st
->vtbl
= &pack_non_delta_vtbl
;
459 /*****************************************************************
463 *****************************************************************/
465 static close_method_decl(incore
)
467 free(st
->u
.incore
.buf
);
471 static read_method_decl(incore
)
473 size_t read_size
= sz
;
474 size_t remainder
= st
->size
- st
->u
.incore
.read_ptr
;
476 if (remainder
<= read_size
)
477 read_size
= remainder
;
479 memcpy(buf
, st
->u
.incore
.buf
+ st
->u
.incore
.read_ptr
, read_size
);
480 st
->u
.incore
.read_ptr
+= read_size
;
485 static struct stream_vtbl incore_vtbl
= {
486 close_istream_incore
,
490 static open_method_decl(incore
)
492 st
->u
.incore
.buf
= read_sha1_file_extended(sha1
, type
, &st
->size
, 0);
493 st
->u
.incore
.read_ptr
= 0;
494 st
->vtbl
= &incore_vtbl
;
496 return st
->u
.incore
.buf
? 0 : -1;
500 /****************************************************************
501 * Users of streaming interface
502 ****************************************************************/
504 int stream_blob_to_fd(int fd
, const struct object_id
*oid
, struct stream_filter
*filter
,
507 struct git_istream
*st
;
508 enum object_type type
;
513 st
= open_istream(oid
, &type
, &sz
, filter
);
516 free_stream_filter(filter
);
519 if (type
!= OBJ_BLOB
)
523 ssize_t wrote
, holeto
;
524 ssize_t readlen
= read_istream(st
, buf
, sizeof(buf
));
530 if (can_seek
&& sizeof(buf
) == readlen
) {
531 for (holeto
= 0; holeto
< readlen
; holeto
++)
534 if (readlen
== holeto
) {
540 if (kept
&& lseek(fd
, kept
, SEEK_CUR
) == (off_t
) -1)
544 wrote
= write_in_full(fd
, buf
, readlen
);
549 if (kept
&& (lseek(fd
, kept
- 1, SEEK_CUR
) == (off_t
) -1 ||
550 xwrite(fd
, "", 1) != 1))