2 * Copyright (c) 2011, Google Inc.
15 typedef int (*open_istream_fn
)(struct git_istream
*,
17 const struct object_id
*,
19 typedef int (*close_istream_fn
)(struct git_istream
*);
20 typedef ssize_t (*read_istream_fn
)(struct git_istream
*, char *, size_t);
23 close_istream_fn close
;
27 #define open_method_decl(name) \
28 int open_istream_ ##name \
29 (struct git_istream *st, struct object_info *oi, \
30 const struct object_id *oid, \
31 enum object_type *type)
33 #define close_method_decl(name) \
34 int close_istream_ ##name \
35 (struct git_istream *st)
37 #define read_method_decl(name) \
38 ssize_t read_istream_ ##name \
39 (struct git_istream *st, char *buf, size_t sz)
41 /* forward declaration */
42 static open_method_decl(incore
);
43 static open_method_decl(loose
);
44 static open_method_decl(pack_non_delta
);
45 static struct git_istream
*attach_stream_filter(struct git_istream
*st
,
46 struct stream_filter
*filter
);
49 static open_istream_fn open_istream_tbl
[] = {
52 open_istream_pack_non_delta
,
55 #define FILTER_BUFFER (1024*16)
57 struct filtered_istream
{
58 struct git_istream
*upstream
;
59 struct stream_filter
*filter
;
60 char ibuf
[FILTER_BUFFER
];
61 char obuf
[FILTER_BUFFER
];
68 const struct stream_vtbl
*vtbl
;
69 unsigned long size
; /* inflated size of full object */
71 enum { z_unused
, z_used
, z_done
, z_error
} z_state
;
75 char *buf
; /* from read_object() */
76 unsigned long read_ptr
;
81 unsigned long mapsize
;
88 struct packed_git
*pack
;
92 struct filtered_istream filtered
;
96 int close_istream(struct git_istream
*st
)
98 int r
= st
->vtbl
->close(st
);
103 ssize_t
read_istream(struct git_istream
*st
, void *buf
, size_t sz
)
105 return st
->vtbl
->read(st
, buf
, sz
);
108 static enum input_source
istream_source(const unsigned char *sha1
,
109 enum object_type
*type
,
110 struct object_info
*oi
)
114 struct object_id oid
;
116 hashcpy(oid
.hash
, sha1
);
120 status
= oid_object_info_extended(&oid
, oi
, 0);
124 switch (oi
->whence
) {
128 if (!oi
->u
.packed
.is_delta
&& big_file_threshold
< size
)
129 return pack_non_delta
;
136 struct git_istream
*open_istream(const struct object_id
*oid
,
137 enum object_type
*type
,
139 struct stream_filter
*filter
)
141 struct git_istream
*st
;
142 struct object_info oi
= OBJECT_INFO_INIT
;
143 const unsigned char *real
= lookup_replace_object(oid
->hash
);
144 enum input_source src
= istream_source(real
, type
, &oi
);
145 struct object_id realoid
;
147 hashcpy(realoid
.hash
, real
);
152 st
= xmalloc(sizeof(*st
));
153 if (open_istream_tbl
[src
](st
, &oi
, &realoid
, type
)) {
154 if (open_istream_incore(st
, &oi
, &realoid
, type
)) {
160 /* Add "&& !is_null_stream_filter(filter)" for performance */
161 struct git_istream
*nst
= attach_stream_filter(st
, filter
);
174 /*****************************************************************
178 *****************************************************************/
180 static void close_deflated_stream(struct git_istream
*st
)
182 if (st
->z_state
== z_used
)
183 git_inflate_end(&st
->z
);
187 /*****************************************************************
191 *****************************************************************/
193 static close_method_decl(filtered
)
195 free_stream_filter(st
->u
.filtered
.filter
);
196 return close_istream(st
->u
.filtered
.upstream
);
199 static read_method_decl(filtered
)
201 struct filtered_istream
*fs
= &(st
->u
.filtered
);
205 /* do we already have filtered output? */
206 if (fs
->o_ptr
< fs
->o_end
) {
207 size_t to_move
= fs
->o_end
- fs
->o_ptr
;
210 memcpy(buf
+ filled
, fs
->obuf
+ fs
->o_ptr
, to_move
);
211 fs
->o_ptr
+= to_move
;
216 fs
->o_end
= fs
->o_ptr
= 0;
218 /* do we have anything to feed the filter with? */
219 if (fs
->i_ptr
< fs
->i_end
) {
220 size_t to_feed
= fs
->i_end
- fs
->i_ptr
;
221 size_t to_receive
= FILTER_BUFFER
;
222 if (stream_filter(fs
->filter
,
223 fs
->ibuf
+ fs
->i_ptr
, &to_feed
,
224 fs
->obuf
, &to_receive
))
226 fs
->i_ptr
= fs
->i_end
- to_feed
;
227 fs
->o_end
= FILTER_BUFFER
- to_receive
;
231 /* tell the filter to drain upon no more input */
232 if (fs
->input_finished
) {
233 size_t to_receive
= FILTER_BUFFER
;
234 if (stream_filter(fs
->filter
,
236 fs
->obuf
, &to_receive
))
238 fs
->o_end
= FILTER_BUFFER
- to_receive
;
243 fs
->i_end
= fs
->i_ptr
= 0;
245 /* refill the input from the upstream */
246 if (!fs
->input_finished
) {
247 fs
->i_end
= read_istream(fs
->upstream
, fs
->ibuf
, FILTER_BUFFER
);
253 fs
->input_finished
= 1;
258 static struct stream_vtbl filtered_vtbl
= {
259 close_istream_filtered
,
260 read_istream_filtered
,
263 static struct git_istream
*attach_stream_filter(struct git_istream
*st
,
264 struct stream_filter
*filter
)
266 struct git_istream
*ifs
= xmalloc(sizeof(*ifs
));
267 struct filtered_istream
*fs
= &(ifs
->u
.filtered
);
269 ifs
->vtbl
= &filtered_vtbl
;
272 fs
->i_end
= fs
->i_ptr
= 0;
273 fs
->o_end
= fs
->o_ptr
= 0;
274 fs
->input_finished
= 0;
275 ifs
->size
= -1; /* unknown */
279 /*****************************************************************
281 * Loose object stream
283 *****************************************************************/
285 static read_method_decl(loose
)
287 size_t total_read
= 0;
289 switch (st
->z_state
) {
298 if (st
->u
.loose
.hdr_used
< st
->u
.loose
.hdr_avail
) {
299 size_t to_copy
= st
->u
.loose
.hdr_avail
- st
->u
.loose
.hdr_used
;
302 memcpy(buf
, st
->u
.loose
.hdr
+ st
->u
.loose
.hdr_used
, to_copy
);
303 st
->u
.loose
.hdr_used
+= to_copy
;
304 total_read
+= to_copy
;
307 while (total_read
< sz
) {
310 st
->z
.next_out
= (unsigned char *)buf
+ total_read
;
311 st
->z
.avail_out
= sz
- total_read
;
312 status
= git_inflate(&st
->z
, Z_FINISH
);
314 total_read
= st
->z
.next_out
- (unsigned char *)buf
;
316 if (status
== Z_STREAM_END
) {
317 git_inflate_end(&st
->z
);
318 st
->z_state
= z_done
;
321 if (status
!= Z_OK
&& (status
!= Z_BUF_ERROR
|| total_read
< sz
)) {
322 git_inflate_end(&st
->z
);
323 st
->z_state
= z_error
;
330 static close_method_decl(loose
)
332 close_deflated_stream(st
);
333 munmap(st
->u
.loose
.mapped
, st
->u
.loose
.mapsize
);
337 static struct stream_vtbl loose_vtbl
= {
342 static open_method_decl(loose
)
344 st
->u
.loose
.mapped
= map_sha1_file(oid
->hash
, &st
->u
.loose
.mapsize
);
345 if (!st
->u
.loose
.mapped
)
347 if ((unpack_sha1_header(&st
->z
,
351 sizeof(st
->u
.loose
.hdr
)) < 0) ||
352 (parse_sha1_header(st
->u
.loose
.hdr
, &st
->size
) < 0)) {
353 git_inflate_end(&st
->z
);
354 munmap(st
->u
.loose
.mapped
, st
->u
.loose
.mapsize
);
358 st
->u
.loose
.hdr_used
= strlen(st
->u
.loose
.hdr
) + 1;
359 st
->u
.loose
.hdr_avail
= st
->z
.total_out
;
360 st
->z_state
= z_used
;
362 st
->vtbl
= &loose_vtbl
;
367 /*****************************************************************
369 * Non-delta packed object stream
371 *****************************************************************/
373 static read_method_decl(pack_non_delta
)
375 size_t total_read
= 0;
377 switch (st
->z_state
) {
379 memset(&st
->z
, 0, sizeof(st
->z
));
380 git_inflate_init(&st
->z
);
381 st
->z_state
= z_used
;
391 while (total_read
< sz
) {
393 struct pack_window
*window
= NULL
;
394 unsigned char *mapped
;
396 mapped
= use_pack(st
->u
.in_pack
.pack
, &window
,
397 st
->u
.in_pack
.pos
, &st
->z
.avail_in
);
399 st
->z
.next_out
= (unsigned char *)buf
+ total_read
;
400 st
->z
.avail_out
= sz
- total_read
;
401 st
->z
.next_in
= mapped
;
402 status
= git_inflate(&st
->z
, Z_FINISH
);
404 st
->u
.in_pack
.pos
+= st
->z
.next_in
- mapped
;
405 total_read
= st
->z
.next_out
- (unsigned char *)buf
;
408 if (status
== Z_STREAM_END
) {
409 git_inflate_end(&st
->z
);
410 st
->z_state
= z_done
;
413 if (status
!= Z_OK
&& status
!= Z_BUF_ERROR
) {
414 git_inflate_end(&st
->z
);
415 st
->z_state
= z_error
;
422 static close_method_decl(pack_non_delta
)
424 close_deflated_stream(st
);
428 static struct stream_vtbl pack_non_delta_vtbl
= {
429 close_istream_pack_non_delta
,
430 read_istream_pack_non_delta
,
433 static open_method_decl(pack_non_delta
)
435 struct pack_window
*window
;
436 enum object_type in_pack_type
;
438 st
->u
.in_pack
.pack
= oi
->u
.packed
.pack
;
439 st
->u
.in_pack
.pos
= oi
->u
.packed
.offset
;
442 in_pack_type
= unpack_object_header(st
->u
.in_pack
.pack
,
447 switch (in_pack_type
) {
449 return -1; /* we do not do deltas for now */
456 st
->z_state
= z_unused
;
457 st
->vtbl
= &pack_non_delta_vtbl
;
462 /*****************************************************************
466 *****************************************************************/
468 static close_method_decl(incore
)
470 free(st
->u
.incore
.buf
);
474 static read_method_decl(incore
)
476 size_t read_size
= sz
;
477 size_t remainder
= st
->size
- st
->u
.incore
.read_ptr
;
479 if (remainder
<= read_size
)
480 read_size
= remainder
;
482 memcpy(buf
, st
->u
.incore
.buf
+ st
->u
.incore
.read_ptr
, read_size
);
483 st
->u
.incore
.read_ptr
+= read_size
;
488 static struct stream_vtbl incore_vtbl
= {
489 close_istream_incore
,
493 static open_method_decl(incore
)
495 st
->u
.incore
.buf
= read_object_file_extended(oid
, type
, &st
->size
, 0);
496 st
->u
.incore
.read_ptr
= 0;
497 st
->vtbl
= &incore_vtbl
;
499 return st
->u
.incore
.buf
? 0 : -1;
503 /****************************************************************
504 * Users of streaming interface
505 ****************************************************************/
507 int stream_blob_to_fd(int fd
, const struct object_id
*oid
, struct stream_filter
*filter
,
510 struct git_istream
*st
;
511 enum object_type type
;
516 st
= open_istream(oid
, &type
, &sz
, filter
);
519 free_stream_filter(filter
);
522 if (type
!= OBJ_BLOB
)
526 ssize_t wrote
, holeto
;
527 ssize_t readlen
= read_istream(st
, buf
, sizeof(buf
));
533 if (can_seek
&& sizeof(buf
) == readlen
) {
534 for (holeto
= 0; holeto
< readlen
; holeto
++)
537 if (readlen
== holeto
) {
543 if (kept
&& lseek(fd
, kept
, SEEK_CUR
) == (off_t
) -1)
547 wrote
= write_in_full(fd
, buf
, readlen
);
552 if (kept
&& (lseek(fd
, kept
- 1, SEEK_CUR
) == (off_t
) -1 ||
553 xwrite(fd
, "", 1) != 1))