?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
pager.h: move declarations for pager.c functions from cache.h
[alt-git.git] / convert.c
blob5a2ea5308d6b1467b1ac1925507b1dacc5d809e4
1 #include "cache.h"
2 #include "advice.h"
3 #include "config.h"
4 #include "convert.h"
5 #include "gettext.h"
6 #include "hex.h"
7 #include "object-store.h"
8 #include "attr.h"
9 #include "run-command.h"
10 #include "quote.h"
11 #include "sigchain.h"
12 #include "pkt-line.h"
13 #include "sub-process.h"
14 #include "trace.h"
15 #include "utf8.h"
16 #include "ll-merge.h"
17 #include "wrapper.h"
18
19 /*
20 * convert.c - convert a file when checking it out and checking it in.
21 *
22 * This should use the pathname to decide on whether it wants to do some
23 * more interesting conversions (automatic gzip/unzip, general format
24 * conversions etc etc), but by default it just does automatic CRLF<->LF
25 * translation when the "text" attribute or "auto_crlf" option is set.
26 */
27
28 /* Stat bits: When BIN is set, the txt bits are unset */
29 #define CONVERT_STAT_BITS_TXT_LF 0x1
30 #define CONVERT_STAT_BITS_TXT_CRLF 0x2
31 #define CONVERT_STAT_BITS_BIN 0x4
32
33 struct text_stat {
34 /* NUL, CR, LF and CRLF counts */
35 unsigned nul, lonecr, lonelf, crlf;
36
37 /* These are just approximations! */
38 unsigned printable, nonprintable;
39 };
40
41 static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
42 {
43 unsigned long i;
44
45 memset(stats, 0, sizeof(*stats));
46
47 for (i = 0; i < size; i++) {
48 unsigned char c = buf[i];
49 if (c == '\r') {
50 if (i+1 < size && buf[i+1] == '\n') {
51 stats->crlf++;
52 i++;
53 } else
54 stats->lonecr++;
55 continue;
56 }
57 if (c == '\n') {
58 stats->lonelf++;
59 continue;
60 }
61 if (c == 127)
62 /* DEL */
63 stats->nonprintable++;
64 else if (c < 32) {
65 switch (c) {
66 /* BS, HT, ESC and FF */
67 case '\b': case '\t': case '\033': case '\014':
68 stats->printable++;
69 break;
70 case 0:
71 stats->nul++;
72 /* fall through */
73 default:
74 stats->nonprintable++;
75 }
76 }
77 else
78 stats->printable++;
79 }
80
81 /* If file ends with EOF then don't count this EOF as non-printable. */
82 if (size >= 1 && buf[size-1] == '\032')
83 stats->nonprintable--;
84 }
85
86 /*
87 * The same heuristics as diff.c::mmfile_is_binary()
88 * We treat files with bare CR as binary
89 */
90 static int convert_is_binary(const struct text_stat *stats)
91 {
92 if (stats->lonecr)
93 return 1;
94 if (stats->nul)
95 return 1;
96 if ((stats->printable >> 7) < stats->nonprintable)
97 return 1;
98 return 0;
99 }
100
101 static unsigned int gather_convert_stats(const char *data, unsigned long size)
102 {
103 struct text_stat stats;
104 int ret = 0;
105 if (!data || !size)
106 return 0;
107 gather_stats(data, size, &stats);
108 if (convert_is_binary(&stats))
109 ret |= CONVERT_STAT_BITS_BIN;
110 if (stats.crlf)
111 ret |= CONVERT_STAT_BITS_TXT_CRLF;
112 if (stats.lonelf)
113 ret |= CONVERT_STAT_BITS_TXT_LF;
114
115 return ret;
116 }
117
118 static const char *gather_convert_stats_ascii(const char *data, unsigned long size)
119 {
120 unsigned int convert_stats = gather_convert_stats(data, size);
121
122 if (convert_stats & CONVERT_STAT_BITS_BIN)
123 return "-text";
124 switch (convert_stats) {
125 case CONVERT_STAT_BITS_TXT_LF:
126 return "lf";
127 case CONVERT_STAT_BITS_TXT_CRLF:
128 return "crlf";
129 case CONVERT_STAT_BITS_TXT_LF | CONVERT_STAT_BITS_TXT_CRLF:
130 return "mixed";
131 default:
132 return "none";
133 }
134 }
135
136 const char *get_cached_convert_stats_ascii(struct index_state *istate,
137 const char *path)
138 {
139 const char *ret;
140 unsigned long sz;
141 void *data = read_blob_data_from_index(istate, path, &sz);
142 ret = gather_convert_stats_ascii(data, sz);
143 free(data);
144 return ret;
145 }
146
147 const char *get_wt_convert_stats_ascii(const char *path)
148 {
149 const char *ret = "";
150 struct strbuf sb = STRBUF_INIT;
151 if (strbuf_read_file(&sb, path, 0) >= 0)
152 ret = gather_convert_stats_ascii(sb.buf, sb.len);
153 strbuf_release(&sb);
154 return ret;
155 }
156
157 static int text_eol_is_crlf(void)
158 {
159 if (auto_crlf == AUTO_CRLF_TRUE)
160 return 1;
161 else if (auto_crlf == AUTO_CRLF_INPUT)
162 return 0;
163 if (core_eol == EOL_CRLF)
164 return 1;
165 if (core_eol == EOL_UNSET && EOL_NATIVE == EOL_CRLF)
166 return 1;
167 return 0;
168 }
169
170 static enum eol output_eol(enum convert_crlf_action crlf_action)
171 {
172 switch (crlf_action) {
173 case CRLF_BINARY:
174 return EOL_UNSET;
175 case CRLF_TEXT_CRLF:
176 return EOL_CRLF;
177 case CRLF_TEXT_INPUT:
178 return EOL_LF;
179 case CRLF_UNDEFINED:
180 case CRLF_AUTO_CRLF:
181 return EOL_CRLF;
182 case CRLF_AUTO_INPUT:
183 return EOL_LF;
184 case CRLF_TEXT:
185 case CRLF_AUTO:
186 /* fall through */
187 return text_eol_is_crlf() ? EOL_CRLF : EOL_LF;
188 }
189 warning(_("illegal crlf_action %d"), (int)crlf_action);
190 return core_eol;
191 }
192
193 static void check_global_conv_flags_eol(const char *path,
194 struct text_stat *old_stats, struct text_stat *new_stats,
195 int conv_flags)
196 {
197 if (old_stats->crlf && !new_stats->crlf ) {
198 /*
199 * CRLFs would not be restored by checkout
200 */
201 if (conv_flags & CONV_EOL_RNDTRP_DIE)
202 die(_("CRLF would be replaced by LF in %s"), path);
203 else if (conv_flags & CONV_EOL_RNDTRP_WARN)
204 warning(_("in the working copy of '%s', CRLF will be"
205 " replaced by LF the next time Git touches"
206 " it"), path);
207 } else if (old_stats->lonelf && !new_stats->lonelf ) {
208 /*
209 * CRLFs would be added by checkout
210 */
211 if (conv_flags & CONV_EOL_RNDTRP_DIE)
212 die(_("LF would be replaced by CRLF in %s"), path);
213 else if (conv_flags & CONV_EOL_RNDTRP_WARN)
214 warning(_("in the working copy of '%s', LF will be"
215 " replaced by CRLF the next time Git touches"
216 " it"), path);
217 }
218 }
219
220 static int has_crlf_in_index(struct index_state *istate, const char *path)
221 {
222 unsigned long sz;
223 void *data;
224 const char *crp;
225 int has_crlf = 0;
226
227 data = read_blob_data_from_index(istate, path, &sz);
228 if (!data)
229 return 0;
230
231 crp = memchr(data, '\r', sz);
232 if (crp) {
233 unsigned int ret_stats;
234 ret_stats = gather_convert_stats(data, sz);
235 if (!(ret_stats & CONVERT_STAT_BITS_BIN) &&
236 (ret_stats & CONVERT_STAT_BITS_TXT_CRLF))
237 has_crlf = 1;
238 }
239 free(data);
240 return has_crlf;
241 }
242
243 static int will_convert_lf_to_crlf(struct text_stat *stats,
244 enum convert_crlf_action crlf_action)
245 {
246 if (output_eol(crlf_action) != EOL_CRLF)
247 return 0;
248 /* No "naked" LF? Nothing to convert, regardless. */
249 if (!stats->lonelf)
250 return 0;
251
252 if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
253 /* If we have any CR or CRLF line endings, we do not touch it */
254 /* This is the new safer autocrlf-handling */
255 if (stats->lonecr || stats->crlf)
256 return 0;
257
258 if (convert_is_binary(stats))
259 return 0;
260 }
261 return 1;
262
263 }
264
265 static int validate_encoding(const char *path, const char *enc,
266 const char *data, size_t len, int die_on_error)
267 {
268 const char *stripped;
269
270 /* We only check for UTF here as UTF?? can be an alias for UTF-?? */
271 if (skip_iprefix(enc, "UTF", &stripped)) {
272 skip_prefix(stripped, "-", &stripped);
273
274 /*
275 * Check for detectable errors in UTF encodings
276 */
277 if (has_prohibited_utf_bom(enc, data, len)) {
278 const char *error_msg = _(
279 "BOM is prohibited in '%s' if encoded as %s");
280 /*
281 * This advice is shown for UTF-??BE and UTF-??LE encodings.
282 * We cut off the last two characters of the encoding name
283 * to generate the encoding name suitable for BOMs.
284 */
285 const char *advise_msg = _(
286 "The file '%s' contains a byte order "
287 "mark (BOM). Please use UTF-%.*s as "
288 "working-tree-encoding.");
289 int stripped_len = strlen(stripped) - strlen("BE");
290 advise(advise_msg, path, stripped_len, stripped);
291 if (die_on_error)
292 die(error_msg, path, enc);
293 else {
294 return error(error_msg, path, enc);
295 }
296
297 } else if (is_missing_required_utf_bom(enc, data, len)) {
298 const char *error_msg = _(
299 "BOM is required in '%s' if encoded as %s");
300 const char *advise_msg = _(
301 "The file '%s' is missing a byte order "
302 "mark (BOM). Please use UTF-%sBE or UTF-%sLE "
303 "(depending on the byte order) as "
304 "working-tree-encoding.");
305 advise(advise_msg, path, stripped, stripped);
306 if (die_on_error)
307 die(error_msg, path, enc);
308 else {
309 return error(error_msg, path, enc);
310 }
311 }
312
313 }
314 return 0;
315 }
316
317 static void trace_encoding(const char *context, const char *path,
318 const char *encoding, const char *buf, size_t len)
319 {
320 static struct trace_key coe = TRACE_KEY_INIT(WORKING_TREE_ENCODING);
321 struct strbuf trace = STRBUF_INIT;
322 int i;
323
324 strbuf_addf(&trace, "%s (%s, considered %s):\n", context, path, encoding);
325 for (i = 0; i < len && buf; ++i) {
326 strbuf_addf(
327 &trace, "| \033[2m%2i:\033[0m %2x \033[2m%c\033[0m%c",
328 i,
329 (unsigned char) buf[i],
330 (buf[i] > 32 && buf[i] < 127 ? buf[i] : ' '),
331 ((i+1) % 8 && (i+1) < len ? ' ' : '\n')
332 );
333 }
334 strbuf_addchars(&trace, '\n', 1);
335
336 trace_strbuf(&coe, &trace);
337 strbuf_release(&trace);
338 }
339
340 static int check_roundtrip(const char *enc_name)
341 {
342 /*
343 * check_roundtrip_encoding contains a string of comma and/or
344 * space separated encodings (eg. "UTF-16, ASCII, CP1125").
345 * Search for the given encoding in that string.
346 */
347 const char *found = strcasestr(check_roundtrip_encoding, enc_name);
348 const char *next;
349 int len;
350 if (!found)
351 return 0;
352 next = found + strlen(enc_name);
353 len = strlen(check_roundtrip_encoding);
354 return (found && (
355 /*
356 * check that the found encoding is at the
357 * beginning of check_roundtrip_encoding or
358 * that it is prefixed with a space or comma
359 */
360 found == check_roundtrip_encoding || (
361 (isspace(found[-1]) || found[-1] == ',')
362 )
363 ) && (
364 /*
365 * check that the found encoding is at the
366 * end of check_roundtrip_encoding or
367 * that it is suffixed with a space or comma
368 */
369 next == check_roundtrip_encoding + len || (
370 next < check_roundtrip_encoding + len &&
371 (isspace(next[0]) || next[0] == ',')
372 )
373 ));
374 }
375
376 static const char *default_encoding = "UTF-8";
377
378 static int encode_to_git(const char *path, const char *src, size_t src_len,
379 struct strbuf *buf, const char *enc, int conv_flags)
380 {
381 char *dst;
382 size_t dst_len;
383 int die_on_error = conv_flags & CONV_WRITE_OBJECT;
384
385 /*
386 * No encoding is specified or there is nothing to encode.
387 * Tell the caller that the content was not modified.
388 */
389 if (!enc || (src && !src_len))
390 return 0;
391
392 /*
393 * Looks like we got called from "would_convert_to_git()".
394 * This means Git wants to know if it would encode (= modify!)
395 * the content. Let's answer with "yes", since an encoding was
396 * specified.
397 */
398 if (!buf && !src)
399 return 1;
400
401 if (validate_encoding(path, enc, src, src_len, die_on_error))
402 return 0;
403
404 trace_encoding("source", path, enc, src, src_len);
405 dst = reencode_string_len(src, src_len, default_encoding, enc,
406 &dst_len);
407 if (!dst) {
408 /*
409 * We could add the blob "as-is" to Git. However, on checkout
410 * we would try to re-encode to the original encoding. This
411 * would fail and we would leave the user with a messed-up
412 * working tree. Let's try to avoid this by screaming loud.
413 */
414 const char* msg = _("failed to encode '%s' from %s to %s");
415 if (die_on_error)
416 die(msg, path, enc, default_encoding);
417 else {
418 error(msg, path, enc, default_encoding);
419 return 0;
420 }
421 }
422 trace_encoding("destination", path, default_encoding, dst, dst_len);
423
424 /*
425 * UTF supports lossless conversion round tripping [1] and conversions
426 * between UTF and other encodings are mostly round trip safe as
427 * Unicode aims to be a superset of all other character encodings.
428 * However, certain encodings (e.g. SHIFT-JIS) are known to have round
429 * trip issues [2]. Check the round trip conversion for all encodings
430 * listed in core.checkRoundtripEncoding.
431 *
432 * The round trip check is only performed if content is written to Git.
433 * This ensures that no information is lost during conversion to/from
434 * the internal UTF-8 representation.
435 *
436 * Please note, the code below is not tested because I was not able to
437 * generate a faulty round trip without an iconv error. Iconv errors
438 * are already caught above.
439 *
440 * [1] http://unicode.org/faq/utf_bom.html#gen2
441 * [2] https://support.microsoft.com/en-us/help/170559/prb-conversion-problem-between-shift-jis-and-unicode
442 */
443 if (die_on_error && check_roundtrip(enc)) {
444 char *re_src;
445 size_t re_src_len;
446
447 re_src = reencode_string_len(dst, dst_len,
448 enc, default_encoding,
449 &re_src_len);
450
451 trace_printf("Checking roundtrip encoding for %s...\n", enc);
452 trace_encoding("reencoded source", path, enc,
453 re_src, re_src_len);
454
455 if (!re_src || src_len != re_src_len ||
456 memcmp(src, re_src, src_len)) {
457 const char* msg = _("encoding '%s' from %s to %s and "
458 "back is not the same");
459 die(msg, path, enc, default_encoding);
460 }
461
462 free(re_src);
463 }
464
465 strbuf_attach(buf, dst, dst_len, dst_len + 1);
466 return 1;
467 }
468
469 static int encode_to_worktree(const char *path, const char *src, size_t src_len,
470 struct strbuf *buf, const char *enc)
471 {
472 char *dst;
473 size_t dst_len;
474
475 /*
476 * No encoding is specified or there is nothing to encode.
477 * Tell the caller that the content was not modified.
478 */
479 if (!enc || (src && !src_len))
480 return 0;
481
482 dst = reencode_string_len(src, src_len, enc, default_encoding,
483 &dst_len);
484 if (!dst) {
485 error(_("failed to encode '%s' from %s to %s"),
486 path, default_encoding, enc);
487 return 0;
488 }
489
490 strbuf_attach(buf, dst, dst_len, dst_len + 1);
491 return 1;
492 }
493
494 static int crlf_to_git(struct index_state *istate,
495 const char *path, const char *src, size_t len,
496 struct strbuf *buf,
497 enum convert_crlf_action crlf_action, int conv_flags)
498 {
499 struct text_stat stats;
500 char *dst;
501 int convert_crlf_into_lf;
502
503 if (crlf_action == CRLF_BINARY ||
504 (src && !len))
505 return 0;
506
507 /*
508 * If we are doing a dry-run and have no source buffer, there is
509 * nothing to analyze; we must assume we would convert.
510 */
511 if (!buf && !src)
512 return 1;
513
514 gather_stats(src, len, &stats);
515 /* Optimization: No CRLF? Nothing to convert, regardless. */
516 convert_crlf_into_lf = !!stats.crlf;
517
518 if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
519 if (convert_is_binary(&stats))
520 return 0;
521 /*
522 * If the file in the index has any CR in it, do not
523 * convert. This is the new safer autocrlf handling,
524 * unless we want to renormalize in a merge or
525 * cherry-pick.
526 */
527 if ((!(conv_flags & CONV_EOL_RENORMALIZE)) &&
528 has_crlf_in_index(istate, path))
529 convert_crlf_into_lf = 0;
530 }
531 if (((conv_flags & CONV_EOL_RNDTRP_WARN) ||
532 ((conv_flags & CONV_EOL_RNDTRP_DIE) && len))) {
533 struct text_stat new_stats;
534 memcpy(&new_stats, &stats, sizeof(new_stats));
535 /* simulate "git add" */
536 if (convert_crlf_into_lf) {
537 new_stats.lonelf += new_stats.crlf;
538 new_stats.crlf = 0;
539 }
540 /* simulate "git checkout" */
541 if (will_convert_lf_to_crlf(&new_stats, crlf_action)) {
542 new_stats.crlf += new_stats.lonelf;
543 new_stats.lonelf = 0;
544 }
545 check_global_conv_flags_eol(path, &stats, &new_stats, conv_flags);
546 }
547 if (!convert_crlf_into_lf)
548 return 0;
549
550 /*
551 * At this point all of our source analysis is done, and we are sure we
552 * would convert. If we are in dry-run mode, we can give an answer.
553 */
554 if (!buf)
555 return 1;
556
557 /* only grow if not in place */
558 if (strbuf_avail(buf) + buf->len < len)
559 strbuf_grow(buf, len - buf->len);
560 dst = buf->buf;
561 if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
562 /*
563 * If we guessed, we already know we rejected a file with
564 * lone CR, and we can strip a CR without looking at what
565 * follow it.
566 */
567 do {
568 unsigned char c = *src++;
569 if (c != '\r')
570 *dst++ = c;
571 } while (--len);
572 } else {
573 do {
574 unsigned char c = *src++;
575 if (! (c == '\r' && (1 < len && *src == '\n')))
576 *dst++ = c;
577 } while (--len);
578 }
579 strbuf_setlen(buf, dst - buf->buf);
580 return 1;
581 }
582
583 static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf,
584 enum convert_crlf_action crlf_action)
585 {
586 char *to_free = NULL;
587 struct text_stat stats;
588
589 if (!len || output_eol(crlf_action) != EOL_CRLF)
590 return 0;
591
592 gather_stats(src, len, &stats);
593 if (!will_convert_lf_to_crlf(&stats, crlf_action))
594 return 0;
595
596 /* are we "faking" in place editing ? */
597 if (src == buf->buf)
598 to_free = strbuf_detach(buf, NULL);
599
600 strbuf_grow(buf, len + stats.lonelf);
601 for (;;) {
602 const char *nl = memchr(src, '\n', len);
603 if (!nl)
604 break;
605 if (nl > src && nl[-1] == '\r') {
606 strbuf_add(buf, src, nl + 1 - src);
607 } else {
608 strbuf_add(buf, src, nl - src);
609 strbuf_addstr(buf, "\r\n");
610 }
611 len -= nl + 1 - src;
612 src = nl + 1;
613 }
614 strbuf_add(buf, src, len);
615
616 free(to_free);
617 return 1;
618 }
619
620 struct filter_params {
621 const char *src;
622 size_t size;
623 int fd;
624 const char *cmd;
625 const char *path;
626 };
627
628 static int filter_buffer_or_fd(int in UNUSED, int out, void *data)
629 {
630 /*
631 * Spawn cmd and feed the buffer contents through its stdin.
632 */
633 struct child_process child_process = CHILD_PROCESS_INIT;
634 struct filter_params *params = (struct filter_params *)data;
635 int write_err, status;
636
637 /* apply % substitution to cmd */
638 struct strbuf cmd = STRBUF_INIT;
639 struct strbuf path = STRBUF_INIT;
640 struct strbuf_expand_dict_entry dict[] = {
641 { "f", NULL, },
642 { NULL, NULL, },
643 };
644
645 /* quote the path to preserve spaces, etc. */
646 sq_quote_buf(&path, params->path);
647 dict[0].value = path.buf;
648
649 /* expand all %f with the quoted path */
650 strbuf_expand(&cmd, params->cmd, strbuf_expand_dict_cb, &dict);
651 strbuf_release(&path);
652
653 strvec_push(&child_process.args, cmd.buf);
654 child_process.use_shell = 1;
655 child_process.in = -1;
656 child_process.out = out;
657
658 if (start_command(&child_process)) {
659 strbuf_release(&cmd);
660 return error(_("cannot fork to run external filter '%s'"),
661 params->cmd);
662 }
663
664 sigchain_push(SIGPIPE, SIG_IGN);
665
666 if (params->src) {
667 write_err = (write_in_full(child_process.in,
668 params->src, params->size) < 0);
669 if (errno == EPIPE)
670 write_err = 0;
671 } else {
672 write_err = copy_fd(params->fd, child_process.in);
673 if (write_err == COPY_WRITE_ERROR && errno == EPIPE)
674 write_err = 0;
675 }
676
677 if (close(child_process.in))
678 write_err = 1;
679 if (write_err)
680 error(_("cannot feed the input to external filter '%s'"),
681 params->cmd);
682
683 sigchain_pop(SIGPIPE);
684
685 status = finish_command(&child_process);
686 if (status)
687 error(_("external filter '%s' failed %d"), params->cmd, status);
688
689 strbuf_release(&cmd);
690 return (write_err || status);
691 }
692
693 static int apply_single_file_filter(const char *path, const char *src, size_t len, int fd,
694 struct strbuf *dst, const char *cmd)
695 {
696 /*
697 * Create a pipeline to have the command filter the buffer's
698 * contents.
699 *
700 * (child --> cmd) --> us
701 */
702 int err = 0;
703 struct strbuf nbuf = STRBUF_INIT;
704 struct async async;
705 struct filter_params params;
706
707 memset(&async, 0, sizeof(async));
708 async.proc = filter_buffer_or_fd;
709 async.data = &params;
710 async.out = -1;
711 params.src = src;
712 params.size = len;
713 params.fd = fd;
714 params.cmd = cmd;
715 params.path = path;
716
717 fflush(NULL);
718 if (start_async(&async))
719 return 0; /* error was already reported */
720
721 if (strbuf_read(&nbuf, async.out, 0) < 0) {
722 err = error(_("read from external filter '%s' failed"), cmd);
723 }
724 if (close(async.out)) {
725 err = error(_("read from external filter '%s' failed"), cmd);
726 }
727 if (finish_async(&async)) {
728 err = error(_("external filter '%s' failed"), cmd);
729 }
730
731 if (!err) {
732 strbuf_swap(dst, &nbuf);
733 }
734 strbuf_release(&nbuf);
735 return !err;
736 }
737
738 #define CAP_CLEAN (1u<<0)
739 #define CAP_SMUDGE (1u<<1)
740 #define CAP_DELAY (1u<<2)
741
742 struct cmd2process {
743 struct subprocess_entry subprocess; /* must be the first member! */
744 unsigned int supported_capabilities;
745 };
746
747 static int subprocess_map_initialized;
748 static struct hashmap subprocess_map;
749
750 static int start_multi_file_filter_fn(struct subprocess_entry *subprocess)
751 {
752 static int versions[] = {2, 0};
753 static struct subprocess_capability capabilities[] = {
754 { "clean", CAP_CLEAN },
755 { "smudge", CAP_SMUDGE },
756 { "delay", CAP_DELAY },
757 { NULL, 0 }
758 };
759 struct cmd2process *entry = (struct cmd2process *)subprocess;
760 return subprocess_handshake(subprocess, "git-filter", versions, NULL,
761 capabilities,
762 &entry->supported_capabilities);
763 }
764
765 static void handle_filter_error(const struct strbuf *filter_status,
766 struct cmd2process *entry,
767 const unsigned int wanted_capability)
768 {
769 if (!strcmp(filter_status->buf, "error"))
770 ; /* The filter signaled a problem with the file. */
771 else if (!strcmp(filter_status->buf, "abort") && wanted_capability) {
772 /*
773 * The filter signaled a permanent problem. Don't try to filter
774 * files with the same command for the lifetime of the current
775 * Git process.
776 */
777 entry->supported_capabilities &= ~wanted_capability;
778 } else {
779 /*
780 * Something went wrong with the protocol filter.
781 * Force shutdown and restart if another blob requires filtering.
782 */
783 error(_("external filter '%s' failed"), entry->subprocess.cmd);
784 subprocess_stop(&subprocess_map, &entry->subprocess);
785 free(entry);
786 }
787 }
788
789 static int apply_multi_file_filter(const char *path, const char *src, size_t len,
790 int fd, struct strbuf *dst, const char *cmd,
791 const unsigned int wanted_capability,
792 const struct checkout_metadata *meta,
793 struct delayed_checkout *dco)
794 {
795 int err;
796 int can_delay = 0;
797 struct cmd2process *entry;
798 struct child_process *process;
799 struct strbuf nbuf = STRBUF_INIT;
800 struct strbuf filter_status = STRBUF_INIT;
801 const char *filter_type;
802
803 if (!subprocess_map_initialized) {
804 subprocess_map_initialized = 1;
805 hashmap_init(&subprocess_map, cmd2process_cmp, NULL, 0);
806 entry = NULL;
807 } else {
808 entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
809 }
810
811 fflush(NULL);
812
813 if (!entry) {
814 entry = xmalloc(sizeof(*entry));
815 entry->supported_capabilities = 0;
816
817 if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_multi_file_filter_fn)) {
818 free(entry);
819 return 0;
820 }
821 }
822 process = &entry->subprocess.process;
823
824 if (!(entry->supported_capabilities & wanted_capability))
825 return 0;
826
827 if (wanted_capability & CAP_CLEAN)
828 filter_type = "clean";
829 else if (wanted_capability & CAP_SMUDGE)
830 filter_type = "smudge";
831 else
832 die(_("unexpected filter type"));
833
834 sigchain_push(SIGPIPE, SIG_IGN);
835
836 assert(strlen(filter_type) < LARGE_PACKET_DATA_MAX - strlen("command=\n"));
837 err = packet_write_fmt_gently(process->in, "command=%s\n", filter_type);
838 if (err)
839 goto done;
840
841 err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
842 if (err) {
843 error(_("path name too long for external filter"));
844 goto done;
845 }
846
847 err = packet_write_fmt_gently(process->in, "pathname=%s\n", path);
848 if (err)
849 goto done;
850
851 if (meta && meta->refname) {
852 err = packet_write_fmt_gently(process->in, "ref=%s\n", meta->refname);
853 if (err)
854 goto done;
855 }
856
857 if (meta && !is_null_oid(&meta->treeish)) {
858 err = packet_write_fmt_gently(process->in, "treeish=%s\n", oid_to_hex(&meta->treeish));
859 if (err)
860 goto done;
861 }
862
863 if (meta && !is_null_oid(&meta->blob)) {
864 err = packet_write_fmt_gently(process->in, "blob=%s\n", oid_to_hex(&meta->blob));
865 if (err)
866 goto done;
867 }
868
869 if ((entry->supported_capabilities & CAP_DELAY) &&
870 dco && dco->state == CE_CAN_DELAY) {
871 can_delay = 1;
872 err = packet_write_fmt_gently(process->in, "can-delay=1\n");
873 if (err)
874 goto done;
875 }
876
877 err = packet_flush_gently(process->in);
878 if (err)
879 goto done;
880
881 if (fd >= 0)
882 err = write_packetized_from_fd_no_flush(fd, process->in);
883 else
884 err = write_packetized_from_buf_no_flush(src, len, process->in);
885 if (err)
886 goto done;
887
888 err = packet_flush_gently(process->in);
889 if (err)
890 goto done;
891
892 err = subprocess_read_status(process->out, &filter_status);
893 if (err)
894 goto done;
895
896 if (can_delay && !strcmp(filter_status.buf, "delayed")) {
897 string_list_insert(&dco->filters, cmd);
898 string_list_insert(&dco->paths, path);
899 } else {
900 /* The filter got the blob and wants to send us a response. */
901 err = strcmp(filter_status.buf, "success");
902 if (err)
903 goto done;
904
905 err = read_packetized_to_strbuf(process->out, &nbuf,
906 PACKET_READ_GENTLE_ON_EOF) < 0;
907 if (err)
908 goto done;
909
910 err = subprocess_read_status(process->out, &filter_status);
911 if (err)
912 goto done;
913
914 err = strcmp(filter_status.buf, "success");
915 }
916
917 done:
918 sigchain_pop(SIGPIPE);
919
920 if (err)
921 handle_filter_error(&filter_status, entry, wanted_capability);
922 else
923 strbuf_swap(dst, &nbuf);
924 strbuf_release(&nbuf);
925 strbuf_release(&filter_status);
926 return !err;
927 }
928
929
930 int async_query_available_blobs(const char *cmd, struct string_list *available_paths)
931 {
932 int err;
933 char *line;
934 struct cmd2process *entry;
935 struct child_process *process;
936 struct strbuf filter_status = STRBUF_INIT;
937
938 assert(subprocess_map_initialized);
939 entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
940 if (!entry) {
941 error(_("external filter '%s' is not available anymore although "
942 "not all paths have been filtered"), cmd);
943 return 0;
944 }
945 process = &entry->subprocess.process;
946 sigchain_push(SIGPIPE, SIG_IGN);
947
948 err = packet_write_fmt_gently(
949 process->in, "command=list_available_blobs\n");
950 if (err)
951 goto done;
952
953 err = packet_flush_gently(process->in);
954 if (err)
955 goto done;
956
957 while ((line = packet_read_line(process->out, NULL))) {
958 const char *path;
959 if (skip_prefix(line, "pathname=", &path))
960 string_list_insert(available_paths, xstrdup(path));
961 else
962 ; /* ignore unknown keys */
963 }
964
965 err = subprocess_read_status(process->out, &filter_status);
966 if (err)
967 goto done;
968
969 err = strcmp(filter_status.buf, "success");
970
971 done:
972 sigchain_pop(SIGPIPE);
973
974 if (err)
975 handle_filter_error(&filter_status, entry, 0);
976 strbuf_release(&filter_status);
977 return !err;
978 }
979
980 static struct convert_driver {
981 const char *name;
982 struct convert_driver *next;
983 const char *smudge;
984 const char *clean;
985 const char *process;
986 int required;
987 } *user_convert, **user_convert_tail;
988
989 static int apply_filter(const char *path, const char *src, size_t len,
990 int fd, struct strbuf *dst, struct convert_driver *drv,
991 const unsigned int wanted_capability,
992 const struct checkout_metadata *meta,
993 struct delayed_checkout *dco)
994 {
995 const char *cmd = NULL;
996
997 if (!drv)
998 return 0;
999
1000 if (!dst)
1001 return 1;
1002
1003 if ((wanted_capability & CAP_CLEAN) && !drv->process && drv->clean)
1004 cmd = drv->clean;
1005 else if ((wanted_capability & CAP_SMUDGE) && !drv->process && drv->smudge)
1006 cmd = drv->smudge;
1007
1008 if (cmd && *cmd)
1009 return apply_single_file_filter(path, src, len, fd, dst, cmd);
1010 else if (drv->process && *drv->process)
1011 return apply_multi_file_filter(path, src, len, fd, dst,
1012 drv->process, wanted_capability, meta, dco);
1013
1014 return 0;
1015 }
1016
1017 static int read_convert_config(const char *var, const char *value, void *cb UNUSED)
1018 {
1019 const char *key, *name;
1020 size_t namelen;
1021 struct convert_driver *drv;
1022
1023 /*
1024 * External conversion drivers are configured using
1025 * "filter.<name>.variable".
1026 */
1027 if (parse_config_key(var, "filter", &name, &namelen, &key) < 0 || !name)
1028 return 0;
1029 for (drv = user_convert; drv; drv = drv->next)
1030 if (!strncmp(drv->name, name, namelen) && !drv->name[namelen])
1031 break;
1032 if (!drv) {
1033 CALLOC_ARRAY(drv, 1);
1034 drv->name = xmemdupz(name, namelen);
1035 *user_convert_tail = drv;
1036 user_convert_tail = &(drv->next);
1037 }
1038
1039 /*
1040 * filter.<name>.smudge and filter.<name>.clean specifies
1041 * the command line:
1042 *
1043 * command-line
1044 *
1045 * The command-line will not be interpolated in any way.
1046 */
1047
1048 if (!strcmp("smudge", key))
1049 return git_config_string(&drv->smudge, var, value);
1050
1051 if (!strcmp("clean", key))
1052 return git_config_string(&drv->clean, var, value);
1053
1054 if (!strcmp("process", key))
1055 return git_config_string(&drv->process, var, value);
1056
1057 if (!strcmp("required", key)) {
1058 drv->required = git_config_bool(var, value);
1059 return 0;
1060 }
1061
1062 return 0;
1063 }
1064
1065 static int count_ident(const char *cp, unsigned long size)
1066 {
1067 /*
1068 * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
1069 */
1070 int cnt = 0;
1071 char ch;
1072
1073 while (size) {
1074 ch = *cp++;
1075 size--;
1076 if (ch != '$')
1077 continue;
1078 if (size < 3)
1079 break;
1080 if (memcmp("Id", cp, 2))
1081 continue;
1082 ch = cp[2];
1083 cp += 3;
1084 size -= 3;
1085 if (ch == '$')
1086 cnt++; /* $Id$ */
1087 if (ch != ':')
1088 continue;
1089
1090 /*
1091 * "$Id: ... "; scan up to the closing dollar sign and discard.
1092 */
1093 while (size) {
1094 ch = *cp++;
1095 size--;
1096 if (ch == '$') {
1097 cnt++;
1098 break;
1099 }
1100 if (ch == '\n')
1101 break;
1102 }
1103 }
1104 return cnt;
1105 }
1106
1107 static int ident_to_git(const char *src, size_t len,
1108 struct strbuf *buf, int ident)
1109 {
1110 char *dst, *dollar;
1111
1112 if (!ident || (src && !count_ident(src, len)))
1113 return 0;
1114
1115 if (!buf)
1116 return 1;
1117
1118 /* only grow if not in place */
1119 if (strbuf_avail(buf) + buf->len < len)
1120 strbuf_grow(buf, len - buf->len);
1121 dst = buf->buf;
1122 for (;;) {
1123 dollar = memchr(src, '$', len);
1124 if (!dollar)
1125 break;
1126 memmove(dst, src, dollar + 1 - src);
1127 dst += dollar + 1 - src;
1128 len -= dollar + 1 - src;
1129 src = dollar + 1;
1130
1131 if (len > 3 && !memcmp(src, "Id:", 3)) {
1132 dollar = memchr(src + 3, '$', len - 3);
1133 if (!dollar)
1134 break;
1135 if (memchr(src + 3, '\n', dollar - src - 3)) {
1136 /* Line break before the next dollar. */
1137 continue;
1138 }
1139
1140 memcpy(dst, "Id$", 3);
1141 dst += 3;
1142 len -= dollar + 1 - src;
1143 src = dollar + 1;
1144 }
1145 }
1146 memmove(dst, src, len);
1147 strbuf_setlen(buf, dst + len - buf->buf);
1148 return 1;
1149 }
1150
1151 static int ident_to_worktree(const char *src, size_t len,
1152 struct strbuf *buf, int ident)
1153 {
1154 struct object_id oid;
1155 char *to_free = NULL, *dollar, *spc;
1156 int cnt;
1157
1158 if (!ident)
1159 return 0;
1160
1161 cnt = count_ident(src, len);
1162 if (!cnt)
1163 return 0;
1164
1165 /* are we "faking" in place editing ? */
1166 if (src == buf->buf)
1167 to_free = strbuf_detach(buf, NULL);
1168 hash_object_file(the_hash_algo, src, len, OBJ_BLOB, &oid);
1169
1170 strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3));
1171 for (;;) {
1172 /* step 1: run to the next '$' */
1173 dollar = memchr(src, '$', len);
1174 if (!dollar)
1175 break;
1176 strbuf_add(buf, src, dollar + 1 - src);
1177 len -= dollar + 1 - src;
1178 src = dollar + 1;
1179
1180 /* step 2: does it looks like a bit like Id:xxx$ or Id$ ? */
1181 if (len < 3 || memcmp("Id", src, 2))
1182 continue;
1183
1184 /* step 3: skip over Id$ or Id:xxxxx$ */
1185 if (src[2] == '$') {
1186 src += 3;
1187 len -= 3;
1188 } else if (src[2] == ':') {
1189 /*
1190 * It's possible that an expanded Id has crept its way into the
1191 * repository, we cope with that by stripping the expansion out.
1192 * This is probably not a good idea, since it will cause changes
1193 * on checkout, which won't go away by stash, but let's keep it
1194 * for git-style ids.
1195 */
1196 dollar = memchr(src + 3, '$', len - 3);
1197 if (!dollar) {
1198 /* incomplete keyword, no more '$', so just quit the loop */
1199 break;
1200 }
1201
1202 if (memchr(src + 3, '\n', dollar - src - 3)) {
1203 /* Line break before the next dollar. */
1204 continue;
1205 }
1206
1207 spc = memchr(src + 4, ' ', dollar - src - 4);
1208 if (spc && spc < dollar-1) {
1209 /* There are spaces in unexpected places.
1210 * This is probably an id from some other
1211 * versioning system. Keep it for now.
1212 */
1213 continue;
1214 }
1215
1216 len -= dollar + 1 - src;
1217 src = dollar + 1;
1218 } else {
1219 /* it wasn't a "Id$" or "Id:xxxx$" */
1220 continue;
1221 }
1222
1223 /* step 4: substitute */
1224 strbuf_addstr(buf, "Id: ");
1225 strbuf_addstr(buf, oid_to_hex(&oid));
1226 strbuf_addstr(buf, " $");
1227 }
1228 strbuf_add(buf, src, len);
1229
1230 free(to_free);
1231 return 1;
1232 }
1233
1234 static const char *git_path_check_encoding(struct attr_check_item *check)
1235 {
1236 const char *value = check->value;
1237
1238 if (ATTR_UNSET(value) || !strlen(value))
1239 return NULL;
1240
1241 if (ATTR_TRUE(value) || ATTR_FALSE(value)) {
1242 die(_("true/false are no valid working-tree-encodings"));
1243 }
1244
1245 /* Don't encode to the default encoding */
1246 if (same_encoding(value, default_encoding))
1247 return NULL;
1248
1249 return value;
1250 }
1251
1252 static enum convert_crlf_action git_path_check_crlf(struct attr_check_item *check)
1253 {
1254 const char *value = check->value;
1255
1256 if (ATTR_TRUE(value))
1257 return CRLF_TEXT;
1258 else if (ATTR_FALSE(value))
1259 return CRLF_BINARY;
1260 else if (ATTR_UNSET(value))
1261 ;
1262 else if (!strcmp(value, "input"))
1263 return CRLF_TEXT_INPUT;
1264 else if (!strcmp(value, "auto"))
1265 return CRLF_AUTO;
1266 return CRLF_UNDEFINED;
1267 }
1268
1269 static enum eol git_path_check_eol(struct attr_check_item *check)
1270 {
1271 const char *value = check->value;
1272
1273 if (ATTR_UNSET(value))
1274 ;
1275 else if (!strcmp(value, "lf"))
1276 return EOL_LF;
1277 else if (!strcmp(value, "crlf"))
1278 return EOL_CRLF;
1279 return EOL_UNSET;
1280 }
1281
1282 static struct convert_driver *git_path_check_convert(struct attr_check_item *check)
1283 {
1284 const char *value = check->value;
1285 struct convert_driver *drv;
1286
1287 if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value))
1288 return NULL;
1289 for (drv = user_convert; drv; drv = drv->next)
1290 if (!strcmp(value, drv->name))
1291 return drv;
1292 return NULL;
1293 }
1294
1295 static int git_path_check_ident(struct attr_check_item *check)
1296 {
1297 const char *value = check->value;
1298
1299 return !!ATTR_TRUE(value);
1300 }
1301
1302 static struct attr_check *check;
1303
1304 void convert_attrs(struct index_state *istate,
1305 struct conv_attrs *ca, const char *path)
1306 {
1307 struct attr_check_item *ccheck = NULL;
1308
1309 if (!check) {
1310 check = attr_check_initl("crlf", "ident", "filter",
1311 "eol", "text", "working-tree-encoding",
1312 NULL);
1313 user_convert_tail = &user_convert;
1314 git_config(read_convert_config, NULL);
1315 }
1316
1317 git_check_attr(istate, NULL, path, check);
1318 ccheck = check->items;
1319 ca->crlf_action = git_path_check_crlf(ccheck + 4);
1320 if (ca->crlf_action == CRLF_UNDEFINED)
1321 ca->crlf_action = git_path_check_crlf(ccheck + 0);
1322 ca->ident = git_path_check_ident(ccheck + 1);
1323 ca->drv = git_path_check_convert(ccheck + 2);
1324 if (ca->crlf_action != CRLF_BINARY) {
1325 enum eol eol_attr = git_path_check_eol(ccheck + 3);
1326 if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_LF)
1327 ca->crlf_action = CRLF_AUTO_INPUT;
1328 else if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_CRLF)
1329 ca->crlf_action = CRLF_AUTO_CRLF;
1330 else if (eol_attr == EOL_LF)
1331 ca->crlf_action = CRLF_TEXT_INPUT;
1332 else if (eol_attr == EOL_CRLF)
1333 ca->crlf_action = CRLF_TEXT_CRLF;
1334 }
1335 ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
1336
1337 /* Save attr and make a decision for action */
1338 ca->attr_action = ca->crlf_action;
1339 if (ca->crlf_action == CRLF_TEXT)
1340 ca->crlf_action = text_eol_is_crlf() ? CRLF_TEXT_CRLF : CRLF_TEXT_INPUT;
1341 if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_FALSE)
1342 ca->crlf_action = CRLF_BINARY;
1343 if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_TRUE)
1344 ca->crlf_action = CRLF_AUTO_CRLF;
1345 if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_INPUT)
1346 ca->crlf_action = CRLF_AUTO_INPUT;
1347 }
1348
1349 void reset_parsed_attributes(void)
1350 {
1351 struct convert_driver *drv, *next;
1352
1353 attr_check_free(check);
1354 check = NULL;
1355 reset_merge_attributes();
1356
1357 for (drv = user_convert; drv; drv = next) {
1358 next = drv->next;
1359 free((void *)drv->name);
1360 free(drv);
1361 }
1362 user_convert = NULL;
1363 user_convert_tail = NULL;
1364 }
1365
1366 int would_convert_to_git_filter_fd(struct index_state *istate, const char *path)
1367 {
1368 struct conv_attrs ca;
1369
1370 convert_attrs(istate, &ca, path);
1371 if (!ca.drv)
1372 return 0;
1373
1374 /*
1375 * Apply a filter to an fd only if the filter is required to succeed.
1376 * We must die if the filter fails, because the original data before
1377 * filtering is not available.
1378 */
1379 if (!ca.drv->required)
1380 return 0;
1381
1382 return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL, NULL);
1383 }
1384
1385 const char *get_convert_attr_ascii(struct index_state *istate, const char *path)
1386 {
1387 struct conv_attrs ca;
1388
1389 convert_attrs(istate, &ca, path);
1390 switch (ca.attr_action) {
1391 case CRLF_UNDEFINED:
1392 return "";
1393 case CRLF_BINARY:
1394 return "-text";
1395 case CRLF_TEXT:
1396 return "text";
1397 case CRLF_TEXT_INPUT:
1398 return "text eol=lf";
1399 case CRLF_TEXT_CRLF:
1400 return "text eol=crlf";
1401 case CRLF_AUTO:
1402 return "text=auto";
1403 case CRLF_AUTO_CRLF:
1404 return "text=auto eol=crlf";
1405 case CRLF_AUTO_INPUT:
1406 return "text=auto eol=lf";
1407 }
1408 return "";
1409 }
1410
1411 int convert_to_git(struct index_state *istate,
1412 const char *path, const char *src, size_t len,
1413 struct strbuf *dst, int conv_flags)
1414 {
1415 int ret = 0;
1416 struct conv_attrs ca;
1417
1418 convert_attrs(istate, &ca, path);
1419
1420 ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN, NULL, NULL);
1421 if (!ret && ca.drv && ca.drv->required)
1422 die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1423
1424 if (ret && dst) {
1425 src = dst->buf;
1426 len = dst->len;
1427 }
1428
1429 ret |= encode_to_git(path, src, len, dst, ca.working_tree_encoding, conv_flags);
1430 if (ret && dst) {
1431 src = dst->buf;
1432 len = dst->len;
1433 }
1434
1435 if (!(conv_flags & CONV_EOL_KEEP_CRLF)) {
1436 ret |= crlf_to_git(istate, path, src, len, dst, ca.crlf_action, conv_flags);
1437 if (ret && dst) {
1438 src = dst->buf;
1439 len = dst->len;
1440 }
1441 }
1442 return ret | ident_to_git(src, len, dst, ca.ident);
1443 }
1444
1445 void convert_to_git_filter_fd(struct index_state *istate,
1446 const char *path, int fd, struct strbuf *dst,
1447 int conv_flags)
1448 {
1449 struct conv_attrs ca;
1450 convert_attrs(istate, &ca, path);
1451
1452 assert(ca.drv);
1453
1454 if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL, NULL))
1455 die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1456
1457 encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags);
1458 crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags);
1459 ident_to_git(dst->buf, dst->len, dst, ca.ident);
1460 }
1461
1462 static int convert_to_working_tree_ca_internal(const struct conv_attrs *ca,
1463 const char *path, const char *src,
1464 size_t len, struct strbuf *dst,
1465 int normalizing,
1466 const struct checkout_metadata *meta,
1467 struct delayed_checkout *dco)
1468 {
1469 int ret = 0, ret_filter = 0;
1470
1471 ret |= ident_to_worktree(src, len, dst, ca->ident);
1472 if (ret) {
1473 src = dst->buf;
1474 len = dst->len;
1475 }
1476 /*
1477 * CRLF conversion can be skipped if normalizing, unless there
1478 * is a smudge or process filter (even if the process filter doesn't
1479 * support smudge). The filters might expect CRLFs.
1480 */
1481 if ((ca->drv && (ca->drv->smudge || ca->drv->process)) || !normalizing) {
1482 ret |= crlf_to_worktree(src, len, dst, ca->crlf_action);
1483 if (ret) {
1484 src = dst->buf;
1485 len = dst->len;
1486 }
1487 }
1488
1489 ret |= encode_to_worktree(path, src, len, dst, ca->working_tree_encoding);
1490 if (ret) {
1491 src = dst->buf;
1492 len = dst->len;
1493 }
1494
1495 ret_filter = apply_filter(
1496 path, src, len, -1, dst, ca->drv, CAP_SMUDGE, meta, dco);
1497 if (!ret_filter && ca->drv && ca->drv->required)
1498 die(_("%s: smudge filter %s failed"), path, ca->drv->name);
1499
1500 return ret | ret_filter;
1501 }
1502
1503 int async_convert_to_working_tree_ca(const struct conv_attrs *ca,
1504 const char *path, const char *src,
1505 size_t len, struct strbuf *dst,
1506 const struct checkout_metadata *meta,
1507 void *dco)
1508 {
1509 return convert_to_working_tree_ca_internal(ca, path, src, len, dst, 0,
1510 meta, dco);
1511 }
1512
1513 int convert_to_working_tree_ca(const struct conv_attrs *ca,
1514 const char *path, const char *src,
1515 size_t len, struct strbuf *dst,
1516 const struct checkout_metadata *meta)
1517 {
1518 return convert_to_working_tree_ca_internal(ca, path, src, len, dst, 0,
1519 meta, NULL);
1520 }
1521
1522 int renormalize_buffer(struct index_state *istate, const char *path,
1523 const char *src, size_t len, struct strbuf *dst)
1524 {
1525 struct conv_attrs ca;
1526 int ret;
1527
1528 convert_attrs(istate, &ca, path);
1529 ret = convert_to_working_tree_ca_internal(&ca, path, src, len, dst, 1,
1530 NULL, NULL);
1531 if (ret) {
1532 src = dst->buf;
1533 len = dst->len;
1534 }
1535 return ret | convert_to_git(istate, path, src, len, dst, CONV_EOL_RENORMALIZE);
1536 }
1537
1538 /*****************************************************************
1539 *
1540 * Streaming conversion support
1541 *
1542 *****************************************************************/
1543
1544 typedef int (*filter_fn)(struct stream_filter *,
1545 const char *input, size_t *isize_p,
1546 char *output, size_t *osize_p);
1547 typedef void (*free_fn)(struct stream_filter *);
1548
1549 struct stream_filter_vtbl {
1550 filter_fn filter;
1551 free_fn free;
1552 };
1553
1554 struct stream_filter {
1555 struct stream_filter_vtbl *vtbl;
1556 };
1557
1558 static int null_filter_fn(struct stream_filter *filter UNUSED,
1559 const char *input, size_t *isize_p,
1560 char *output, size_t *osize_p)
1561 {
1562 size_t count;
1563
1564 if (!input)
1565 return 0; /* we do not keep any states */
1566 count = *isize_p;
1567 if (*osize_p < count)
1568 count = *osize_p;
1569 if (count) {
1570 memmove(output, input, count);
1571 *isize_p -= count;
1572 *osize_p -= count;
1573 }
1574 return 0;
1575 }
1576
1577 static void null_free_fn(struct stream_filter *filter UNUSED)
1578 {
1579 ; /* nothing -- null instances are shared */
1580 }
1581
1582 static struct stream_filter_vtbl null_vtbl = {
1583 .filter = null_filter_fn,
1584 .free = null_free_fn,
1585 };
1586
1587 static struct stream_filter null_filter_singleton = {
1588 .vtbl = &null_vtbl,
1589 };
1590
1591 int is_null_stream_filter(struct stream_filter *filter)
1592 {
1593 return filter == &null_filter_singleton;
1594 }
1595
1596
1597 /*
1598 * LF-to-CRLF filter
1599 */
1600
1601 struct lf_to_crlf_filter {
1602 struct stream_filter filter;
1603 unsigned has_held:1;
1604 char held;
1605 };
1606
1607 static int lf_to_crlf_filter_fn(struct stream_filter *filter,
1608 const char *input, size_t *isize_p,
1609 char *output, size_t *osize_p)
1610 {
1611 size_t count, o = 0;
1612 struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter;
1613
1614 /*
1615 * We may be holding onto the CR to see if it is followed by a
1616 * LF, in which case we would need to go to the main loop.
1617 * Otherwise, just emit it to the output stream.
1618 */
1619 if (lf_to_crlf->has_held && (lf_to_crlf->held != '\r' || !input)) {
1620 output[o++] = lf_to_crlf->held;
1621 lf_to_crlf->has_held = 0;
1622 }
1623
1624 /* We are told to drain */
1625 if (!input) {
1626 *osize_p -= o;
1627 return 0;
1628 }
1629
1630 count = *isize_p;
1631 if (count || lf_to_crlf->has_held) {
1632 size_t i;
1633 int was_cr = 0;
1634
1635 if (lf_to_crlf->has_held) {
1636 was_cr = 1;
1637 lf_to_crlf->has_held = 0;
1638 }
1639
1640 for (i = 0; o < *osize_p && i < count; i++) {
1641 char ch = input[i];
1642
1643 if (ch == '\n') {
1644 output[o++] = '\r';
1645 } else if (was_cr) {
1646 /*
1647 * Previous round saw CR and it is not followed
1648 * by a LF; emit the CR before processing the
1649 * current character.
1650 */
1651 output[o++] = '\r';
1652 }
1653
1654 /*
1655 * We may have consumed the last output slot,
1656 * in which case we need to break out of this
1657 * loop; hold the current character before
1658 * returning.
1659 */
1660 if (*osize_p <= o) {
1661 lf_to_crlf->has_held = 1;
1662 lf_to_crlf->held = ch;
1663 continue; /* break but increment i */
1664 }
1665
1666 if (ch == '\r') {
1667 was_cr = 1;
1668 continue;
1669 }
1670
1671 was_cr = 0;
1672 output[o++] = ch;
1673 }
1674
1675 *osize_p -= o;
1676 *isize_p -= i;
1677
1678 if (!lf_to_crlf->has_held && was_cr) {
1679 lf_to_crlf->has_held = 1;
1680 lf_to_crlf->held = '\r';
1681 }
1682 }
1683 return 0;
1684 }
1685
1686 static void lf_to_crlf_free_fn(struct stream_filter *filter)
1687 {
1688 free(filter);
1689 }
1690
1691 static struct stream_filter_vtbl lf_to_crlf_vtbl = {
1692 .filter = lf_to_crlf_filter_fn,
1693 .free = lf_to_crlf_free_fn,
1694 };
1695
1696 static struct stream_filter *lf_to_crlf_filter(void)
1697 {
1698 struct lf_to_crlf_filter *lf_to_crlf = xcalloc(1, sizeof(*lf_to_crlf));
1699
1700 lf_to_crlf->filter.vtbl = &lf_to_crlf_vtbl;
1701 return (struct stream_filter *)lf_to_crlf;
1702 }
1703
1704 /*
1705 * Cascade filter
1706 */
1707 #define FILTER_BUFFER 1024
1708 struct cascade_filter {
1709 struct stream_filter filter;
1710 struct stream_filter *one;
1711 struct stream_filter *two;
1712 char buf[FILTER_BUFFER];
1713 int end, ptr;
1714 };
1715
1716 static int cascade_filter_fn(struct stream_filter *filter,
1717 const char *input, size_t *isize_p,
1718 char *output, size_t *osize_p)
1719 {
1720 struct cascade_filter *cas = (struct cascade_filter *) filter;
1721 size_t filled = 0;
1722 size_t sz = *osize_p;
1723 size_t to_feed, remaining;
1724
1725 /*
1726 * input -- (one) --> buf -- (two) --> output
1727 */
1728 while (filled < sz) {
1729 remaining = sz - filled;
1730
1731 /* do we already have something to feed two with? */
1732 if (cas->ptr < cas->end) {
1733 to_feed = cas->end - cas->ptr;
1734 if (stream_filter(cas->two,
1735 cas->buf + cas->ptr, &to_feed,
1736 output + filled, &remaining))
1737 return -1;
1738 cas->ptr += (cas->end - cas->ptr) - to_feed;
1739 filled = sz - remaining;
1740 continue;
1741 }
1742
1743 /* feed one from upstream and have it emit into our buffer */
1744 to_feed = input ? *isize_p : 0;
1745 if (input && !to_feed)
1746 break;
1747 remaining = sizeof(cas->buf);
1748 if (stream_filter(cas->one,
1749 input, &to_feed,
1750 cas->buf, &remaining))
1751 return -1;
1752 cas->end = sizeof(cas->buf) - remaining;
1753 cas->ptr = 0;
1754 if (input) {
1755 size_t fed = *isize_p - to_feed;
1756 *isize_p -= fed;
1757 input += fed;
1758 }
1759
1760 /* do we know that we drained one completely? */
1761 if (input || cas->end)
1762 continue;
1763
1764 /* tell two to drain; we have nothing more to give it */
1765 to_feed = 0;
1766 remaining = sz - filled;
1767 if (stream_filter(cas->two,
1768 NULL, &to_feed,
1769 output + filled, &remaining))
1770 return -1;
1771 if (remaining == (sz - filled))
1772 break; /* completely drained two */
1773 filled = sz - remaining;
1774 }
1775 *osize_p -= filled;
1776 return 0;
1777 }
1778
1779 static void cascade_free_fn(struct stream_filter *filter)
1780 {
1781 struct cascade_filter *cas = (struct cascade_filter *)filter;
1782 free_stream_filter(cas->one);
1783 free_stream_filter(cas->two);
1784 free(filter);
1785 }
1786
1787 static struct stream_filter_vtbl cascade_vtbl = {
1788 .filter = cascade_filter_fn,
1789 .free = cascade_free_fn,
1790 };
1791
1792 static struct stream_filter *cascade_filter(struct stream_filter *one,
1793 struct stream_filter *two)
1794 {
1795 struct cascade_filter *cascade;
1796
1797 if (!one || is_null_stream_filter(one))
1798 return two;
1799 if (!two || is_null_stream_filter(two))
1800 return one;
1801
1802 cascade = xmalloc(sizeof(*cascade));
1803 cascade->one = one;
1804 cascade->two = two;
1805 cascade->end = cascade->ptr = 0;
1806 cascade->filter.vtbl = &cascade_vtbl;
1807 return (struct stream_filter *)cascade;
1808 }
1809
1810 /*
1811 * ident filter
1812 */
1813 #define IDENT_DRAINING (-1)
1814 #define IDENT_SKIPPING (-2)
1815 struct ident_filter {
1816 struct stream_filter filter;
1817 struct strbuf left;
1818 int state;
1819 char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */
1820 };
1821
1822 static int is_foreign_ident(const char *str)
1823 {
1824 int i;
1825
1826 if (!skip_prefix(str, "$Id: ", &str))
1827 return 0;
1828 for (i = 0; str[i]; i++) {
1829 if (isspace(str[i]) && str[i+1] != '$')
1830 return 1;
1831 }
1832 return 0;
1833 }
1834
1835 static void ident_drain(struct ident_filter *ident, char **output_p, size_t *osize_p)
1836 {
1837 size_t to_drain = ident->left.len;
1838
1839 if (*osize_p < to_drain)
1840 to_drain = *osize_p;
1841 if (to_drain) {
1842 memcpy(*output_p, ident->left.buf, to_drain);
1843 strbuf_remove(&ident->left, 0, to_drain);
1844 *output_p += to_drain;
1845 *osize_p -= to_drain;
1846 }
1847 if (!ident->left.len)
1848 ident->state = 0;
1849 }
1850
1851 static int ident_filter_fn(struct stream_filter *filter,
1852 const char *input, size_t *isize_p,
1853 char *output, size_t *osize_p)
1854 {
1855 struct ident_filter *ident = (struct ident_filter *)filter;
1856 static const char head[] = "$Id";
1857
1858 if (!input) {
1859 /* drain upon eof */
1860 switch (ident->state) {
1861 default:
1862 strbuf_add(&ident->left, head, ident->state);
1863 /* fallthrough */
1864 case IDENT_SKIPPING:
1865 /* fallthrough */
1866 case IDENT_DRAINING:
1867 ident_drain(ident, &output, osize_p);
1868 }
1869 return 0;
1870 }
1871
1872 while (*isize_p || (ident->state == IDENT_DRAINING)) {
1873 int ch;
1874
1875 if (ident->state == IDENT_DRAINING) {
1876 ident_drain(ident, &output, osize_p);
1877 if (!*osize_p)
1878 break;
1879 continue;
1880 }
1881
1882 ch = *(input++);
1883 (*isize_p)--;
1884
1885 if (ident->state == IDENT_SKIPPING) {
1886 /*
1887 * Skipping until '$' or LF, but keeping them
1888 * in case it is a foreign ident.
1889 */
1890 strbuf_addch(&ident->left, ch);
1891 if (ch != '\n' && ch != '$')
1892 continue;
1893 if (ch == '$' && !is_foreign_ident(ident->left.buf)) {
1894 strbuf_setlen(&ident->left, sizeof(head) - 1);
1895 strbuf_addstr(&ident->left, ident->ident);
1896 }
1897 ident->state = IDENT_DRAINING;
1898 continue;
1899 }
1900
1901 if (ident->state < sizeof(head) &&
1902 head[ident->state] == ch) {
1903 ident->state++;
1904 continue;
1905 }
1906
1907 if (ident->state)
1908 strbuf_add(&ident->left, head, ident->state);
1909 if (ident->state == sizeof(head) - 1) {
1910 if (ch != ':' && ch != '$') {
1911 strbuf_addch(&ident->left, ch);
1912 ident->state = 0;
1913 continue;
1914 }
1915
1916 if (ch == ':') {
1917 strbuf_addch(&ident->left, ch);
1918 ident->state = IDENT_SKIPPING;
1919 } else {
1920 strbuf_addstr(&ident->left, ident->ident);
1921 ident->state = IDENT_DRAINING;
1922 }
1923 continue;
1924 }
1925
1926 strbuf_addch(&ident->left, ch);
1927 ident->state = IDENT_DRAINING;
1928 }
1929 return 0;
1930 }
1931
1932 static void ident_free_fn(struct stream_filter *filter)
1933 {
1934 struct ident_filter *ident = (struct ident_filter *)filter;
1935 strbuf_release(&ident->left);
1936 free(filter);
1937 }
1938
1939 static struct stream_filter_vtbl ident_vtbl = {
1940 .filter = ident_filter_fn,
1941 .free = ident_free_fn,
1942 };
1943
1944 static struct stream_filter *ident_filter(const struct object_id *oid)
1945 {
1946 struct ident_filter *ident = xmalloc(sizeof(*ident));
1947
1948 xsnprintf(ident->ident, sizeof(ident->ident),
1949 ": %s $", oid_to_hex(oid));
1950 strbuf_init(&ident->left, 0);
1951 ident->filter.vtbl = &ident_vtbl;
1952 ident->state = 0;
1953 return (struct stream_filter *)ident;
1954 }
1955
1956 /*
1957 * Return an appropriately constructed filter for the given ca, or NULL if
1958 * the contents cannot be filtered without reading the whole thing
1959 * in-core.
1960 *
1961 * Note that you would be crazy to set CRLF, smudge/clean or ident to a
1962 * large binary blob you would want us not to slurp into the memory!
1963 */
1964 struct stream_filter *get_stream_filter_ca(const struct conv_attrs *ca,
1965 const struct object_id *oid)
1966 {
1967 struct stream_filter *filter = NULL;
1968
1969 if (classify_conv_attrs(ca) != CA_CLASS_STREAMABLE)
1970 return NULL;
1971
1972 if (ca->ident)
1973 filter = ident_filter(oid);
1974
1975 if (output_eol(ca->crlf_action) == EOL_CRLF)
1976 filter = cascade_filter(filter, lf_to_crlf_filter());
1977 else
1978 filter = cascade_filter(filter, &null_filter_singleton);
1979
1980 return filter;
1981 }
1982
1983 struct stream_filter *get_stream_filter(struct index_state *istate,
1984 const char *path,
1985 const struct object_id *oid)
1986 {
1987 struct conv_attrs ca;
1988 convert_attrs(istate, &ca, path);
1989 return get_stream_filter_ca(&ca, oid);
1990 }
1991
1992 void free_stream_filter(struct stream_filter *filter)
1993 {
1994 filter->vtbl->free(filter);
1995 }
1996
1997 int stream_filter(struct stream_filter *filter,
1998 const char *input, size_t *isize_p,
1999 char *output, size_t *osize_p)
2000 {
2001 return filter->vtbl->filter(filter, input, isize_p, output, osize_p);
2002 }
2003
2004 void init_checkout_metadata(struct checkout_metadata *meta, const char *refname,
2005 const struct object_id *treeish,
2006 const struct object_id *blob)
2007 {
2008 memset(meta, 0, sizeof(*meta));
2009 if (refname)
2010 meta->refname = refname;
2011 if (treeish)
2012 oidcpy(&meta->treeish, treeish);
2013 if (blob)
2014 oidcpy(&meta->blob, blob);
2015 }
2016
2017 void clone_checkout_metadata(struct checkout_metadata *dst,
2018 const struct checkout_metadata *src,
2019 const struct object_id *blob)
2020 {
2021 memcpy(dst, src, sizeof(*dst));
2022 if (blob)
2023 oidcpy(&dst->blob, blob);
2024 }
2025
2026 enum conv_attrs_classification classify_conv_attrs(const struct conv_attrs *ca)
2027 {
2028 if (ca->drv) {
2029 if (ca->drv->process)
2030 return CA_CLASS_INCORE_PROCESS;
2031 if (ca->drv->smudge || ca->drv->clean)
2032 return CA_CLASS_INCORE_FILTER;
2033 }
2034
2035 if (ca->working_tree_encoding)
2036 return CA_CLASS_INCORE;
2037
2038 if (ca->crlf_action == CRLF_AUTO || ca->crlf_action == CRLF_AUTO_CRLF)
2039 return CA_CLASS_INCORE;
2040
2041 return CA_CLASS_STREAMABLE;
2042 }
Alternate source for your git.git needs