notes doc: split up run-on sentences

[git.git] / fetch-pack.c

#include "cache.h"
#include "alloc.h"
#include "repository.h"
#include "config.h"
#include "environment.h"
#include "gettext.h"
#include "hex.h"
#include "lockfile.h"
#include "refs.h"
#include "pkt-line.h"
#include "commit.h"
#include "tag.h"
#include "exec-cmd.h"
#include "pack.h"
#include "sideband.h"
#include "fetch-pack.h"
#include "remote.h"
#include "run-command.h"
#include "connect.h"
#include "transport.h"
#include "version.h"
#include "oid-array.h"
#include "oidset.h"
#include "packfile.h"
#include "object-store.h"
#include "connected.h"
#include "fetch-negotiator.h"
#include "fsck.h"
#include "shallow.h"
#include "commit-reach.h"
#include "commit-graph.h"
#include "sigchain.h"
#include "mergesort.h"
#include "wrapper.h"

static int transfer_unpack_limit = -1;
static int fetch_unpack_limit = -1;
static int unpack_limit = 100;
static int prefer_ofs_delta = 1;
static int no_done;
static int deepen_since_ok;
static int deepen_not_ok;
static int fetch_fsck_objects = -1;
static int transfer_fsck_objects = -1;
static int agent_supported;
static int server_supports_filtering;
static int advertise_sid;
static struct shallow_lock shallow_lock;
static const char *alternate_shallow_file;
static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
static struct strbuf fsck_msg_types = STRBUF_INIT;
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;

/* Remember to update object flag allocation in object.h */
#define COMPLETE        (1U << 0)
#define ALTERNATE       (1U << 1)
#define COMMON          (1U << 6)
#define REACH_SCRATCH   (1U << 7)

/*
 * After sending this many "have"s if we do not get any new ACK , we
 * give up traversing our history.
 */
#define MAX_IN_VAIN 256

static int multi_ack, use_sideband;
/* Allow specifying sha1 if it is a ref tip. */
#define ALLOW_TIP_SHA1  01
/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
#define ALLOW_REACHABLE_SHA1    02
static unsigned int allow_unadvertised_object_request;

__attribute__((format (printf, 2, 3)))
static inline void print_verbose(const struct fetch_pack_args *args,
                                 const char *fmt, ...)
{
        va_list params;

        if (!args->verbose)
                return;

        va_start(params, fmt);
        vfprintf(stderr, fmt, params);
        va_end(params);
        fputc('\n', stderr);
}

struct alternate_object_cache {
        struct object **items;
        size_t nr, alloc;
};

static void cache_one_alternate(const struct object_id *oid,
                                void *vcache)
{
        struct alternate_object_cache *cache = vcache;
        struct object *obj = parse_object(the_repository, oid);

        if (!obj || (obj->flags & ALTERNATE))
                return;

        obj->flags |= ALTERNATE;
        ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
        cache->items[cache->nr++] = obj;
}

static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
                                      void (*cb)(struct fetch_negotiator *,
                                                 struct object *))
{
        static int initialized;
        static struct alternate_object_cache cache;
        size_t i;

        if (!initialized) {
                for_each_alternate_ref(cache_one_alternate, &cache);
                initialized = 1;
        }

        for (i = 0; i < cache.nr; i++)
                cb(negotiator, cache.items[i]);
}

static struct commit *deref_without_lazy_fetch_extended(const struct object_id *oid,
                                                        int mark_tags_complete,
                                                        enum object_type *type,
                                                        unsigned int oi_flags)
{
        struct object_info info = { .typep = type };
        struct commit *commit;

        commit = lookup_commit_in_graph(the_repository, oid);
        if (commit)
                return commit;

        while (1) {
                if (oid_object_info_extended(the_repository, oid, &info,
                                             oi_flags))
                        return NULL;
                if (*type == OBJ_TAG) {
                        struct tag *tag = (struct tag *)
                                parse_object(the_repository, oid);

                        if (!tag->tagged)
                                return NULL;
                        if (mark_tags_complete)
                                tag->object.flags |= COMPLETE;
                        oid = &tag->tagged->oid;
                } else {
                        break;
                }
        }

        if (*type == OBJ_COMMIT) {
                struct commit *commit = lookup_commit(the_repository, oid);
                if (!commit || repo_parse_commit(the_repository, commit))
                        return NULL;
                return commit;
        }

        return NULL;
}


static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
                                               int mark_tags_complete)
{
        enum object_type type;
        unsigned flags = OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK;
        return deref_without_lazy_fetch_extended(oid, mark_tags_complete,
                                                 &type, flags);
}

static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
                               const struct object_id *oid)
{
        struct commit *c = deref_without_lazy_fetch(oid, 0);

        if (c)
                negotiator->add_tip(negotiator, c);
        return 0;
}

static int rev_list_insert_ref_oid(const char *refname UNUSED,
                                   const struct object_id *oid,
                                   int flag UNUSED,
                                   void *cb_data)
{
        return rev_list_insert_ref(cb_data, oid);
}

enum ack_type {
        NAK = 0,
        ACK,
        ACK_continue,
        ACK_common,
        ACK_ready
};

static void consume_shallow_list(struct fetch_pack_args *args,
                                 struct packet_reader *reader)
{
        if (args->stateless_rpc && args->deepen) {
                /* If we sent a depth we will get back "duplicate"
                 * shallow and unshallow commands every time there
                 * is a block of have lines exchanged.
                 */
                while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
                        if (starts_with(reader->line, "shallow "))
                                continue;
                        if (starts_with(reader->line, "unshallow "))
                                continue;
                        die(_("git fetch-pack: expected shallow list"));
                }
                if (reader->status != PACKET_READ_FLUSH)
                        die(_("git fetch-pack: expected a flush packet after shallow list"));
        }
}

static enum ack_type get_ack(struct packet_reader *reader,
                             struct object_id *result_oid)
{
        int len;
        const char *arg;

        if (packet_reader_read(reader) != PACKET_READ_NORMAL)
                die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
        len = reader->pktlen;

        if (!strcmp(reader->line, "NAK"))
                return NAK;
        if (skip_prefix(reader->line, "ACK ", &arg)) {
                const char *p;
                if (!parse_oid_hex(arg, result_oid, &p)) {
                        len -= p - reader->line;
                        if (len < 1)
                                return ACK;
                        if (strstr(p, "continue"))
                                return ACK_continue;
                        if (strstr(p, "common"))
                                return ACK_common;
                        if (strstr(p, "ready"))
                                return ACK_ready;
                        return ACK;
                }
        }
        die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
}

static void send_request(struct fetch_pack_args *args,
                         int fd, struct strbuf *buf)
{
        if (args->stateless_rpc) {
                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
                packet_flush(fd);
        } else {
                if (write_in_full(fd, buf->buf, buf->len) < 0)
                        die_errno(_("unable to write to remote"));
        }
}

static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
                                        struct object *obj)
{
        rev_list_insert_ref(negotiator, &obj->oid);
}

#define INITIAL_FLUSH 16
#define PIPESAFE_FLUSH 32
#define LARGE_FLUSH 16384

static int next_flush(int stateless_rpc, int count)
{
        if (stateless_rpc) {
                if (count < LARGE_FLUSH)
                        count <<= 1;
                else
                        count = count * 11 / 10;
        } else {
                if (count < PIPESAFE_FLUSH)
                        count <<= 1;
                else
                        count += PIPESAFE_FLUSH;
        }
        return count;
}

static void mark_tips(struct fetch_negotiator *negotiator,
                      const struct oid_array *negotiation_tips)
{
        int i;

        if (!negotiation_tips) {
                for_each_rawref(rev_list_insert_ref_oid, negotiator);
                return;
        }

        for (i = 0; i < negotiation_tips->nr; i++)
                rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
        return;
}

static void send_filter(struct fetch_pack_args *args,
                        struct strbuf *req_buf,
                        int server_supports_filter)
{
        if (args->filter_options.choice) {
                const char *spec =
                        expand_list_objects_filter_spec(&args->filter_options);
                if (server_supports_filter) {
                        print_verbose(args, _("Server supports filter"));
                        packet_buf_write(req_buf, "filter %s", spec);
                        trace2_data_string("fetch", the_repository,
                                           "filter/effective", spec);
                } else {
                        warning("filtering not recognized by server, ignoring");
                        trace2_data_string("fetch", the_repository,
                                           "filter/unsupported", spec);
                }
        } else {
                trace2_data_string("fetch", the_repository,
                                   "filter/none", "");
        }
}

static int find_common(struct fetch_negotiator *negotiator,
                       struct fetch_pack_args *args,
                       int fd[2], struct object_id *result_oid,
                       struct ref *refs)
{
        int fetching;
        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
        int negotiation_round = 0, haves = 0;
        const struct object_id *oid;
        unsigned in_vain = 0;
        int got_continue = 0;
        int got_ready = 0;
        struct strbuf req_buf = STRBUF_INIT;
        size_t state_len = 0;
        struct packet_reader reader;

        if (args->stateless_rpc && multi_ack == 1)
                die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");

        packet_reader_init(&reader, fd[0], NULL, 0,
                           PACKET_READ_CHOMP_NEWLINE |
                           PACKET_READ_DIE_ON_ERR_PACKET);

        mark_tips(negotiator, args->negotiation_tips);
        for_each_cached_alternate(negotiator, insert_one_alternate_object);

        fetching = 0;
        for ( ; refs ; refs = refs->next) {
                struct object_id *remote = &refs->old_oid;
                const char *remote_hex;
                struct object *o;

                if (!args->refetch) {
                        /*
                        * If that object is complete (i.e. it is an ancestor of a
                        * local ref), we tell them we have it but do not have to
                        * tell them about its ancestors, which they already know
                        * about.
                        *
                        * We use lookup_object here because we are only
                        * interested in the case we *know* the object is
                        * reachable and we have already scanned it.
                        */
                        if (((o = lookup_object(the_repository, remote)) != NULL) &&
                                        (o->flags & COMPLETE)) {
                                continue;
                        }
                }

                remote_hex = oid_to_hex(remote);
                if (!fetching) {
                        struct strbuf c = STRBUF_INIT;
                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
                        if (no_done)            strbuf_addstr(&c, " no-done");
                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
                        if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
                        if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
                        if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
                                                            git_user_agent_sanitized());
                        if (advertise_sid)
                                strbuf_addf(&c, " session-id=%s", trace2_session_id());
                        if (args->filter_options.choice)
                                strbuf_addstr(&c, " filter");
                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
                        strbuf_release(&c);
                } else
                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
                fetching++;
        }

        if (!fetching) {
                strbuf_release(&req_buf);
                packet_flush(fd[1]);
                return 1;
        }

        if (is_repository_shallow(the_repository))
                write_shallow_commits(&req_buf, 1, NULL);
        if (args->depth > 0)
                packet_buf_write(&req_buf, "deepen %d", args->depth);
        if (args->deepen_since) {
                timestamp_t max_age = approxidate(args->deepen_since);
                packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
        }
        if (args->deepen_not) {
                int i;
                for (i = 0; i < args->deepen_not->nr; i++) {
                        struct string_list_item *s = args->deepen_not->items + i;
                        packet_buf_write(&req_buf, "deepen-not %s", s->string);
                }
        }
        send_filter(args, &req_buf, server_supports_filtering);
        packet_buf_flush(&req_buf);
        state_len = req_buf.len;

        if (args->deepen) {
                const char *arg;
                struct object_id oid;

                send_request(args, fd[1], &req_buf);
                while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
                        if (skip_prefix(reader.line, "shallow ", &arg)) {
                                if (get_oid_hex(arg, &oid))
                                        die(_("invalid shallow line: %s"), reader.line);
                                register_shallow(the_repository, &oid);
                                continue;
                        }
                        if (skip_prefix(reader.line, "unshallow ", &arg)) {
                                if (get_oid_hex(arg, &oid))
                                        die(_("invalid unshallow line: %s"), reader.line);
                                if (!lookup_object(the_repository, &oid))
                                        die(_("object not found: %s"), reader.line);
                                /* make sure that it is parsed as shallow */
                                if (!parse_object(the_repository, &oid))
                                        die(_("error in object: %s"), reader.line);
                                if (unregister_shallow(&oid))
                                        die(_("no shallow found: %s"), reader.line);
                                continue;
                        }
                        die(_("expected shallow/unshallow, got %s"), reader.line);
                }
        } else if (!args->stateless_rpc)
                send_request(args, fd[1], &req_buf);

        if (!args->stateless_rpc) {
                /* If we aren't using the stateless-rpc interface
                 * we don't need to retain the headers.
                 */
                strbuf_setlen(&req_buf, 0);
                state_len = 0;
        }

        trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
        flushes = 0;
        retval = -1;
        while ((oid = negotiator->next(negotiator))) {
                packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
                print_verbose(args, "have %s", oid_to_hex(oid));
                in_vain++;
                haves++;
                if (flush_at <= ++count) {
                        int ack;

                        negotiation_round++;
                        trace2_region_enter_printf("negotiation_v0_v1", "round",
                                                   the_repository, "%d",
                                                   negotiation_round);
                        trace2_data_intmax("negotiation_v0_v1", the_repository,
                                           "haves_added", haves);
                        trace2_data_intmax("negotiation_v0_v1", the_repository,
                                           "in_vain", in_vain);
                        haves = 0;
                        packet_buf_flush(&req_buf);
                        send_request(args, fd[1], &req_buf);
                        strbuf_setlen(&req_buf, state_len);
                        flushes++;
                        flush_at = next_flush(args->stateless_rpc, count);

                        /*
                         * We keep one window "ahead" of the other side, and
                         * will wait for an ACK only on the next one
                         */
                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
                                continue;

                        consume_shallow_list(args, &reader);
                        do {
                                ack = get_ack(&reader, result_oid);
                                if (ack)
                                        print_verbose(args, _("got %s %d %s"), "ack",
                                                      ack, oid_to_hex(result_oid));
                                switch (ack) {
                                case ACK:
                                        trace2_region_leave_printf("negotiation_v0_v1", "round",
                                                                   the_repository, "%d",
                                                                   negotiation_round);
                                        flushes = 0;
                                        multi_ack = 0;
                                        retval = 0;
                                        goto done;
                                case ACK_common:
                                case ACK_ready:
                                case ACK_continue: {
                                        struct commit *commit =
                                                lookup_commit(the_repository,
                                                              result_oid);
                                        int was_common;

                                        if (!commit)
                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
                                        was_common = negotiator->ack(negotiator, commit);
                                        if (args->stateless_rpc
                                         && ack == ACK_common
                                         && !was_common) {
                                                /* We need to replay the have for this object
                                                 * on the next RPC request so the peer knows
                                                 * it is in common with us.
                                                 */
                                                const char *hex = oid_to_hex(result_oid);
                                                packet_buf_write(&req_buf, "have %s\n", hex);
                                                state_len = req_buf.len;
                                                haves++;
                                                /*
                                                 * Reset in_vain because an ack
                                                 * for this commit has not been
                                                 * seen.
                                                 */
                                                in_vain = 0;
                                        } else if (!args->stateless_rpc
                                                   || ack != ACK_common)
                                                in_vain = 0;
                                        retval = 0;
                                        got_continue = 1;
                                        if (ack == ACK_ready)
                                                got_ready = 1;
                                        break;
                                        }
                                }
                        } while (ack);
                        flushes--;
                        trace2_region_leave_printf("negotiation_v0_v1", "round",
                                                   the_repository, "%d",
                                                   negotiation_round);
                        if (got_continue && MAX_IN_VAIN < in_vain) {
                                print_verbose(args, _("giving up"));
                                break; /* give up */
                        }
                        if (got_ready)
                                break;
                }
        }
done:
        trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
        trace2_data_intmax("negotiation_v0_v1", the_repository, "total_rounds",
                           negotiation_round);
        if (!got_ready || !no_done) {
                packet_buf_write(&req_buf, "done\n");
                send_request(args, fd[1], &req_buf);
        }
        print_verbose(args, _("done"));
        if (retval != 0) {
                multi_ack = 0;
                flushes++;
        }
        strbuf_release(&req_buf);

        if (!got_ready || !no_done)
                consume_shallow_list(args, &reader);
        while (flushes || multi_ack) {
                int ack = get_ack(&reader, result_oid);
                if (ack) {
                        print_verbose(args, _("got %s (%d) %s"), "ack",
                                      ack, oid_to_hex(result_oid));
                        if (ack == ACK)
                                return 0;
                        multi_ack = 1;
                        continue;
                }
                flushes--;
        }
        /* it is no error to fetch into a completely empty repo */
        return count ? retval : 0;
}

static struct commit_list *complete;

static int mark_complete(const struct object_id *oid)
{
        struct commit *commit = deref_without_lazy_fetch(oid, 1);

        if (commit && !(commit->object.flags & COMPLETE)) {
                commit->object.flags |= COMPLETE;
                commit_list_insert(commit, &complete);
        }
        return 0;
}

static int mark_complete_oid(const char *refname UNUSED,
                             const struct object_id *oid,
                             int flag UNUSED,
                             void *cb_data UNUSED)
{
        return mark_complete(oid);
}

static void mark_recent_complete_commits(struct fetch_pack_args *args,
                                         timestamp_t cutoff)
{
        while (complete && cutoff <= complete->item->date) {
                print_verbose(args, _("Marking %s as complete"),
                              oid_to_hex(&complete->item->object.oid));
                pop_most_recent_commit(&complete, COMPLETE);
        }
}

static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
{
        for (; refs; refs = refs->next)
                oidset_insert(oids, &refs->old_oid);
}

static int is_unmatched_ref(const struct ref *ref)
{
        struct object_id oid;
        const char *p;
        return  ref->match_status == REF_NOT_MATCHED &&
                !parse_oid_hex(ref->name, &oid, &p) &&
                *p == '\0' &&
                oideq(&oid, &ref->old_oid);
}

static void filter_refs(struct fetch_pack_args *args,
                        struct ref **refs,
                        struct ref **sought, int nr_sought)
{
        struct ref *newlist = NULL;
        struct ref **newtail = &newlist;
        struct ref *unmatched = NULL;
        struct ref *ref, *next;
        struct oidset tip_oids = OIDSET_INIT;
        int i;
        int strict = !(allow_unadvertised_object_request &
                       (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));

        i = 0;
        for (ref = *refs; ref; ref = next) {
                int keep = 0;
                next = ref->next;

                if (starts_with(ref->name, "refs/") &&
                    check_refname_format(ref->name, 0)) {
                        /*
                         * trash or a peeled value; do not even add it to
                         * unmatched list
                         */
                        free_one_ref(ref);
                        continue;
                } else {
                        while (i < nr_sought) {
                                int cmp = strcmp(ref->name, sought[i]->name);
                                if (cmp < 0)
                                        break; /* definitely do not have it */
                                else if (cmp == 0) {
                                        keep = 1; /* definitely have it */
                                        sought[i]->match_status = REF_MATCHED;
                                }
                                i++;
                        }

                        if (!keep && args->fetch_all &&
                            (!args->deepen || !starts_with(ref->name, "refs/tags/")))
                                keep = 1;
                }

                if (keep) {
                        *newtail = ref;
                        ref->next = NULL;
                        newtail = &ref->next;
                } else {
                        ref->next = unmatched;
                        unmatched = ref;
                }
        }

        if (strict) {
                for (i = 0; i < nr_sought; i++) {
                        ref = sought[i];
                        if (!is_unmatched_ref(ref))
                                continue;

                        add_refs_to_oidset(&tip_oids, unmatched);
                        add_refs_to_oidset(&tip_oids, newlist);
                        break;
                }
        }

        /* Append unmatched requests to the list */
        for (i = 0; i < nr_sought; i++) {
                ref = sought[i];
                if (!is_unmatched_ref(ref))
                        continue;

                if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
                        ref->match_status = REF_MATCHED;
                        *newtail = copy_ref(ref);
                        newtail = &(*newtail)->next;
                } else {
                        ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
                }
        }

        oidset_clear(&tip_oids);
        free_refs(unmatched);

        *refs = newlist;
}

static void mark_alternate_complete(struct fetch_negotiator *negotiator UNUSED,
                                    struct object *obj)
{
        mark_complete(&obj->oid);
}

struct loose_object_iter {
        struct oidset *loose_object_set;
        struct ref *refs;
};

/*
 * Mark recent commits available locally and reachable from a local ref as
 * COMPLETE.
 *
 * The cutoff time for recency is determined by this heuristic: it is the
 * earliest commit time of the objects in refs that are commits and that we know
 * the commit time of.
 */
static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
                                         struct fetch_pack_args *args,
                                         struct ref **refs)
{
        struct ref *ref;
        int old_save_commit_buffer = save_commit_buffer;
        timestamp_t cutoff = 0;

        if (args->refetch)
                return;

        save_commit_buffer = 0;

        trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
        for (ref = *refs; ref; ref = ref->next) {
                struct commit *commit;

                commit = lookup_commit_in_graph(the_repository, &ref->old_oid);
                if (!commit) {
                        struct object *o;

                        if (!repo_has_object_file_with_flags(the_repository, &ref->old_oid,
                                                             OBJECT_INFO_QUICK |
                                                             OBJECT_INFO_SKIP_FETCH_OBJECT))
                                continue;
                        o = parse_object(the_repository, &ref->old_oid);
                        if (!o || o->type != OBJ_COMMIT)
                                continue;

                        commit = (struct commit *)o;
                }

                /*
                 * We already have it -- which may mean that we were
                 * in sync with the other side at some time after
                 * that (it is OK if we guess wrong here).
                 */
                if (!cutoff || cutoff < commit->date)
                        cutoff = commit->date;
        }
        trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);

        /*
         * This block marks all local refs as COMPLETE, and then recursively marks all
         * parents of those refs as COMPLETE.
         */
        trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
        if (!args->deepen) {
                for_each_rawref(mark_complete_oid, NULL);
                for_each_cached_alternate(NULL, mark_alternate_complete);
                commit_list_sort_by_date(&complete);
                if (cutoff)
                        mark_recent_complete_commits(args, cutoff);
        }
        trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);

        /*
         * Mark all complete remote refs as common refs.
         * Don't mark them common yet; the server has to be told so first.
         */
        trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
        for (ref = *refs; ref; ref = ref->next) {
                struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0);

                if (!c || !(c->object.flags & COMPLETE))
                        continue;

                negotiator->known_common(negotiator, c);
        }
        trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);

        save_commit_buffer = old_save_commit_buffer;
}

/*
 * Returns 1 if every object pointed to by the given remote refs is available
 * locally and reachable from a local ref, and 0 otherwise.
 */
static int everything_local(struct fetch_pack_args *args,
                            struct ref **refs)
{
        struct ref *ref;
        int retval;

        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
                const struct object_id *remote = &ref->old_oid;
                struct object *o;

                o = lookup_object(the_repository, remote);
                if (!o || !(o->flags & COMPLETE)) {
                        retval = 0;
                        print_verbose(args, "want %s (%s)", oid_to_hex(remote),
                                      ref->name);
                        continue;
                }
                print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
                              ref->name);
        }

        return retval;
}

static int sideband_demux(int in UNUSED, int out, void *data)
{
        int *xd = data;
        int ret;

        ret = recv_sideband("fetch-pack", xd[0], out);
        close(out);
        return ret;
}

static void create_promisor_file(const char *keep_name,
                                 struct ref **sought, int nr_sought)
{
        struct strbuf promisor_name = STRBUF_INIT;
        int suffix_stripped;

        strbuf_addstr(&promisor_name, keep_name);
        suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
        if (!suffix_stripped)
                BUG("name of pack lockfile should end with .keep (was '%s')",
                    keep_name);
        strbuf_addstr(&promisor_name, ".promisor");

        write_promisor_file(promisor_name.buf, sought, nr_sought);

        strbuf_release(&promisor_name);
}

static void parse_gitmodules_oids(int fd, struct oidset *gitmodules_oids)
{
        int len = the_hash_algo->hexsz + 1; /* hash + NL */

        do {
                char hex_hash[GIT_MAX_HEXSZ + 1];
                int read_len = read_in_full(fd, hex_hash, len);
                struct object_id oid;
                const char *end;

                if (!read_len)
                        return;
                if (read_len != len)
                        die("invalid length read %d", read_len);
                if (parse_oid_hex(hex_hash, &oid, &end) || *end != '\n')
                        die("invalid hash");
                oidset_insert(gitmodules_oids, &oid);
        } while (1);
}

static void add_index_pack_keep_option(struct strvec *args)
{
        char hostname[HOST_NAME_MAX + 1];

        if (xgethostname(hostname, sizeof(hostname)))
                xsnprintf(hostname, sizeof(hostname), "localhost");
        strvec_pushf(args, "--keep=fetch-pack %"PRIuMAX " on %s",
                     (uintmax_t)getpid(), hostname);
}

/*
 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
 * stored there. (It must be freed by the caller.)
 */
static int get_pack(struct fetch_pack_args *args,
                    int xd[2], struct string_list *pack_lockfiles,
                    struct strvec *index_pack_args,
                    struct ref **sought, int nr_sought,
                    struct oidset *gitmodules_oids)
{
        struct async demux;
        int do_keep = args->keep_pack;
        const char *cmd_name;
        struct pack_header header;
        int pass_header = 0;
        struct child_process cmd = CHILD_PROCESS_INIT;
        int fsck_objects = 0;
        int ret;

        memset(&demux, 0, sizeof(demux));
        if (use_sideband) {
                /* xd[] is talking with upload-pack; subprocess reads from
                 * xd[0], spits out band#2 to stderr, and feeds us band#1
                 * through demux->out.
                 */
                demux.proc = sideband_demux;
                demux.data = xd;
                demux.out = -1;
                demux.isolate_sigpipe = 1;
                if (start_async(&demux))
                        die(_("fetch-pack: unable to fork off sideband demultiplexer"));
        }
        else
                demux.out = xd[0];

        if (!args->keep_pack && unpack_limit && !index_pack_args) {

                if (read_pack_header(demux.out, &header))
                        die(_("protocol error: bad pack header"));
                pass_header = 1;
                if (ntohl(header.hdr_entries) < unpack_limit)
                        do_keep = 0;
                else
                        do_keep = 1;
        }

        if (alternate_shallow_file) {
                strvec_push(&cmd.args, "--shallow-file");
                strvec_push(&cmd.args, alternate_shallow_file);
        }

        if (fetch_fsck_objects >= 0
            ? fetch_fsck_objects
            : transfer_fsck_objects >= 0
            ? transfer_fsck_objects
            : 0)
                fsck_objects = 1;

        if (do_keep || args->from_promisor || index_pack_args || fsck_objects) {
                if (pack_lockfiles || fsck_objects)
                        cmd.out = -1;
                cmd_name = "index-pack";
                strvec_push(&cmd.args, cmd_name);
                strvec_push(&cmd.args, "--stdin");
                if (!args->quiet && !args->no_progress)
                        strvec_push(&cmd.args, "-v");
                if (args->use_thin_pack)
                        strvec_push(&cmd.args, "--fix-thin");
                if ((do_keep || index_pack_args) && (args->lock_pack || unpack_limit))
                        add_index_pack_keep_option(&cmd.args);
                if (!index_pack_args && args->check_self_contained_and_connected)
                        strvec_push(&cmd.args, "--check-self-contained-and-connected");
                else
                        /*
                         * We cannot perform any connectivity checks because
                         * not all packs have been downloaded; let the caller
                         * have this responsibility.
                         */
                        args->check_self_contained_and_connected = 0;

                if (args->from_promisor)
                        /*
                         * create_promisor_file() may be called afterwards but
                         * we still need index-pack to know that this is a
                         * promisor pack. For example, if transfer.fsckobjects
                         * is true, index-pack needs to know that .gitmodules
                         * is a promisor object (so that it won't complain if
                         * it is missing).
                         */
                        strvec_push(&cmd.args, "--promisor");
        }
        else {
                cmd_name = "unpack-objects";
                strvec_push(&cmd.args, cmd_name);
                if (args->quiet || args->no_progress)
                        strvec_push(&cmd.args, "-q");
                args->check_self_contained_and_connected = 0;
        }

        if (pass_header)
                strvec_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
                             ntohl(header.hdr_version),
                                 ntohl(header.hdr_entries));
        if (fsck_objects) {
                if (args->from_promisor || index_pack_args)
                        /*
                         * We cannot use --strict in index-pack because it
                         * checks both broken objects and links, but we only
                         * want to check for broken objects.
                         */
                        strvec_push(&cmd.args, "--fsck-objects");
                else
                        strvec_pushf(&cmd.args, "--strict%s",
                                     fsck_msg_types.buf);
        }

        if (index_pack_args) {
                int i;

                for (i = 0; i < cmd.args.nr; i++)
                        strvec_push(index_pack_args, cmd.args.v[i]);
        }

        sigchain_push(SIGPIPE, SIG_IGN);

        cmd.in = demux.out;
        cmd.git_cmd = 1;
        if (start_command(&cmd))
                die(_("fetch-pack: unable to fork off %s"), cmd_name);
        if (do_keep && (pack_lockfiles || fsck_objects)) {
                int is_well_formed;
                char *pack_lockfile = index_pack_lockfile(cmd.out, &is_well_formed);

                if (!is_well_formed)
                        die(_("fetch-pack: invalid index-pack output"));
                if (pack_lockfile)
                        string_list_append_nodup(pack_lockfiles, pack_lockfile);
                parse_gitmodules_oids(cmd.out, gitmodules_oids);
                close(cmd.out);
        }

        if (!use_sideband)
                /* Closed by start_command() */
                xd[0] = -1;

        ret = finish_command(&cmd);
        if (!ret || (args->check_self_contained_and_connected && ret == 1))
                args->self_contained_and_connected =
                        args->check_self_contained_and_connected &&
                        ret == 0;
        else
                die(_("%s failed"), cmd_name);
        if (use_sideband && finish_async(&demux))
                die(_("error in sideband demultiplexer"));

        sigchain_pop(SIGPIPE);

        /*
         * Now that index-pack has succeeded, write the promisor file using the
         * obtained .keep filename if necessary
         */
        if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
                create_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);

        return 0;
}

static int ref_compare_name(const struct ref *a, const struct ref *b)
{
        return strcmp(a->name, b->name);
}

DEFINE_LIST_SORT(static, sort_ref_list, struct ref, next);

static int cmp_ref_by_name(const void *a_, const void *b_)
{
        const struct ref *a = *((const struct ref **)a_);
        const struct ref *b = *((const struct ref **)b_);
        return strcmp(a->name, b->name);
}

static struct ref *do_fetch_pack(struct fetch_pack_args *args,
                                 int fd[2],
                                 const struct ref *orig_ref,
                                 struct ref **sought, int nr_sought,
                                 struct shallow_info *si,
                                 struct string_list *pack_lockfiles)
{
        struct repository *r = the_repository;
        struct ref *ref = copy_ref_list(orig_ref);
        struct object_id oid;
        const char *agent_feature;
        int agent_len;
        struct fetch_negotiator negotiator_alloc;
        struct fetch_negotiator *negotiator;

        negotiator = &negotiator_alloc;
        if (args->refetch) {
                fetch_negotiator_init_noop(negotiator);
        } else {
                fetch_negotiator_init(r, negotiator);
        }

        sort_ref_list(&ref, ref_compare_name);
        QSORT(sought, nr_sought, cmp_ref_by_name);

        if ((agent_feature = server_feature_value("agent", &agent_len))) {
                agent_supported = 1;
                if (agent_len)
                        print_verbose(args, _("Server version is %.*s"),
                                      agent_len, agent_feature);
        }

        if (!server_supports("session-id"))
                advertise_sid = 0;

        if (server_supports("shallow"))
                print_verbose(args, _("Server supports %s"), "shallow");
        else if (args->depth > 0 || is_repository_shallow(r))
                die(_("Server does not support shallow clients"));
        if (args->depth > 0 || args->deepen_since || args->deepen_not)
                args->deepen = 1;
        if (server_supports("multi_ack_detailed")) {
                print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
                multi_ack = 2;
                if (server_supports("no-done")) {
                        print_verbose(args, _("Server supports %s"), "no-done");
                        if (args->stateless_rpc)
                                no_done = 1;
                }
        }
        else if (server_supports("multi_ack")) {
                print_verbose(args, _("Server supports %s"), "multi_ack");
                multi_ack = 1;
        }
        if (server_supports("side-band-64k")) {
                print_verbose(args, _("Server supports %s"), "side-band-64k");
                use_sideband = 2;
        }
        else if (server_supports("side-band")) {
                print_verbose(args, _("Server supports %s"), "side-band");
                use_sideband = 1;
        }
        if (server_supports("allow-tip-sha1-in-want")) {
                print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
        }
        if (server_supports("allow-reachable-sha1-in-want")) {
                print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
        }
        if (server_supports("thin-pack"))
                print_verbose(args, _("Server supports %s"), "thin-pack");
        else
                args->use_thin_pack = 0;
        if (server_supports("no-progress"))
                print_verbose(args, _("Server supports %s"), "no-progress");
        else
                args->no_progress = 0;
        if (server_supports("include-tag"))
                print_verbose(args, _("Server supports %s"), "include-tag");
        else
                args->include_tag = 0;
        if (server_supports("ofs-delta"))
                print_verbose(args, _("Server supports %s"), "ofs-delta");
        else
                prefer_ofs_delta = 0;

        if (server_supports("filter")) {
                server_supports_filtering = 1;
                print_verbose(args, _("Server supports %s"), "filter");
        } else if (args->filter_options.choice) {
                warning("filtering not recognized by server, ignoring");
        }

        if (server_supports("deepen-since")) {
                print_verbose(args, _("Server supports %s"), "deepen-since");
                deepen_since_ok = 1;
        } else if (args->deepen_since)
                die(_("Server does not support --shallow-since"));
        if (server_supports("deepen-not")) {
                print_verbose(args, _("Server supports %s"), "deepen-not");
                deepen_not_ok = 1;
        } else if (args->deepen_not)
                die(_("Server does not support --shallow-exclude"));
        if (server_supports("deepen-relative"))
                print_verbose(args, _("Server supports %s"), "deepen-relative");
        else if (args->deepen_relative)
                die(_("Server does not support --deepen"));
        if (!server_supports_hash(the_hash_algo->name, NULL))
                die(_("Server does not support this repository's object format"));

        mark_complete_and_common_ref(negotiator, args, &ref);
        filter_refs(args, &ref, sought, nr_sought);
        if (!args->refetch && everything_local(args, &ref)) {
                packet_flush(fd[1]);
                goto all_done;
        }
        if (find_common(negotiator, args, fd, &oid, ref) < 0)
                if (!args->keep_pack)
                        /* When cloning, it is not unusual to have
                         * no common commit.
                         */
                        warning(_("no common commits"));

        if (args->stateless_rpc)
                packet_flush(fd[1]);
        if (args->deepen)
                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
                                        NULL);
        else if (si->nr_ours || si->nr_theirs) {
                if (args->reject_shallow_remote)
                        die(_("source repository is shallow, reject to clone."));
                alternate_shallow_file = setup_temporary_shallow(si->shallow);
        } else
                alternate_shallow_file = NULL;
        if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
                     &fsck_options.gitmodules_found))
                die(_("git fetch-pack: fetch failed."));
        if (fsck_finish(&fsck_options))
                die("fsck failed");

 all_done:
        if (negotiator)
                negotiator->release(negotiator);
        return ref;
}

static void add_shallow_requests(struct strbuf *req_buf,
                                 const struct fetch_pack_args *args)
{
        if (is_repository_shallow(the_repository))
                write_shallow_commits(req_buf, 1, NULL);
        if (args->depth > 0)
                packet_buf_write(req_buf, "deepen %d", args->depth);
        if (args->deepen_since) {
                timestamp_t max_age = approxidate(args->deepen_since);
                packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
        }
        if (args->deepen_not) {
                int i;
                for (i = 0; i < args->deepen_not->nr; i++) {
                        struct string_list_item *s = args->deepen_not->items + i;
                        packet_buf_write(req_buf, "deepen-not %s", s->string);
                }
        }
        if (args->deepen_relative)
                packet_buf_write(req_buf, "deepen-relative\n");
}

static void add_wants(const struct ref *wants, struct strbuf *req_buf)
{
        int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);

        for ( ; wants ; wants = wants->next) {
                const struct object_id *remote = &wants->old_oid;
                struct object *o;

                /*
                 * If that object is complete (i.e. it is an ancestor of a
                 * local ref), we tell them we have it but do not have to
                 * tell them about its ancestors, which they already know
                 * about.
                 *
                 * We use lookup_object here because we are only
                 * interested in the case we *know* the object is
                 * reachable and we have already scanned it.
                 */
                if (((o = lookup_object(the_repository, remote)) != NULL) &&
                    (o->flags & COMPLETE)) {
                        continue;
                }

                if (!use_ref_in_want || wants->exact_oid)
                        packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
                else
                        packet_buf_write(req_buf, "want-ref %s\n", wants->name);
        }
}

static void add_common(struct strbuf *req_buf, struct oidset *common)
{
        struct oidset_iter iter;
        const struct object_id *oid;
        oidset_iter_init(common, &iter);

        while ((oid = oidset_iter_next(&iter))) {
                packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
        }
}

static int add_haves(struct fetch_negotiator *negotiator,
                     struct strbuf *req_buf,
                     int *haves_to_send)
{
        int haves_added = 0;
        const struct object_id *oid;

        while ((oid = negotiator->next(negotiator))) {
                packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
                if (++haves_added >= *haves_to_send)
                        break;
        }

        /* Increase haves to send on next round */
        *haves_to_send = next_flush(1, *haves_to_send);

        return haves_added;
}

static void write_fetch_command_and_capabilities(struct strbuf *req_buf,
                                                 const struct string_list *server_options)
{
        const char *hash_name;

        ensure_server_supports_v2("fetch");
        packet_buf_write(req_buf, "command=fetch");
        if (server_supports_v2("agent"))
                packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized());
        if (advertise_sid && server_supports_v2("session-id"))
                packet_buf_write(req_buf, "session-id=%s", trace2_session_id());
        if (server_options && server_options->nr) {
                int i;
                ensure_server_supports_v2("server-option");
                for (i = 0; i < server_options->nr; i++)
                        packet_buf_write(req_buf, "server-option=%s",
                                         server_options->items[i].string);
        }

        if (server_feature_v2("object-format", &hash_name)) {
                int hash_algo = hash_algo_by_name(hash_name);
                if (hash_algo_by_ptr(the_hash_algo) != hash_algo)
                        die(_("mismatched algorithms: client %s; server %s"),
                            the_hash_algo->name, hash_name);
                packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name);
        } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1) {
                die(_("the server does not support algorithm '%s'"),
                    the_hash_algo->name);
        }
        packet_buf_delim(req_buf);
}

static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
                              struct fetch_pack_args *args,
                              const struct ref *wants, struct oidset *common,
                              int *haves_to_send, int *in_vain,
                              int sideband_all, int seen_ack)
{
        int haves_added;
        int done_sent = 0;
        struct strbuf req_buf = STRBUF_INIT;

        write_fetch_command_and_capabilities(&req_buf, args->server_options);

        if (args->use_thin_pack)
                packet_buf_write(&req_buf, "thin-pack");
        if (args->no_progress)
                packet_buf_write(&req_buf, "no-progress");
        if (args->include_tag)
                packet_buf_write(&req_buf, "include-tag");
        if (prefer_ofs_delta)
                packet_buf_write(&req_buf, "ofs-delta");
        if (sideband_all)
                packet_buf_write(&req_buf, "sideband-all");

        /* Add shallow-info and deepen request */
        if (server_supports_feature("fetch", "shallow", 0))
                add_shallow_requests(&req_buf, args);
        else if (is_repository_shallow(the_repository) || args->deepen)
                die(_("Server does not support shallow requests"));

        /* Add filter */
        send_filter(args, &req_buf,
                    server_supports_feature("fetch", "filter", 0));

        if (server_supports_feature("fetch", "packfile-uris", 0)) {
                int i;
                struct strbuf to_send = STRBUF_INIT;

                for (i = 0; i < uri_protocols.nr; i++) {
                        const char *s = uri_protocols.items[i].string;

                        if (!strcmp(s, "https") || !strcmp(s, "http")) {
                                if (to_send.len)
                                        strbuf_addch(&to_send, ',');
                                strbuf_addstr(&to_send, s);
                        }
                }
                if (to_send.len) {
                        packet_buf_write(&req_buf, "packfile-uris %s",
                                         to_send.buf);
                        strbuf_release(&to_send);
                }
        }

        /* add wants */
        add_wants(wants, &req_buf);

        /* Add all of the common commits we've found in previous rounds */
        add_common(&req_buf, common);

        haves_added = add_haves(negotiator, &req_buf, haves_to_send);
        *in_vain += haves_added;
        trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added);
        trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain);
        if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) {
                /* Send Done */
                packet_buf_write(&req_buf, "done\n");
                done_sent = 1;
        }

        /* Send request */
        packet_buf_flush(&req_buf);
        if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
                die_errno(_("unable to write request to remote"));

        strbuf_release(&req_buf);
        return done_sent;
}

/*
 * Processes a section header in a server's response and checks if it matches
 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
 * not consumed); if 0, the line will be consumed and the function will die if
 * the section header doesn't match what was expected.
 */
static int process_section_header(struct packet_reader *reader,
                                  const char *section, int peek)
{
        int ret = 0;

        if (packet_reader_peek(reader) == PACKET_READ_NORMAL &&
            !strcmp(reader->line, section))
                ret = 1;

        if (!peek) {
                if (!ret) {
                        if (reader->line)
                                die(_("expected '%s', received '%s'"),
                                    section, reader->line);
                        else
                                die(_("expected '%s'"), section);
                }
                packet_reader_read(reader);
        }

        return ret;
}

static int process_ack(struct fetch_negotiator *negotiator,
                       struct packet_reader *reader,
                       struct object_id *common_oid,
                       int *received_ready)
{
        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
                const char *arg;

                if (!strcmp(reader->line, "NAK"))
                        continue;

                if (skip_prefix(reader->line, "ACK ", &arg)) {
                        if (!get_oid_hex(arg, common_oid)) {
                                struct commit *commit;
                                commit = lookup_commit(the_repository, common_oid);
                                if (negotiator)
                                        negotiator->ack(negotiator, commit);
                        }
                        return 1;
                }

                if (!strcmp(reader->line, "ready")) {
                        *received_ready = 1;
                        continue;
                }

                die(_("unexpected acknowledgment line: '%s'"), reader->line);
        }

        if (reader->status != PACKET_READ_FLUSH &&
            reader->status != PACKET_READ_DELIM)
                die(_("error processing acks: %d"), reader->status);

        /*
         * If an "acknowledgments" section is sent, a packfile is sent if and
         * only if "ready" was sent in this section. The other sections
         * ("shallow-info" and "wanted-refs") are sent only if a packfile is
         * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
         * otherwise.
         */
        if (*received_ready && reader->status != PACKET_READ_DELIM)
                /*
                 * TRANSLATORS: The parameter will be 'ready', a protocol
                 * keyword.
                 */
                die(_("expected packfile to be sent after '%s'"), "ready");
        if (!*received_ready && reader->status != PACKET_READ_FLUSH)
                /*
                 * TRANSLATORS: The parameter will be 'ready', a protocol
                 * keyword.
                 */
                die(_("expected no other sections to be sent after no '%s'"), "ready");

        return 0;
}

static void receive_shallow_info(struct fetch_pack_args *args,
                                 struct packet_reader *reader,
                                 struct oid_array *shallows,
                                 struct shallow_info *si)
{
        int unshallow_received = 0;

        process_section_header(reader, "shallow-info", 0);
        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
                const char *arg;
                struct object_id oid;

                if (skip_prefix(reader->line, "shallow ", &arg)) {
                        if (get_oid_hex(arg, &oid))
                                die(_("invalid shallow line: %s"), reader->line);
                        oid_array_append(shallows, &oid);
                        continue;
                }
                if (skip_prefix(reader->line, "unshallow ", &arg)) {
                        if (get_oid_hex(arg, &oid))
                                die(_("invalid unshallow line: %s"), reader->line);
                        if (!lookup_object(the_repository, &oid))
                                die(_("object not found: %s"), reader->line);
                        /* make sure that it is parsed as shallow */
                        if (!parse_object(the_repository, &oid))
                                die(_("error in object: %s"), reader->line);
                        if (unregister_shallow(&oid))
                                die(_("no shallow found: %s"), reader->line);
                        unshallow_received = 1;
                        continue;
                }
                die(_("expected shallow/unshallow, got %s"), reader->line);
        }

        if (reader->status != PACKET_READ_FLUSH &&
            reader->status != PACKET_READ_DELIM)
                die(_("error processing shallow info: %d"), reader->status);

        if (args->deepen || unshallow_received) {
                /*
                 * Treat these as shallow lines caused by our depth settings.
                 * In v0, these lines cannot cause refs to be rejected; do the
                 * same.
                 */
                int i;

                for (i = 0; i < shallows->nr; i++)
                        register_shallow(the_repository, &shallows->oid[i]);
                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
                                        NULL);
                args->deepen = 1;
        } else if (shallows->nr) {
                /*
                 * Treat these as shallow lines caused by the remote being
                 * shallow. In v0, remote refs that reach these objects are
                 * rejected (unless --update-shallow is set); do the same.
                 */
                prepare_shallow_info(si, shallows);
                if (si->nr_ours || si->nr_theirs) {
                        if (args->reject_shallow_remote)
                                die(_("source repository is shallow, reject to clone."));
                        alternate_shallow_file =
                                setup_temporary_shallow(si->shallow);
                } else
                        alternate_shallow_file = NULL;
        } else {
                alternate_shallow_file = NULL;
        }
}

static int cmp_name_ref(const void *name, const void *ref)
{
        return strcmp(name, (*(struct ref **)ref)->name);
}

static void receive_wanted_refs(struct packet_reader *reader,
                                struct ref **sought, int nr_sought)
{
        process_section_header(reader, "wanted-refs", 0);
        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
                struct object_id oid;
                const char *end;
                struct ref **found;

                if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
                        die(_("expected wanted-ref, got '%s'"), reader->line);

                found = bsearch(end, sought, nr_sought, sizeof(*sought),
                                cmp_name_ref);
                if (!found)
                        die(_("unexpected wanted-ref: '%s'"), reader->line);
                oidcpy(&(*found)->old_oid, &oid);
        }

        if (reader->status != PACKET_READ_DELIM)
                die(_("error processing wanted refs: %d"), reader->status);
}

static void receive_packfile_uris(struct packet_reader *reader,
                                  struct string_list *uris)
{
        process_section_header(reader, "packfile-uris", 0);
        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
                if (reader->pktlen < the_hash_algo->hexsz ||
                    reader->line[the_hash_algo->hexsz] != ' ')
                        die("expected '<hash> <uri>', got: %s\n", reader->line);

                string_list_append(uris, reader->line);
        }
        if (reader->status != PACKET_READ_DELIM)
                die("expected DELIM");
}

enum fetch_state {
        FETCH_CHECK_LOCAL = 0,
        FETCH_SEND_REQUEST,
        FETCH_PROCESS_ACKS,
        FETCH_GET_PACK,
        FETCH_DONE,
};

static void do_check_stateless_delimiter(int stateless_rpc,
                                         struct packet_reader *reader)
{
        check_stateless_delimiter(stateless_rpc, reader,
                                  _("git fetch-pack: expected response end packet"));
}

static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
                                    int fd[2],
                                    const struct ref *orig_ref,
                                    struct ref **sought, int nr_sought,
                                    struct oid_array *shallows,
                                    struct shallow_info *si,
                                    struct string_list *pack_lockfiles)
{
        struct repository *r = the_repository;
        struct ref *ref = copy_ref_list(orig_ref);
        enum fetch_state state = FETCH_CHECK_LOCAL;
        struct oidset common = OIDSET_INIT;
        struct packet_reader reader;
        int in_vain = 0, negotiation_started = 0;
        int negotiation_round = 0;
        int haves_to_send = INITIAL_FLUSH;
        struct fetch_negotiator negotiator_alloc;
        struct fetch_negotiator *negotiator;
        int seen_ack = 0;
        struct object_id common_oid;
        int received_ready = 0;
        struct string_list packfile_uris = STRING_LIST_INIT_DUP;
        int i;
        struct strvec index_pack_args = STRVEC_INIT;

        negotiator = &negotiator_alloc;
        if (args->refetch)
                fetch_negotiator_init_noop(negotiator);
        else
                fetch_negotiator_init(r, negotiator);

        packet_reader_init(&reader, fd[0], NULL, 0,
                           PACKET_READ_CHOMP_NEWLINE |
                           PACKET_READ_DIE_ON_ERR_PACKET);
        if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
            server_supports_feature("fetch", "sideband-all", 0)) {
                reader.use_sideband = 1;
                reader.me = "fetch-pack";
        }

        while (state != FETCH_DONE) {
                switch (state) {
                case FETCH_CHECK_LOCAL:
                        sort_ref_list(&ref, ref_compare_name);
                        QSORT(sought, nr_sought, cmp_ref_by_name);

                        /* v2 supports these by default */
                        allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
                        use_sideband = 2;
                        if (args->depth > 0 || args->deepen_since || args->deepen_not)
                                args->deepen = 1;

                        /* Filter 'ref' by 'sought' and those that aren't local */
                        mark_complete_and_common_ref(negotiator, args, &ref);
                        filter_refs(args, &ref, sought, nr_sought);
                        if (!args->refetch && everything_local(args, &ref))
                                state = FETCH_DONE;
                        else
                                state = FETCH_SEND_REQUEST;

                        mark_tips(negotiator, args->negotiation_tips);
                        for_each_cached_alternate(negotiator,
                                                  insert_one_alternate_object);
                        break;
                case FETCH_SEND_REQUEST:
                        if (!negotiation_started) {
                                negotiation_started = 1;
                                trace2_region_enter("fetch-pack",
                                                    "negotiation_v2",
                                                    the_repository);
                        }
                        negotiation_round++;
                        trace2_region_enter_printf("negotiation_v2", "round",
                                                   the_repository, "%d",
                                                   negotiation_round);
                        if (send_fetch_request(negotiator, fd[1], args, ref,
                                               &common,
                                               &haves_to_send, &in_vain,
                                               reader.use_sideband,
                                               seen_ack)) {
                                trace2_region_leave_printf("negotiation_v2", "round",
                                                           the_repository, "%d",
                                                           negotiation_round);
                                state = FETCH_GET_PACK;
                        }
                        else
                                state = FETCH_PROCESS_ACKS;
                        break;
                case FETCH_PROCESS_ACKS:
                        /* Process ACKs/NAKs */
                        process_section_header(&reader, "acknowledgments", 0);
                        while (process_ack(negotiator, &reader, &common_oid,
                                           &received_ready)) {
                                in_vain = 0;
                                seen_ack = 1;
                                oidset_insert(&common, &common_oid);
                        }
                        trace2_region_leave_printf("negotiation_v2", "round",
                                                   the_repository, "%d",
                                                   negotiation_round);
                        if (received_ready) {
                                /*
                                 * Don't check for response delimiter; get_pack() will
                                 * read the rest of this response.
                                 */
                                state = FETCH_GET_PACK;
                        } else {
                                do_check_stateless_delimiter(args->stateless_rpc, &reader);
                                state = FETCH_SEND_REQUEST;
                        }
                        break;
                case FETCH_GET_PACK:
                        trace2_region_leave("fetch-pack",
                                            "negotiation_v2",
                                            the_repository);
                        trace2_data_intmax("negotiation_v2", the_repository,
                                           "total_rounds", negotiation_round);
                        /* Check for shallow-info section */
                        if (process_section_header(&reader, "shallow-info", 1))
                                receive_shallow_info(args, &reader, shallows, si);

                        if (process_section_header(&reader, "wanted-refs", 1))
                                receive_wanted_refs(&reader, sought, nr_sought);

                        /* get the pack(s) */
                        if (git_env_bool("GIT_TRACE_REDACT", 1))
                                reader.options |= PACKET_READ_REDACT_URI_PATH;
                        if (process_section_header(&reader, "packfile-uris", 1))
                                receive_packfile_uris(&reader, &packfile_uris);
                        /* We don't expect more URIs. Reset to avoid expensive URI check. */
                        reader.options &= ~PACKET_READ_REDACT_URI_PATH;

                        process_section_header(&reader, "packfile", 0);

                        /*
                         * this is the final request we'll make of the server;
                         * do a half-duplex shutdown to indicate that they can
                         * hang up as soon as the pack is sent.
                         */
                        close(fd[1]);
                        fd[1] = -1;

                        if (get_pack(args, fd, pack_lockfiles,
                                     packfile_uris.nr ? &index_pack_args : NULL,
                                     sought, nr_sought, &fsck_options.gitmodules_found))
                                die(_("git fetch-pack: fetch failed."));
                        do_check_stateless_delimiter(args->stateless_rpc, &reader);

                        state = FETCH_DONE;
                        break;
                case FETCH_DONE:
                        continue;
                }
        }

        for (i = 0; i < packfile_uris.nr; i++) {
                int j;
                struct child_process cmd = CHILD_PROCESS_INIT;
                char packname[GIT_MAX_HEXSZ + 1];
                const char *uri = packfile_uris.items[i].string +
                        the_hash_algo->hexsz + 1;

                strvec_push(&cmd.args, "http-fetch");
                strvec_pushf(&cmd.args, "--packfile=%.*s",
                             (int) the_hash_algo->hexsz,
                             packfile_uris.items[i].string);
                for (j = 0; j < index_pack_args.nr; j++)
                        strvec_pushf(&cmd.args, "--index-pack-arg=%s",
                                     index_pack_args.v[j]);
                strvec_push(&cmd.args, uri);
                cmd.git_cmd = 1;
                cmd.no_stdin = 1;
                cmd.out = -1;
                if (start_command(&cmd))
                        die("fetch-pack: unable to spawn http-fetch");

                if (read_in_full(cmd.out, packname, 5) < 0 ||
                    memcmp(packname, "keep\t", 5))
                        die("fetch-pack: expected keep then TAB at start of http-fetch output");

                if (read_in_full(cmd.out, packname,
                                 the_hash_algo->hexsz + 1) < 0 ||
                    packname[the_hash_algo->hexsz] != '\n')
                        die("fetch-pack: expected hash then LF at end of http-fetch output");

                packname[the_hash_algo->hexsz] = '\0';

                parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);

                close(cmd.out);

                if (finish_command(&cmd))
                        die("fetch-pack: unable to finish http-fetch");

                if (memcmp(packfile_uris.items[i].string, packname,
                           the_hash_algo->hexsz))
                        die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
                            uri, (int) the_hash_algo->hexsz,
                            packfile_uris.items[i].string);

                string_list_append_nodup(pack_lockfiles,
                                         xstrfmt("%s/pack/pack-%s.keep",
                                                 get_object_directory(),
                                                 packname));
        }
        string_list_clear(&packfile_uris, 0);
        strvec_clear(&index_pack_args);

        if (fsck_finish(&fsck_options))
                die("fsck failed");

        if (negotiator)
                negotiator->release(negotiator);

        oidset_clear(&common);
        return ref;
}

static int fetch_pack_config_cb(const char *var, const char *value, void *cb)
{
        if (strcmp(var, "fetch.fsck.skiplist") == 0) {
                const char *path;

                if (git_config_pathname(&path, var, value))
                        return 1;
                strbuf_addf(&fsck_msg_types, "%cskiplist=%s",
                        fsck_msg_types.len ? ',' : '=', path);
                free((char *)path);
                return 0;
        }

        if (skip_prefix(var, "fetch.fsck.", &var)) {
                if (is_valid_msg_type(var, value))
                        strbuf_addf(&fsck_msg_types, "%c%s=%s",
                                fsck_msg_types.len ? ',' : '=', var, value);
                else
                        warning("Skipping unknown msg id '%s'", var);
                return 0;
        }

        return git_default_config(var, value, cb);
}

static void fetch_pack_config(void)
{
        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
        git_config_get_bool("transfer.advertisesid", &advertise_sid);
        if (!uri_protocols.nr) {
                char *str;

                if (!git_config_get_string("fetch.uriprotocols", &str) && str) {
                        string_list_split(&uri_protocols, str, ',', -1);
                        free(str);
                }
        }

        git_config(fetch_pack_config_cb, NULL);
}

static void fetch_pack_setup(void)
{
        static int did_setup;
        if (did_setup)
                return;
        fetch_pack_config();
        if (0 <= transfer_unpack_limit)
                unpack_limit = transfer_unpack_limit;
        else if (0 <= fetch_unpack_limit)
                unpack_limit = fetch_unpack_limit;
        did_setup = 1;
}

static int remove_duplicates_in_refs(struct ref **ref, int nr)
{
        struct string_list names = STRING_LIST_INIT_NODUP;
        int src, dst;

        for (src = dst = 0; src < nr; src++) {
                struct string_list_item *item;
                item = string_list_insert(&names, ref[src]->name);
                if (item->util)
                        continue; /* already have it */
                item->util = ref[src];
                if (src != dst)
                        ref[dst] = ref[src];
                dst++;
        }
        for (src = dst; src < nr; src++)
                ref[src] = NULL;
        string_list_clear(&names, 0);
        return dst;
}

static void update_shallow(struct fetch_pack_args *args,
                           struct ref **sought, int nr_sought,
                           struct shallow_info *si)
{
        struct oid_array ref = OID_ARRAY_INIT;
        int *status;
        int i;

        if (args->deepen && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
                        unlink_or_warn(git_path_shallow(the_repository));
                        rollback_shallow_file(the_repository, &shallow_lock);
                } else
                        commit_shallow_file(the_repository, &shallow_lock);
                alternate_shallow_file = NULL;
                return;
        }

        if (!si->shallow || !si->shallow->nr)
                return;

        if (args->cloning) {
                /*
                 * remote is shallow, but this is a clone, there are
                 * no objects in repo to worry about. Accept any
                 * shallow points that exist in the pack (iow in repo
                 * after get_pack() and reprepare_packed_git())
                 */
                struct oid_array extra = OID_ARRAY_INIT;
                struct object_id *oid = si->shallow->oid;
                for (i = 0; i < si->shallow->nr; i++)
                        if (repo_has_object_file(the_repository, &oid[i]))
                                oid_array_append(&extra, &oid[i]);
                if (extra.nr) {
                        setup_alternate_shallow(&shallow_lock,
                                                &alternate_shallow_file,
                                                &extra);
                        commit_shallow_file(the_repository, &shallow_lock);
                        alternate_shallow_file = NULL;
                }
                oid_array_clear(&extra);
                return;
        }

        if (!si->nr_ours && !si->nr_theirs)
                return;

        remove_nonexistent_theirs_shallow(si);
        if (!si->nr_ours && !si->nr_theirs)
                return;
        for (i = 0; i < nr_sought; i++)
                oid_array_append(&ref, &sought[i]->old_oid);
        si->ref = &ref;

        if (args->update_shallow) {
                /*
                 * remote is also shallow, .git/shallow may be updated
                 * so all refs can be accepted. Make sure we only add
                 * shallow roots that are actually reachable from new
                 * refs.
                 */
                struct oid_array extra = OID_ARRAY_INIT;
                struct object_id *oid = si->shallow->oid;
                assign_shallow_commits_to_refs(si, NULL, NULL);
                if (!si->nr_ours && !si->nr_theirs) {
                        oid_array_clear(&ref);
                        return;
                }
                for (i = 0; i < si->nr_ours; i++)
                        oid_array_append(&extra, &oid[si->ours[i]]);
                for (i = 0; i < si->nr_theirs; i++)
                        oid_array_append(&extra, &oid[si->theirs[i]]);
                setup_alternate_shallow(&shallow_lock,
                                        &alternate_shallow_file,
                                        &extra);
                commit_shallow_file(the_repository, &shallow_lock);
                oid_array_clear(&extra);
                oid_array_clear(&ref);
                alternate_shallow_file = NULL;
                return;
        }

        /*
         * remote is also shallow, check what ref is safe to update
         * without updating .git/shallow
         */
        CALLOC_ARRAY(status, nr_sought);
        assign_shallow_commits_to_refs(si, NULL, status);
        if (si->nr_ours || si->nr_theirs) {
                for (i = 0; i < nr_sought; i++)
                        if (status[i])
                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
        }
        free(status);
        oid_array_clear(&ref);
}

static const struct object_id *iterate_ref_map(void *cb_data)
{
        struct ref **rm = cb_data;
        struct ref *ref = *rm;

        if (!ref)
                return NULL;
        *rm = ref->next;
        return &ref->old_oid;
}

struct ref *fetch_pack(struct fetch_pack_args *args,
                       int fd[],
                       const struct ref *ref,
                       struct ref **sought, int nr_sought,
                       struct oid_array *shallow,
                       struct string_list *pack_lockfiles,
                       enum protocol_version version)
{
        struct ref *ref_cpy;
        struct shallow_info si;
        struct oid_array shallows_scratch = OID_ARRAY_INIT;

        fetch_pack_setup();
        if (nr_sought)
                nr_sought = remove_duplicates_in_refs(sought, nr_sought);

        if (version != protocol_v2 && !ref) {
                packet_flush(fd[1]);
                die(_("no matching remote head"));
        }
        if (version == protocol_v2) {
                if (shallow->nr)
                        BUG("Protocol V2 does not provide shallows at this point in the fetch");
                memset(&si, 0, sizeof(si));
                ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
                                           &shallows_scratch, &si,
                                           pack_lockfiles);
        } else {
                prepare_shallow_info(&si, shallow);
                ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
                                        &si, pack_lockfiles);
        }
        reprepare_packed_git(the_repository);

        if (!args->cloning && args->deepen) {
                struct check_connected_options opt = CHECK_CONNECTED_INIT;
                struct ref *iterator = ref_cpy;
                opt.shallow_file = alternate_shallow_file;
                if (args->deepen)
                        opt.is_deepening_fetch = 1;
                if (check_connected(iterate_ref_map, &iterator, &opt)) {
                        error(_("remote did not send all necessary objects"));
                        free_refs(ref_cpy);
                        ref_cpy = NULL;
                        rollback_shallow_file(the_repository, &shallow_lock);
                        goto cleanup;
                }
                args->connectivity_checked = 1;
        }

        update_shallow(args, sought, nr_sought, &si);
cleanup:
        clear_shallow_info(&si);
        oid_array_clear(&shallows_scratch);
        return ref_cpy;
}

static int add_to_object_array(const struct object_id *oid, void *data)
{
        struct object_array *a = data;

        add_object_array(lookup_object(the_repository, oid), "", a);
        return 0;
}

static void clear_common_flag(struct oidset *s)
{
        struct oidset_iter iter;
        const struct object_id *oid;
        oidset_iter_init(s, &iter);

        while ((oid = oidset_iter_next(&iter))) {
                struct object *obj = lookup_object(the_repository, oid);
                obj->flags &= ~COMMON;
        }
}

void negotiate_using_fetch(const struct oid_array *negotiation_tips,
                           const struct string_list *server_options,
                           int stateless_rpc,
                           int fd[],
                           struct oidset *acked_commits)
{
        struct fetch_negotiator negotiator;
        struct packet_reader reader;
        struct object_array nt_object_array = OBJECT_ARRAY_INIT;
        struct strbuf req_buf = STRBUF_INIT;
        int haves_to_send = INITIAL_FLUSH;
        int in_vain = 0;
        int seen_ack = 0;
        int last_iteration = 0;
        int negotiation_round = 0;
        timestamp_t min_generation = GENERATION_NUMBER_INFINITY;

        fetch_negotiator_init(the_repository, &negotiator);
        mark_tips(&negotiator, negotiation_tips);

        packet_reader_init(&reader, fd[0], NULL, 0,
                           PACKET_READ_CHOMP_NEWLINE |
                           PACKET_READ_DIE_ON_ERR_PACKET);

        oid_array_for_each((struct oid_array *) negotiation_tips,
                           add_to_object_array,
                           &nt_object_array);

        trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository);
        while (!last_iteration) {
                int haves_added;
                struct object_id common_oid;
                int received_ready = 0;

                negotiation_round++;

                trace2_region_enter_printf("negotiate_using_fetch", "round",
                                           the_repository, "%d",
                                           negotiation_round);
                strbuf_reset(&req_buf);
                write_fetch_command_and_capabilities(&req_buf, server_options);

                packet_buf_write(&req_buf, "wait-for-done");

                haves_added = add_haves(&negotiator, &req_buf, &haves_to_send);
                in_vain += haves_added;
                if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN))
                        last_iteration = 1;

                trace2_data_intmax("negotiate_using_fetch", the_repository,
                                   "haves_added", haves_added);
                trace2_data_intmax("negotiate_using_fetch", the_repository,
                                   "in_vain", in_vain);

                /* Send request */
                packet_buf_flush(&req_buf);
                if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0)
                        die_errno(_("unable to write request to remote"));

                /* Process ACKs/NAKs */
                process_section_header(&reader, "acknowledgments", 0);
                while (process_ack(&negotiator, &reader, &common_oid,
                                   &received_ready)) {
                        struct commit *commit = lookup_commit(the_repository,
                                                              &common_oid);
                        if (commit) {
                                timestamp_t generation;

                                parse_commit_or_die(commit);
                                commit->object.flags |= COMMON;
                                generation = commit_graph_generation(commit);
                                if (generation < min_generation)
                                        min_generation = generation;
                        }
                        in_vain = 0;
                        seen_ack = 1;
                        oidset_insert(acked_commits, &common_oid);
                }
                if (received_ready)
                        die(_("unexpected 'ready' from remote"));
                else
                        do_check_stateless_delimiter(stateless_rpc, &reader);
                if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
                                                 REACH_SCRATCH, 0,
                                                 min_generation))
                        last_iteration = 1;
                trace2_region_leave_printf("negotiation", "round",
                                           the_repository, "%d",
                                           negotiation_round);
        }
        trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository);
        trace2_data_intmax("negotiate_using_fetch", the_repository,
                           "total_rounds", negotiation_round);
        clear_common_flag(acked_commits);
        strbuf_release(&req_buf);
}

int report_unmatched_refs(struct ref **sought, int nr_sought)
{
        int i, ret = 0;

        for (i = 0; i < nr_sought; i++) {
                if (!sought[i])
                        continue;
                switch (sought[i]->match_status) {
                case REF_MATCHED:
                        continue;
                case REF_NOT_MATCHED:
                        error(_("no such remote ref %s"), sought[i]->name);
                        break;
                case REF_UNADVERTISED_NOT_ALLOWED:
                        error(_("Server does not allow request for unadvertised object %s"),
                              sought[i]->name);
                        break;
                }
                ret = 1;
        }
        return ret;
}