From c940699d02aaff2e060315095d1799be41bc99d3 Mon Sep 17 00:00:00 2001 From: Kirill Date: Tue, 2 Sep 2008 21:48:02 -0400 Subject: [PATCH] Make mingw_spawn* buildable by VS 2008 The biggest chunk is the new git-compat-util.h and cache.h that are compatible with both gcc and VS 2008. The rest of the changes: - sha1_file.c: use only part of the original file; - compat/mingw.[ch]: minor changes caused by difference in header files, easily portable back to git (see cosmetic changes below); - usage.c: minor changes, related to the use of NORETURN attribute, easily portable back to git (see cosmetic changes below); - strbuf.c: very minor changes to fix some warnings. There are several kinds of cosmetic changes in this commit: - VS 2008 enforces C much stricter than gcc; for example, it does not allow declaring variables in the middle of a function; - apparently, VS 2008 is much pickier about the placement of declaration specification (see ticktrack in compat/mingw.c); - headers are slightly different (see struct pollfd in compat/mingw.h); - va_copy is apparently unavailable in VS 2008; - VS 2008 is apparently much pickier about possible loss of data and type mismatch, hence some additional casting [in strbuf.c]; - VS 2008 does not have __attribute__ and does not allow to use NORETURN in variable or parameter declarations [in usage.c]. This changes to the "borrowed" files do not break the build of git, but no tests have been exec'd. --- Makefile | 13 +- cache.h | 63 ++ compat/mingw.c | 18 +- compat/mingw.h | 7 +- git-compat-util.h | 183 ++++ sha1_file.c | 2539 ++--------------------------------------------------- strbuf.c | 6 +- usage.c | 10 +- 8 files changed, 341 insertions(+), 2498 deletions(-) create mode 100644 cache.h create mode 100644 git-compat-util.h rewrite sha1_file.c (97%) diff --git a/Makefile b/Makefile index 6e27640..bde8552 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ OBJECTS=ext.o debug.o dll.o factory.o menu.o systeminfo.o registry.o exec.o -CFLAGS=-O -g +CFLAGS=-O -g -DNO_MMAP -DNO_PREAD -DNO_STRLCPY +COMPAT_H = cache.h git-compat-util.h hash.h strbuf.h compat/mingw.h +COMPAT_OBJ = date.o sha1_file.o strbuf.o usage.o wrapper.o \ + compat/mingw.o compat/mmap.o compat/pread.o compat/strlcpy.o \ + compat/winansi.o TARGET=git_shell_ext.dll MSYSGIT_PATH=$(shell cd /; pwd -W | sed -e 's|/|\\\\\\\\|g') @@ -10,9 +14,9 @@ all: $(TARGET) .o:.c $(CC) $(CFLAGS) $< -o $@ -$(TARGET): $(OBJECTS) git_shell_ext.def +$(TARGET): $(OBJECTS) $(COMPAT_OBJ) git_shell_ext.def dllwrap.exe --enable-stdcall-fixup --def git_shell_ext.def \ - $(OBJECTS) -o $@ -luuid -loleaut32 -lole32 + $(OBJECTS) $(COMPAT_OBJ) -o $@ -luuid -loleaut32 -lole32 -lws2_32 # gcc $(LDFLAGS) -o $@ $(OBJECTS) -lole32 -luuid -loleaut32 # dlltool -d git_shell_ext.def -l $@ $(OBJECTS) @@ -24,6 +28,7 @@ menu.o: menu.h ext.h debug.h systeminfo.h exec.h systeminfo.o: systeminfo.h registry.o: registry.h exec.o: debug.h systeminfo.h exec.h +$(COMPAT_OBJ) : $(COMPAT_H) install: all regsvr32 -s -n -i:machine $(DLL_PATH) @@ -38,4 +43,4 @@ uninstall-user: all regsvr32 -u -s $(DLL_PATH) clean: - -rm -f $(OBJECTS) $(TARGET) + -rm -f $(OBJECTS) $(COMPAT_OBJ) $(TARGET) diff --git a/cache.h b/cache.h new file mode 100644 index 0000000..6a3fb23 --- /dev/null +++ b/cache.h @@ -0,0 +1,63 @@ +#ifndef CACHE_H +#define CACHE_H + +#include "git-compat-util.h" +#include "strbuf.h" + +enum date_mode { + DATE_NORMAL = 0, + DATE_RELATIVE, + DATE_SHORT, + DATE_LOCAL, + DATE_ISO8601, + DATE_RFC2822 +}; + +#define alloc_nr(x) (((x)+16)*3/2) + +/* + * Realloc the buffer pointed at by variable 'x' so that it can hold + * at least 'nr' entries; the number of entries currently allocated + * is 'alloc', using the standard growing factor alloc_nr() macro. + * + * DO NOT USE any expression with side-effect for 'x' or 'alloc'. + */ +#define ALLOC_GROW(x, nr, alloc) \ + do { \ + if ((nr) > alloc) { \ + if (alloc_nr(alloc) < (nr)) \ + alloc = (nr); \ + else \ + alloc = alloc_nr(alloc); \ + x = xrealloc((x), alloc * sizeof(*(x))); \ + } \ + } while(0) + +struct pack_window { + struct pack_window *next; + unsigned char *base; + off_t offset; + size_t len; + unsigned int last_used; + unsigned int inuse_cnt; +}; + +extern struct packed_git { + struct packed_git *next; + struct pack_window *windows; + off_t pack_size; + const void *index_data; + size_t index_size; + uint32_t num_objects; + uint32_t num_bad_objects; + unsigned char *bad_object_sha1; + int index_version; + time_t mtime; + int pack_fd; + int pack_local; + unsigned char sha1[20]; + /* something like ".git/objects/pack/xxxxx.pack" */ + char pack_name[FLEX_ARRAY]; /* more */ +} *packed_git; + +#endif /* CACHE_H */ diff --git a/compat/mingw.c b/compat/mingw.c index 4cdc5f6..1e080ba 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -8,13 +8,15 @@ int mingw_open (const char *filename, int oflags, ...) { va_list args; unsigned mode; + int fd; + va_start(args, oflags); mode = va_arg(args, int); va_end(args); if (!strcmp(filename, "/dev/null")) filename = "nul"; - int fd = open(filename, oflags, mode); + fd = open(filename, oflags, mode); if (fd < 0 && (oflags & O_CREAT) && errno == EACCES) { DWORD attrs = GetFileAttributes(filename); if (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY)) @@ -178,7 +180,7 @@ int mingw_fstat(int fd, struct mingw_stat *buf) static inline void time_t_to_filetime(time_t t, FILETIME *ft) { long long winTime = t * 10000000LL + 116444736000000000LL; - ft->dwLowDateTime = winTime; + ft->dwLowDateTime = (DWORD)winTime; ft->dwHighDateTime = winTime >> 32; } @@ -281,7 +283,8 @@ int pipe(int filedes[2]) int poll(struct pollfd *ufds, unsigned int nfds, int timeout) { - int i, pending; + unsigned int i; + int pending; if (timeout != -1) return errno = EINVAL, error("poll timeout not supported"); @@ -514,10 +517,11 @@ static char **get_path_split(void) static void free_path_split(char **path) { + char **p; if (!path) return; - char **p = path; + p = path; while (*p) free(*p++); free(path); @@ -916,7 +920,7 @@ static sig_handler_t timer_fn = SIG_DFL; * length to call the signal handler. */ -static __stdcall unsigned ticktack(void *dummy) +static unsigned __stdcall ticktack(void *dummy) { while (WaitForSingleObject(timer_event, timer_interval) == WAIT_TIMEOUT) { if (timer_fn == SIG_DFL) @@ -1012,9 +1016,11 @@ int sigaction(int sig, struct sigaction *in, struct sigaction *out) #undef signal sig_handler_t mingw_signal(int sig, sig_handler_t handler) { + sig_handler_t old; + if (sig != SIGALRM) return signal(sig, handler); - sig_handler_t old = timer_fn; + old = timer_fn; timer_fn = handler; return old; } diff --git a/compat/mingw.h b/compat/mingw.h index a52e657..de7bf29 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -38,6 +38,8 @@ struct passwd { char *pw_dir; }; +#ifdef __GNUC__ +/* in MSVC, it's defined in winsock2.h */ struct pollfd { int fd; /* file descriptor */ short events; /* requested events */ @@ -45,6 +47,7 @@ struct pollfd { }; #define POLLIN 1 #define POLLHUP 2 +#endif /* __GNUC__ */ typedef void (__cdecl *sig_handler_t)(int); struct sigaction { @@ -196,13 +199,15 @@ sig_handler_t mingw_signal(int sig, sig_handler_t handler); /* * ANSI emulation wrappers */ - +#ifndef _MSC_VER +/* Until va_copy available is MSVC */ int winansi_fputs(const char *str, FILE *stream); int winansi_printf(const char *format, ...) __attribute__((format (printf, 1, 2))); int winansi_fprintf(FILE *stream, const char *format, ...) __attribute__((format (printf, 2, 3))); #define fputs winansi_fputs #define printf(...) winansi_printf(__VA_ARGS__) #define fprintf(...) winansi_fprintf(__VA_ARGS__) +#endif /* * git specific compatibility diff --git a/git-compat-util.h b/git-compat-util.h new file mode 100644 index 0000000..3a690e1 --- /dev/null +++ b/git-compat-util.h @@ -0,0 +1,183 @@ +#ifndef GIT_COMPAT_UTIL_H +#define GIT_COMPAT_UTIL_H + +#ifndef FLEX_ARRAY +/* + * See if our compiler is known to support flexible array members. + */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEX_ARRAY /* empty */ +#elif defined(__GNUC__) +# if (__GNUC__ >= 3) +# define FLEX_ARRAY /* empty */ +# else +# define FLEX_ARRAY 0 /* older GNU extension */ +# endif +#elif defined(_MSC_VER) +#define FLEX_ARRAY /* empty */ +#endif + +/* + * Otherwise, default to safer but a bit wasteful traditional style + */ +#ifndef FLEX_ARRAY +# define FLEX_ARRAY 1 +#endif +#endif + +#define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +/* MSVC defines mkdir here, not in io.h */ +#include +#endif + +#ifndef PATH_MAX +/* + * with MSVC, if we define _POSIX_ we loose half of useful functions + * (e.g. chmod, open, close), otherwise we don't have PATH_MAX + * because of #ifdef in . + * So, when build under MSVC, don't define _POSIX_ + */ +#define PATH_MAX 512 +#endif + +/* from mingw/include/sys/types.h */ +#ifndef _SSIZE_T_ +#define _SSIZE_T_ +typedef long _ssize_t; + +#ifndef _NO_OLDNAMES +typedef _ssize_t ssize_t; +#endif +#endif /* Not _SSIZE_T_ */ + +#ifndef _MODE_T_ +#define _MODE_T_ +typedef unsigned short _mode_t; + +#ifndef _NO_OLDNAMES +typedef _mode_t mode_t; +#endif +#endif /* Not _MODE_T_ */ + + +/* from mingw/include/io.h */ +/* Some defines for _access nAccessMode (MS doesn't define them, but + * it doesn't seem to hurt to add them). */ +#define F_OK 0 /* Check for file existence */ +/* Well maybe it does hurt. On newer versions of MSVCRT, an access mode + of 1 causes invalid parameter error. */ +#define X_OK 1 /* MS access() doesn't check for execute permission. */ +#define W_OK 2 /* Check for write permission */ +#define R_OK 4 /* Check for read permission */ + +/* from mingw/include/string.h */ +#define strcasecmp(__sz1, __sz2) (_stricmp((__sz1), (__sz2))) + +/* from mingw/include/stdint.h */ +typedef unsigned uint32_t; + +#ifdef __GNUC__ +#define NORETURN __attribute__((__noreturn__)) +#else + +#if defined(_MSC_VER) +#define NORETURN __declspec(noreturn) +#else +#define NORETURN +#endif /* _MSC_VER */ + +#ifndef __attribute__ +#define __attribute__(x) +#endif +#define __CRT_INLINE +#endif + +extern int error(const char * format, ...); +extern NORETURN void die(const char *err, ...) __attribute__((format (printf, 1, 2))); + +#ifndef HAVE_STRCHRNUL +#define strchrnul gitstrchrnul +static inline char *gitstrchrnul(const char *s, int c) +{ + while (*s && *s != c) + s++; + return (char *)s; +} +#endif + +extern time_t tm_to_time_t(const struct tm *tm); + +extern void release_pack_memory(size_t, int); + +extern char *xstrdup(const char *str); +extern void *xmalloc(size_t size); +extern void *xrealloc(void *ptr, size_t size); +extern void *xcalloc(size_t nmemb, size_t size); +extern ssize_t xread(int fd, void *buf, size_t len); + +#ifdef NO_STRLCPY +#define strlcpy gitstrlcpy +extern size_t gitstrlcpy(char *, const char *, size_t); +#endif + +#define snprintf _snprintf + +#ifdef NO_MMAP + +#ifndef PROT_READ +#define PROT_READ 1 +#define PROT_WRITE 2 +#define MAP_PRIVATE 1 +#define MAP_FAILED ((void*)-1) +#endif + +#define mmap git_mmap +#define munmap git_munmap +extern void *git_mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset); +extern int git_munmap(void *start, size_t length); + +/* This value must be multiple of (pagesize * 2) */ +#define DEFAULT_PACKED_GIT_WINDOW_SIZE (1 * 1024 * 1024) + +#else /* NO_MMAP */ + +#include + +/* This value must be multiple of (pagesize * 2) */ +#define DEFAULT_PACKED_GIT_WINDOW_SIZE \ + (sizeof(void*) >= 8 \ + ? 1 * 1024 * 1024 * 1024 \ + : 32 * 1024 * 1024) + +#endif /* NO_MMAP */ + +#ifdef NO_PREAD +#define pread git_pread +extern ssize_t git_pread(int fd, void *buf, size_t count, off_t offset); +#endif +/* + * Forward decl that will remind us if its twin in cache.h changes. + * This function is used in compat/pread.c. But we can't include + * cache.h there. + */ +extern ssize_t read_in_full(int fd, void *buf, size_t count); + +#include "compat/mingw.h" + +#endif diff --git a/sha1_file.c b/sha1_file.c dissimilarity index 97% index 32e4664..35f84e9 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1,2479 +1,60 @@ -/* - * GIT - The information manager from hell - * - * Copyright (C) Linus Torvalds, 2005 - * - * This handles basic git sha1 object files - packing, unpacking, - * creation etc. - */ -#include "cache.h" -#include "delta.h" -#include "pack.h" -#include "blob.h" -#include "commit.h" -#include "tag.h" -#include "tree.h" -#include "refs.h" -#include "pack-revindex.h" -#include "sha1-lookup.h" - -#ifndef O_NOATIME -#if defined(__linux__) && (defined(__i386__) || defined(__PPC__)) -#define O_NOATIME 01000000 -#else -#define O_NOATIME 0 -#endif -#endif - -#ifdef NO_C99_FORMAT -#define SZ_FMT "lu" -static unsigned long sz_fmt(size_t s) { return (unsigned long)s; } -#else -#define SZ_FMT "zu" -static size_t sz_fmt(size_t s) { return s; } -#endif - -const unsigned char null_sha1[20]; - -const signed char hexval_table[256] = { - -1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */ - 0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */ - 8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */ - -1, 10, 11, 12, 13, 14, 15, -1, /* 40-47 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */ - -1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */ - -1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */ - -1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */ - -1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */ - -1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */ - -1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */ - -1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */ - -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ -}; - -int get_sha1_hex(const char *hex, unsigned char *sha1) -{ - int i; - for (i = 0; i < 20; i++) { - unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]); - if (val & ~0xff) - return -1; - *sha1++ = val; - hex += 2; - } - return 0; -} - -static inline int offset_1st_component(const char *path) -{ - if (has_dos_drive_prefix(path)) - return 2 + (path[2] == '/'); - return *path == '/'; -} - -int safe_create_leading_directories(char *path) -{ - char *pos = path + offset_1st_component(path); - struct stat st; - - while (pos) { - pos = strchr(pos, '/'); - if (!pos) - break; - *pos = 0; - if (!stat(path, &st)) { - /* path exists */ - if (!S_ISDIR(st.st_mode)) { - *pos = '/'; - return -3; - } - } - else if (mkdir(path, 0777)) { - *pos = '/'; - return -1; - } - else if (adjust_shared_perm(path)) { - *pos = '/'; - return -2; - } - *pos++ = '/'; - } - return 0; -} - -int safe_create_leading_directories_const(const char *path) -{ - /* path points to cache entries, so xstrdup before messing with it */ - char *buf = xstrdup(path); - int result = safe_create_leading_directories(buf); - free(buf); - return result; -} - -char *sha1_to_hex(const unsigned char *sha1) -{ - static int bufno; - static char hexbuffer[4][50]; - static const char hex[] = "0123456789abcdef"; - char *buffer = hexbuffer[3 & ++bufno], *buf = buffer; - int i; - - for (i = 0; i < 20; i++) { - unsigned int val = *sha1++; - *buf++ = hex[val >> 4]; - *buf++ = hex[val & 0xf]; - } - *buf = '\0'; - - return buffer; -} - -static void fill_sha1_path(char *pathbuf, const unsigned char *sha1) -{ - int i; - for (i = 0; i < 20; i++) { - static char hex[] = "0123456789abcdef"; - unsigned int val = sha1[i]; - char *pos = pathbuf + i*2 + (i > 0); - *pos++ = hex[val >> 4]; - *pos = hex[val & 0xf]; - } -} - -/* - * NOTE! This returns a statically allocated buffer, so you have to be - * careful about using it. Do an "xstrdup()" if you need to save the - * filename. - * - * Also note that this returns the location for creating. Reading - * SHA1 file can happen from any alternate directory listed in the - * DB_ENVIRONMENT environment variable if it is not found in - * the primary object database. - */ -char *sha1_file_name(const unsigned char *sha1) -{ - static char *name, *base; - - if (!base) { - const char *sha1_file_directory = get_object_directory(); - int len = strlen(sha1_file_directory); - base = xmalloc(len + 60); - memcpy(base, sha1_file_directory, len); - memset(base+len, 0, 60); - base[len] = '/'; - base[len+3] = '/'; - name = base + len + 1; - } - fill_sha1_path(name, sha1); - return base; -} - -static char *sha1_get_pack_name(const unsigned char *sha1, - char **name, char **base, const char *which) -{ - static const char hex[] = "0123456789abcdef"; - char *buf; - int i; - - if (!*base) { - const char *sha1_file_directory = get_object_directory(); - int len = strlen(sha1_file_directory); - *base = xmalloc(len + 60); - sprintf(*base, "%s/pack/pack-1234567890123456789012345678901234567890.%s", - sha1_file_directory, which); - *name = *base + len + 11; - } - - buf = *name; - - for (i = 0; i < 20; i++) { - unsigned int val = *sha1++; - *buf++ = hex[val >> 4]; - *buf++ = hex[val & 0xf]; - } - - return *base; -} - -char *sha1_pack_name(const unsigned char *sha1) -{ - static char *name, *base; - - return sha1_get_pack_name(sha1, &name, &base, "pack"); -} - -char *sha1_pack_index_name(const unsigned char *sha1) -{ - static char *name, *base; - - return sha1_get_pack_name(sha1, &name, &base, "idx"); -} - -struct alternate_object_database *alt_odb_list; -static struct alternate_object_database **alt_odb_tail; - -static void read_info_alternates(const char * alternates, int depth); - -/* - * Prepare alternate object database registry. - * - * The variable alt_odb_list points at the list of struct - * alternate_object_database. The elements on this list come from - * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT - * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates, - * whose contents is similar to that environment variable but can be - * LF separated. Its base points at a statically allocated buffer that - * contains "/the/directory/corresponding/to/.git/objects/...", while - * its name points just after the slash at the end of ".git/objects/" - * in the example above, and has enough space to hold 40-byte hex - * SHA1, an extra slash for the first level indirection, and the - * terminating NUL. - */ -static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth) -{ - struct stat st; - const char *objdir = get_object_directory(); - struct alternate_object_database *ent; - struct alternate_object_database *alt; - /* 43 = 40-byte + 2 '/' + terminating NUL */ - int pfxlen = len; - int entlen = pfxlen + 43; - int base_len = -1; - - if (!is_absolute_path(entry) && relative_base) { - /* Relative alt-odb */ - if (base_len < 0) - base_len = strlen(relative_base) + 1; - entlen += base_len; - pfxlen += base_len; - } - ent = xmalloc(sizeof(*ent) + entlen); - - if (!is_absolute_path(entry) && relative_base) { - memcpy(ent->base, relative_base, base_len - 1); - ent->base[base_len - 1] = '/'; - memcpy(ent->base + base_len, entry, len); - } - else - memcpy(ent->base, entry, pfxlen); - - ent->name = ent->base + pfxlen + 1; - ent->base[pfxlen + 3] = '/'; - ent->base[pfxlen] = ent->base[entlen-1] = 0; - - /* Detect cases where alternate disappeared */ - if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) { - error("object directory %s does not exist; " - "check .git/objects/info/alternates.", - ent->base); - free(ent); - return -1; - } - - /* Prevent the common mistake of listing the same - * thing twice, or object directory itself. - */ - for (alt = alt_odb_list; alt; alt = alt->next) { - if (!memcmp(ent->base, alt->base, pfxlen)) { - free(ent); - return -1; - } - } - if (!memcmp(ent->base, objdir, pfxlen)) { - free(ent); - return -1; - } - - /* add the alternate entry */ - *alt_odb_tail = ent; - alt_odb_tail = &(ent->next); - ent->next = NULL; - - /* recursively add alternates */ - read_info_alternates(ent->base, depth + 1); - - ent->base[pfxlen] = '/'; - - return 0; -} - -static void link_alt_odb_entries(const char *alt, const char *ep, int sep, - const char *relative_base, int depth) -{ - const char *cp, *last; - - if (depth > 5) { - error("%s: ignoring alternate object stores, nesting too deep.", - relative_base); - return; - } - - last = alt; - while (last < ep) { - cp = last; - if (cp < ep && *cp == '#') { - while (cp < ep && *cp != sep) - cp++; - last = cp + 1; - continue; - } - while (cp < ep && *cp != sep) - cp++; - if (last != cp) { - if (!is_absolute_path(last) && depth) { - error("%s: ignoring relative alternate object store %s", - relative_base, last); - } else { - link_alt_odb_entry(last, cp - last, - relative_base, depth); - } - } - while (cp < ep && *cp == sep) - cp++; - last = cp; - } -} - -static void read_info_alternates(const char * relative_base, int depth) -{ - char *map; - size_t mapsz; - struct stat st; - const char alt_file_name[] = "info/alternates"; - /* Given that relative_base is no longer than PATH_MAX, - ensure that "path" has enough space to append "/", the - file name, "info/alternates", and a trailing NUL. */ - char path[PATH_MAX + 1 + sizeof alt_file_name]; - int fd; - - sprintf(path, "%s/%s", relative_base, alt_file_name); - fd = open(path, O_RDONLY); - if (fd < 0) - return; - if (fstat(fd, &st) || (st.st_size == 0)) { - close(fd); - return; - } - mapsz = xsize_t(st.st_size); - map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - - link_alt_odb_entries(map, map + mapsz, '\n', relative_base, depth); - - munmap(map, mapsz); -} - -void add_to_alternates_file(const char *reference) -{ - struct lock_file *lock = xcalloc(1, sizeof(struct lock_file)); - int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), 1); - char *alt = mkpath("%s/objects\n", reference); - write_or_die(fd, alt, strlen(alt)); - if (commit_lock_file(lock)) - die("could not close alternates file"); - if (alt_odb_tail) - link_alt_odb_entries(alt, alt + strlen(alt), '\n', NULL, 0); -} - -void prepare_alt_odb(void) -{ - const char *alt; - - if (alt_odb_tail) - return; - - alt = getenv(ALTERNATE_DB_ENVIRONMENT); - if (!alt) alt = ""; - - alt_odb_tail = &alt_odb_list; - link_alt_odb_entries(alt, alt + strlen(alt), PATH_SEP, NULL, 0); - - read_info_alternates(get_object_directory(), 0); -} - -static int has_loose_object(const unsigned char *sha1) -{ - char *name = sha1_file_name(sha1); - struct alternate_object_database *alt; - - if (!access(name, F_OK)) - return 1; - prepare_alt_odb(); - for (alt = alt_odb_list; alt; alt = alt->next) { - name = alt->name; - fill_sha1_path(name, sha1); - if (!access(alt->base, F_OK)) - return 1; - } - return 0; -} - -static unsigned int pack_used_ctr; -static unsigned int pack_mmap_calls; -static unsigned int peak_pack_open_windows; -static unsigned int pack_open_windows; -static size_t peak_pack_mapped; -static size_t pack_mapped; -struct packed_git *packed_git; - -void pack_report(void) -{ - fprintf(stderr, - "pack_report: getpagesize() = %10" SZ_FMT "\n" - "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n" - "pack_report: core.packedGitLimit = %10" SZ_FMT "\n", - sz_fmt(getpagesize()), - sz_fmt(packed_git_window_size), - sz_fmt(packed_git_limit)); - fprintf(stderr, - "pack_report: pack_used_ctr = %10u\n" - "pack_report: pack_mmap_calls = %10u\n" - "pack_report: pack_open_windows = %10u / %10u\n" - "pack_report: pack_mapped = " - "%10" SZ_FMT " / %10" SZ_FMT "\n", - pack_used_ctr, - pack_mmap_calls, - pack_open_windows, peak_pack_open_windows, - sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped)); -} - -static int check_packed_git_idx(const char *path, struct packed_git *p) -{ - void *idx_map; - struct pack_idx_header *hdr; - size_t idx_size; - uint32_t version, nr, i, *index; - int fd = open(path, O_RDONLY); - struct stat st; - - if (fd < 0) - return -1; - if (fstat(fd, &st)) { - close(fd); - return -1; - } - idx_size = xsize_t(st.st_size); - if (idx_size < 4 * 256 + 20 + 20) { - close(fd); - return error("index file %s is too small", path); - } - idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - - hdr = idx_map; - if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) { - version = ntohl(hdr->idx_version); - if (version < 2 || version > 2) { - munmap(idx_map, idx_size); - return error("index file %s is version %"PRIu32 - " and is not supported by this binary" - " (try upgrading GIT to a newer version)", - path, version); - } - } else - version = 1; - - nr = 0; - index = idx_map; - if (version > 1) - index += 2; /* skip index header */ - for (i = 0; i < 256; i++) { - uint32_t n = ntohl(index[i]); - if (n < nr) { - munmap(idx_map, idx_size); - return error("non-monotonic index %s", path); - } - nr = n; - } - - if (version == 1) { - /* - * Total size: - * - 256 index entries 4 bytes each - * - 24-byte entries * nr (20-byte sha1 + 4-byte offset) - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum - */ - if (idx_size != 4*256 + nr * 24 + 20 + 20) { - munmap(idx_map, idx_size); - return error("wrong index v1 file size in %s", path); - } - } else if (version == 2) { - /* - * Minimum size: - * - 8 bytes of header - * - 256 index entries 4 bytes each - * - 20-byte sha1 entry * nr - * - 4-byte crc entry * nr - * - 4-byte offset entry * nr - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum - * And after the 4-byte offset table might be a - * variable sized table containing 8-byte entries - * for offsets larger than 2^31. - */ - unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; - unsigned long max_size = min_size; - if (nr) - max_size += (nr - 1)*8; - if (idx_size < min_size || idx_size > max_size) { - munmap(idx_map, idx_size); - return error("wrong index v2 file size in %s", path); - } - if (idx_size != min_size && - /* - * make sure we can deal with large pack offsets. - * 31-bit signed offset won't be enough, neither - * 32-bit unsigned one will be. - */ - (sizeof(off_t) <= 4)) { - munmap(idx_map, idx_size); - return error("pack too large for current definition of off_t in %s", path); - } - } - - p->index_version = version; - p->index_data = idx_map; - p->index_size = idx_size; - p->num_objects = nr; - return 0; -} - -int open_pack_index(struct packed_git *p) -{ - char *idx_name; - int ret; - - if (p->index_data) - return 0; - - idx_name = xstrdup(p->pack_name); - strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx"); - ret = check_packed_git_idx(idx_name, p); - free(idx_name); - return ret; -} - -static void scan_windows(struct packed_git *p, - struct packed_git **lru_p, - struct pack_window **lru_w, - struct pack_window **lru_l) -{ - struct pack_window *w, *w_l; - - for (w_l = NULL, w = p->windows; w; w = w->next) { - if (!w->inuse_cnt) { - if (!*lru_w || w->last_used < (*lru_w)->last_used) { - *lru_p = p; - *lru_w = w; - *lru_l = w_l; - } - } - w_l = w; - } -} - -static int unuse_one_window(struct packed_git *current, int keep_fd) -{ - struct packed_git *p, *lru_p = NULL; - struct pack_window *lru_w = NULL, *lru_l = NULL; - - if (current) - scan_windows(current, &lru_p, &lru_w, &lru_l); - for (p = packed_git; p; p = p->next) - scan_windows(p, &lru_p, &lru_w, &lru_l); - if (lru_p) { - munmap(lru_w->base, lru_w->len); - pack_mapped -= lru_w->len; - if (lru_l) - lru_l->next = lru_w->next; - else { - lru_p->windows = lru_w->next; - if (!lru_p->windows && lru_p->pack_fd != keep_fd) { - close(lru_p->pack_fd); - lru_p->pack_fd = -1; - } - } - free(lru_w); - pack_open_windows--; - return 1; - } - return 0; -} - -void release_pack_memory(size_t need, int fd) -{ - size_t cur = pack_mapped; - while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd)) - ; /* nothing */ -} - -void close_pack_windows(struct packed_git *p) -{ - while (p->windows) { - struct pack_window *w = p->windows; - - if (w->inuse_cnt) - die("pack '%s' still has open windows to it", - p->pack_name); - munmap(w->base, w->len); - pack_mapped -= w->len; - pack_open_windows--; - p->windows = w->next; - free(w); - } -} - -void unuse_pack(struct pack_window **w_cursor) -{ - struct pack_window *w = *w_cursor; - if (w) { - w->inuse_cnt--; - *w_cursor = NULL; - } -} - -/* - * Do not call this directly as this leaks p->pack_fd on error return; - * call open_packed_git() instead. - */ -static int open_packed_git_1(struct packed_git *p) -{ - struct stat st; - struct pack_header hdr; - unsigned char sha1[20]; - unsigned char *idx_sha1; - long fd_flag; - - if (!p->index_data && open_pack_index(p)) - return error("packfile %s index unavailable", p->pack_name); - - p->pack_fd = open(p->pack_name, O_RDONLY); - if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) - return -1; - - /* If we created the struct before we had the pack we lack size. */ - if (!p->pack_size) { - if (!S_ISREG(st.st_mode)) - return error("packfile %s not a regular file", p->pack_name); - p->pack_size = st.st_size; - } else if (p->pack_size != st.st_size) - return error("packfile %s size changed", p->pack_name); - - /* We leave these file descriptors open with sliding mmap; - * there is no point keeping them open across exec(), though. - */ - fd_flag = fcntl(p->pack_fd, F_GETFD, 0); - if (fd_flag < 0) - return error("cannot determine file descriptor flags"); - fd_flag |= FD_CLOEXEC; - if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1) - return error("cannot set FD_CLOEXEC"); - - /* Verify we recognize this pack file format. */ - if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr)) - return error("file %s is far too short to be a packfile", p->pack_name); - if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) - return error("file %s is not a GIT packfile", p->pack_name); - if (!pack_version_ok(hdr.hdr_version)) - return error("packfile %s is version %"PRIu32" and not" - " supported (try upgrading GIT to a newer version)", - p->pack_name, ntohl(hdr.hdr_version)); - - /* Verify the pack matches its index. */ - if (p->num_objects != ntohl(hdr.hdr_entries)) - return error("packfile %s claims to have %"PRIu32" objects" - " while index indicates %"PRIu32" objects", - p->pack_name, ntohl(hdr.hdr_entries), - p->num_objects); - if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1) - return error("end of packfile %s is unavailable", p->pack_name); - if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1)) - return error("packfile %s signature is unavailable", p->pack_name); - idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40; - if (hashcmp(sha1, idx_sha1)) - return error("packfile %s does not match index", p->pack_name); - return 0; -} - -static int open_packed_git(struct packed_git *p) -{ - if (!open_packed_git_1(p)) - return 0; - if (p->pack_fd != -1) { - close(p->pack_fd); - p->pack_fd = -1; - } - return -1; -} - -static int in_window(struct pack_window *win, off_t offset) -{ - /* We must promise at least 20 bytes (one hash) after the - * offset is available from this window, otherwise the offset - * is not actually in this window and a different window (which - * has that one hash excess) must be used. This is to support - * the object header and delta base parsing routines below. - */ - off_t win_off = win->offset; - return win_off <= offset - && (offset + 20) <= (win_off + win->len); -} - -unsigned char* use_pack(struct packed_git *p, - struct pack_window **w_cursor, - off_t offset, - unsigned int *left) -{ - struct pack_window *win = *w_cursor; - - if (p->pack_fd == -1 && open_packed_git(p)) - die("packfile %s cannot be accessed", p->pack_name); - - /* Since packfiles end in a hash of their content and its - * pointless to ask for an offset into the middle of that - * hash, and the in_window function above wouldn't match - * don't allow an offset too close to the end of the file. - */ - if (offset > (p->pack_size - 20)) - die("offset beyond end of packfile (truncated pack?)"); - - if (!win || !in_window(win, offset)) { - if (win) - win->inuse_cnt--; - for (win = p->windows; win; win = win->next) { - if (in_window(win, offset)) - break; - } - if (!win) { - size_t window_align = packed_git_window_size / 2; - off_t len; - win = xcalloc(1, sizeof(*win)); - win->offset = (offset / window_align) * window_align; - len = p->pack_size - win->offset; - if (len > packed_git_window_size) - len = packed_git_window_size; - win->len = (size_t)len; - pack_mapped += win->len; - while (packed_git_limit < pack_mapped - && unuse_one_window(p, p->pack_fd)) - ; /* nothing */ - win->base = xmmap(NULL, win->len, - PROT_READ, MAP_PRIVATE, - p->pack_fd, win->offset); - if (win->base == MAP_FAILED) - die("packfile %s cannot be mapped: %s", - p->pack_name, - strerror(errno)); - pack_mmap_calls++; - pack_open_windows++; - if (pack_mapped > peak_pack_mapped) - peak_pack_mapped = pack_mapped; - if (pack_open_windows > peak_pack_open_windows) - peak_pack_open_windows = pack_open_windows; - win->next = p->windows; - p->windows = win; - } - } - if (win != *w_cursor) { - win->last_used = pack_used_ctr++; - win->inuse_cnt++; - *w_cursor = win; - } - offset -= win->offset; - if (left) - *left = win->len - xsize_t(offset); - return win->base + offset; -} - -static struct packed_git *alloc_packed_git(int extra) -{ - struct packed_git *p = xmalloc(sizeof(*p) + extra); - memset(p, 0, sizeof(*p)); - p->pack_fd = -1; - return p; -} - -struct packed_git *add_packed_git(const char *path, int path_len, int local) -{ - struct stat st; - struct packed_git *p = alloc_packed_git(path_len + 2); - - /* - * Make sure a corresponding .pack file exists and that - * the index looks sane. - */ - path_len -= strlen(".idx"); - if (path_len < 1) { - free(p); - return NULL; - } - memcpy(p->pack_name, path, path_len); - strcpy(p->pack_name + path_len, ".pack"); - if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) { - free(p); - return NULL; - } - - /* ok, it looks sane as far as we can check without - * actually mapping the pack file. - */ - p->pack_size = st.st_size; - p->pack_local = local; - p->mtime = st.st_mtime; - if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1)) - hashclr(p->sha1); - return p; -} - -struct packed_git *parse_pack_index(unsigned char *sha1) -{ - const char *idx_path = sha1_pack_index_name(sha1); - const char *path = sha1_pack_name(sha1); - struct packed_git *p = alloc_packed_git(strlen(path) + 1); - - strcpy(p->pack_name, path); - hashcpy(p->sha1, sha1); - if (check_packed_git_idx(idx_path, p)) { - free(p); - return NULL; - } - - return p; -} - -void install_packed_git(struct packed_git *pack) -{ - pack->next = packed_git; - packed_git = pack; -} - -static void prepare_packed_git_one(char *objdir, int local) -{ - /* Ensure that this buffer is large enough so that we can - append "/pack/" without clobbering the stack even if - strlen(objdir) were PATH_MAX. */ - char path[PATH_MAX + 1 + 4 + 1 + 1]; - int len; - DIR *dir; - struct dirent *de; - - sprintf(path, "%s/pack", objdir); - len = strlen(path); - dir = opendir(path); - if (!dir) { - if (errno != ENOENT) - error("unable to open object pack directory: %s: %s", - path, strerror(errno)); - return; - } - path[len++] = '/'; - while ((de = readdir(dir)) != NULL) { - int namelen = strlen(de->d_name); - struct packed_git *p; - - if (!has_extension(de->d_name, ".idx")) - continue; - - if (len + namelen + 1 > sizeof(path)) - continue; - - /* Don't reopen a pack we already have. */ - strcpy(path + len, de->d_name); - for (p = packed_git; p; p = p->next) { - if (!memcmp(path, p->pack_name, len + namelen - 4)) - break; - } - if (p) - continue; - /* See if it really is a valid .idx file with corresponding - * .pack file that we can map. - */ - p = add_packed_git(path, len + namelen, local); - if (!p) - continue; - install_packed_git(p); - } - closedir(dir); -} - -static int sort_pack(const void *a_, const void *b_) -{ - struct packed_git *a = *((struct packed_git **)a_); - struct packed_git *b = *((struct packed_git **)b_); - int st; - - /* - * Local packs tend to contain objects specific to our - * variant of the project than remote ones. In addition, - * remote ones could be on a network mounted filesystem. - * Favor local ones for these reasons. - */ - st = a->pack_local - b->pack_local; - if (st) - return -st; - - /* - * Younger packs tend to contain more recent objects, - * and more recent objects tend to get accessed more - * often. - */ - if (a->mtime < b->mtime) - return 1; - else if (a->mtime == b->mtime) - return 0; - return -1; -} - -static void rearrange_packed_git(void) -{ - struct packed_git **ary, *p; - int i, n; - - for (n = 0, p = packed_git; p; p = p->next) - n++; - if (n < 2) - return; - - /* prepare an array of packed_git for easier sorting */ - ary = xcalloc(n, sizeof(struct packed_git *)); - for (n = 0, p = packed_git; p; p = p->next) - ary[n++] = p; - - qsort(ary, n, sizeof(struct packed_git *), sort_pack); - - /* link them back again */ - for (i = 0; i < n - 1; i++) - ary[i]->next = ary[i + 1]; - ary[n - 1]->next = NULL; - packed_git = ary[0]; - - free(ary); -} - -static int prepare_packed_git_run_once = 0; -void prepare_packed_git(void) -{ - struct alternate_object_database *alt; - - if (prepare_packed_git_run_once) - return; - prepare_packed_git_one(get_object_directory(), 1); - prepare_alt_odb(); - for (alt = alt_odb_list; alt; alt = alt->next) { - alt->name[-1] = 0; - prepare_packed_git_one(alt->base, 0); - alt->name[-1] = '/'; - } - rearrange_packed_git(); - prepare_packed_git_run_once = 1; -} - -void reprepare_packed_git(void) -{ - prepare_packed_git_run_once = 0; - prepare_packed_git(); -} - -static void mark_bad_packed_object(struct packed_git *p, - const unsigned char *sha1) -{ - unsigned i; - for (i = 0; i < p->num_bad_objects; i++) - if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i)) - return; - p->bad_object_sha1 = xrealloc(p->bad_object_sha1, 20 * (p->num_bad_objects + 1)); - hashcpy(p->bad_object_sha1 + 20 * p->num_bad_objects, sha1); - p->num_bad_objects++; -} - -static int has_packed_and_bad(const unsigned char *sha1) -{ - struct packed_git *p; - unsigned i; - - for (p = packed_git; p; p = p->next) - for (i = 0; i < p->num_bad_objects; i++) - if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i)) - return 1; - return 0; -} - -int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type) -{ - unsigned char real_sha1[20]; - hash_sha1_file(map, size, type, real_sha1); - return hashcmp(sha1, real_sha1) ? -1 : 0; -} - -static int git_open_noatime(const char *name) -{ - static int sha1_file_open_flag = O_NOATIME; - int fd = open(name, O_RDONLY | sha1_file_open_flag); - - /* Might the failure be due to O_NOATIME? */ - if (fd < 0 && errno != ENOENT && sha1_file_open_flag) { - fd = open(name, O_RDONLY); - if (fd >= 0) - sha1_file_open_flag = 0; - } - return fd; -} - -static int open_sha1_file(const unsigned char *sha1) -{ - int fd; - char *name = sha1_file_name(sha1); - struct alternate_object_database *alt; - - fd = git_open_noatime(name); - if (fd >= 0) - return fd; - - prepare_alt_odb(); - errno = ENOENT; - for (alt = alt_odb_list; alt; alt = alt->next) { - name = alt->name; - fill_sha1_path(name, sha1); - fd = git_open_noatime(alt->base); - if (fd >= 0) - return fd; - } - return -1; -} - -static void *map_sha1_file(const unsigned char *sha1, unsigned long *size) -{ - void *map; - int fd; - - fd = open_sha1_file(sha1); - map = NULL; - if (fd >= 0) { - struct stat st; - - if (!fstat(fd, &st)) { - *size = xsize_t(st.st_size); - map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0); - } - close(fd); - } - return map; -} - -static int legacy_loose_object(unsigned char *map) -{ - unsigned int word; - - /* - * Is it a zlib-compressed buffer? If so, the first byte - * must be 0x78 (15-bit window size, deflated), and the - * first 16-bit word is evenly divisible by 31 - */ - word = (map[0] << 8) + map[1]; - if (map[0] == 0x78 && !(word % 31)) - return 1; - else - return 0; -} - -unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep) -{ - unsigned shift; - unsigned char c; - unsigned long size; - unsigned long used = 0; - - c = buf[used++]; - *type = (c >> 4) & 7; - size = c & 15; - shift = 4; - while (c & 0x80) { - if (len <= used) - return 0; - if (sizeof(long) * 8 <= shift) - return 0; - c = buf[used++]; - size += (c & 0x7f) << shift; - shift += 7; - } - *sizep = size; - return used; -} - -static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) -{ - unsigned long size, used; - static const char valid_loose_object_type[8] = { - 0, /* OBJ_EXT */ - 1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */ - 0, /* "delta" and others are invalid in a loose object */ - }; - enum object_type type; - - /* Get the data stream */ - memset(stream, 0, sizeof(*stream)); - stream->next_in = map; - stream->avail_in = mapsize; - stream->next_out = buffer; - stream->avail_out = bufsiz; - - if (legacy_loose_object(map)) { - inflateInit(stream); - return inflate(stream, 0); - } - - - /* - * There used to be a second loose object header format which - * was meant to mimic the in-pack format, allowing for direct - * copy of the object data. This format turned up not to be - * really worth it and we don't write it any longer. But we - * can still read it. - */ - used = unpack_object_header_gently(map, mapsize, &type, &size); - if (!used || !valid_loose_object_type[type]) - return -1; - map += used; - mapsize -= used; - - /* Set up the stream for the rest.. */ - stream->next_in = map; - stream->avail_in = mapsize; - inflateInit(stream); - - /* And generate the fake traditional header */ - stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", - typename(type), size); - return 0; -} - -static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1) -{ - int bytes = strlen(buffer) + 1; - unsigned char *buf = xmalloc(1+size); - unsigned long n; - int status = Z_OK; - - n = stream->total_out - bytes; - if (n > size) - n = size; - memcpy(buf, (char *) buffer + bytes, n); - bytes = n; - if (bytes <= size) { - /* - * The above condition must be (bytes <= size), not - * (bytes < size). In other words, even though we - * expect no more output and set avail_out to zer0, - * the input zlib stream may have bytes that express - * "this concludes the stream", and we *do* want to - * eat that input. - * - * Otherwise we would not be able to test that we - * consumed all the input to reach the expected size; - * we also want to check that zlib tells us that all - * went well with status == Z_STREAM_END at the end. - */ - stream->next_out = buf + bytes; - stream->avail_out = size - bytes; - while (status == Z_OK) - status = inflate(stream, Z_FINISH); - } - buf[size] = 0; - if (status == Z_STREAM_END && !stream->avail_in) { - inflateEnd(stream); - return buf; - } - - if (status < 0) - error("corrupt loose object '%s'", sha1_to_hex(sha1)); - else if (stream->avail_in) - error("garbage at end of loose object '%s'", - sha1_to_hex(sha1)); - free(buf); - return NULL; -} - -/* - * We used to just use "sscanf()", but that's actually way - * too permissive for what we want to check. So do an anal - * object header parse by hand. - */ -static int parse_sha1_header(const char *hdr, unsigned long *sizep) -{ - char type[10]; - int i; - unsigned long size; - - /* - * The type can be at most ten bytes (including the - * terminating '\0' that we add), and is followed by - * a space. - */ - i = 0; - for (;;) { - char c = *hdr++; - if (c == ' ') - break; - type[i++] = c; - if (i >= sizeof(type)) - return -1; - } - type[i] = 0; - - /* - * The length must follow immediately, and be in canonical - * decimal format (ie "010" is not valid). - */ - size = *hdr++ - '0'; - if (size > 9) - return -1; - if (size) { - for (;;) { - unsigned long c = *hdr - '0'; - if (c > 9) - break; - hdr++; - size = size * 10 + c; - } - } - *sizep = size; - - /* - * The length must be followed by a zero byte - */ - return *hdr ? -1 : type_from_string(type); -} - -static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) -{ - int ret; - z_stream stream; - char hdr[8192]; - - ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)); - if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0) - return NULL; - - return unpack_sha1_rest(&stream, hdr, *size, sha1); -} - -unsigned long get_size_from_delta(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos) -{ - const unsigned char *data; - unsigned char delta_head[20], *in; - z_stream stream; - int st; - - memset(&stream, 0, sizeof(stream)); - stream.next_out = delta_head; - stream.avail_out = sizeof(delta_head); - - inflateInit(&stream); - do { - in = use_pack(p, w_curs, curpos, &stream.avail_in); - stream.next_in = in; - st = inflate(&stream, Z_FINISH); - curpos += stream.next_in - in; - } while ((st == Z_OK || st == Z_BUF_ERROR) && - stream.total_out < sizeof(delta_head)); - inflateEnd(&stream); - if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) - die("delta data unpack-initial failed"); - - /* Examine the initial part of the delta to figure out - * the result size. - */ - data = delta_head; - - /* ignore base size */ - get_delta_hdr_size(&data, delta_head+sizeof(delta_head)); - - /* Read the result size */ - return get_delta_hdr_size(&data, delta_head+sizeof(delta_head)); -} - -static off_t get_delta_base(struct packed_git *p, - struct pack_window **w_curs, - off_t *curpos, - enum object_type type, - off_t delta_obj_offset) -{ - unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL); - off_t base_offset; - - /* use_pack() assured us we have [base_info, base_info + 20) - * as a range that we can look at without walking off the - * end of the mapped window. Its actually the hash size - * that is assured. An OFS_DELTA longer than the hash size - * is stupid, as then a REF_DELTA would be smaller to store. - */ - if (type == OBJ_OFS_DELTA) { - unsigned used = 0; - unsigned char c = base_info[used++]; - base_offset = c & 127; - while (c & 128) { - base_offset += 1; - if (!base_offset || MSB(base_offset, 7)) - return 0; /* overflow */ - c = base_info[used++]; - base_offset = (base_offset << 7) + (c & 127); - } - base_offset = delta_obj_offset - base_offset; - if (base_offset >= delta_obj_offset) - return 0; /* out of bound */ - *curpos += used; - } else if (type == OBJ_REF_DELTA) { - /* The base entry _must_ be in the same pack */ - base_offset = find_pack_entry_one(base_info, p); - *curpos += 20; - } else - die("I am totally screwed"); - return base_offset; -} - -/* forward declaration for a mutually recursive function */ -static int packed_object_info(struct packed_git *p, off_t offset, - unsigned long *sizep); - -static int packed_delta_info(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - enum object_type type, - off_t obj_offset, - unsigned long *sizep) -{ - off_t base_offset; - - base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset); - type = packed_object_info(p, base_offset, NULL); - - /* We choose to only get the type of the base object and - * ignore potentially corrupt pack file that expects the delta - * based on a base with a wrong size. This saves tons of - * inflate() calls. - */ - if (sizep) - *sizep = get_size_from_delta(p, w_curs, curpos); - - return type; -} - -static int unpack_object_header(struct packed_git *p, - struct pack_window **w_curs, - off_t *curpos, - unsigned long *sizep) -{ - unsigned char *base; - unsigned int left; - unsigned long used; - enum object_type type; - - /* use_pack() assures us we have [base, base + 20) available - * as a range that we can look at at. (Its actually the hash - * size that is assured.) With our object header encoding - * the maximum deflated object size is 2^137, which is just - * insane, so we know won't exceed what we have been given. - */ - base = use_pack(p, w_curs, *curpos, &left); - used = unpack_object_header_gently(base, left, &type, sizep); - if (!used) - die("object offset outside of pack file"); - *curpos += used; - - return type; -} - -const char *packed_object_info_detail(struct packed_git *p, - off_t obj_offset, - unsigned long *size, - unsigned long *store_size, - unsigned int *delta_chain_length, - unsigned char *base_sha1) -{ - struct pack_window *w_curs = NULL; - off_t curpos; - unsigned long dummy; - unsigned char *next_sha1; - enum object_type type; - struct revindex_entry *revidx; - - *delta_chain_length = 0; - curpos = obj_offset; - type = unpack_object_header(p, &w_curs, &curpos, size); - - revidx = find_pack_revindex(p, obj_offset); - *store_size = revidx[1].offset - obj_offset; - - for (;;) { - switch (type) { - default: - die("pack %s contains unknown object type %d", - p->pack_name, type); - case OBJ_COMMIT: - case OBJ_TREE: - case OBJ_BLOB: - case OBJ_TAG: - unuse_pack(&w_curs); - return typename(type); - case OBJ_OFS_DELTA: - obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset); - if (!obj_offset) - die("pack %s contains bad delta base reference of type %s", - p->pack_name, typename(type)); - if (*delta_chain_length == 0) { - revidx = find_pack_revindex(p, obj_offset); - hashcpy(base_sha1, nth_packed_object_sha1(p, revidx->nr)); - } - break; - case OBJ_REF_DELTA: - next_sha1 = use_pack(p, &w_curs, curpos, NULL); - if (*delta_chain_length == 0) - hashcpy(base_sha1, next_sha1); - obj_offset = find_pack_entry_one(next_sha1, p); - break; - } - (*delta_chain_length)++; - curpos = obj_offset; - type = unpack_object_header(p, &w_curs, &curpos, &dummy); - } -} - -static int packed_object_info(struct packed_git *p, off_t obj_offset, - unsigned long *sizep) -{ - struct pack_window *w_curs = NULL; - unsigned long size; - off_t curpos = obj_offset; - enum object_type type; - - type = unpack_object_header(p, &w_curs, &curpos, &size); - - switch (type) { - case OBJ_OFS_DELTA: - case OBJ_REF_DELTA: - type = packed_delta_info(p, &w_curs, curpos, - type, obj_offset, sizep); - break; - case OBJ_COMMIT: - case OBJ_TREE: - case OBJ_BLOB: - case OBJ_TAG: - if (sizep) - *sizep = size; - break; - default: - die("pack %s contains unknown object type %d", - p->pack_name, type); - } - unuse_pack(&w_curs); - return type; -} - -static void *unpack_compressed_entry(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - unsigned long size) -{ - int st; - z_stream stream; - unsigned char *buffer, *in; - - buffer = xmalloc(size + 1); - buffer[size] = 0; - memset(&stream, 0, sizeof(stream)); - stream.next_out = buffer; - stream.avail_out = size; - - inflateInit(&stream); - do { - in = use_pack(p, w_curs, curpos, &stream.avail_in); - stream.next_in = in; - st = inflate(&stream, Z_FINISH); - curpos += stream.next_in - in; - } while (st == Z_OK || st == Z_BUF_ERROR); - inflateEnd(&stream); - if ((st != Z_STREAM_END) || stream.total_out != size) { - free(buffer); - return NULL; - } - - return buffer; -} - -#define MAX_DELTA_CACHE (256) - -static size_t delta_base_cached; - -static struct delta_base_cache_lru_list { - struct delta_base_cache_lru_list *prev; - struct delta_base_cache_lru_list *next; -} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru }; - -static struct delta_base_cache_entry { - struct delta_base_cache_lru_list lru; - void *data; - struct packed_git *p; - off_t base_offset; - unsigned long size; - enum object_type type; -} delta_base_cache[MAX_DELTA_CACHE]; - -static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset) -{ - unsigned long hash; - - hash = (unsigned long)p + (unsigned long)base_offset; - hash += (hash >> 8) + (hash >> 16); - return hash % MAX_DELTA_CACHE; -} - -static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset, - unsigned long *base_size, enum object_type *type, int keep_cache) -{ - void *ret; - unsigned long hash = pack_entry_hash(p, base_offset); - struct delta_base_cache_entry *ent = delta_base_cache + hash; - - ret = ent->data; - if (ret && ent->p == p && ent->base_offset == base_offset) - goto found_cache_entry; - return unpack_entry(p, base_offset, type, base_size); - -found_cache_entry: - if (!keep_cache) { - ent->data = NULL; - ent->lru.next->prev = ent->lru.prev; - ent->lru.prev->next = ent->lru.next; - delta_base_cached -= ent->size; - } else { - ret = xmemdupz(ent->data, ent->size); - } - *type = ent->type; - *base_size = ent->size; - return ret; -} - -static inline void release_delta_base_cache(struct delta_base_cache_entry *ent) -{ - if (ent->data) { - free(ent->data); - ent->data = NULL; - ent->lru.next->prev = ent->lru.prev; - ent->lru.prev->next = ent->lru.next; - delta_base_cached -= ent->size; - } -} - -static void add_delta_base_cache(struct packed_git *p, off_t base_offset, - void *base, unsigned long base_size, enum object_type type) -{ - unsigned long hash = pack_entry_hash(p, base_offset); - struct delta_base_cache_entry *ent = delta_base_cache + hash; - struct delta_base_cache_lru_list *lru; - - release_delta_base_cache(ent); - delta_base_cached += base_size; - - for (lru = delta_base_cache_lru.next; - delta_base_cached > delta_base_cache_limit - && lru != &delta_base_cache_lru; - lru = lru->next) { - struct delta_base_cache_entry *f = (void *)lru; - if (f->type == OBJ_BLOB) - release_delta_base_cache(f); - } - for (lru = delta_base_cache_lru.next; - delta_base_cached > delta_base_cache_limit - && lru != &delta_base_cache_lru; - lru = lru->next) { - struct delta_base_cache_entry *f = (void *)lru; - release_delta_base_cache(f); - } - - ent->p = p; - ent->base_offset = base_offset; - ent->type = type; - ent->data = base; - ent->size = base_size; - ent->lru.next = &delta_base_cache_lru; - ent->lru.prev = delta_base_cache_lru.prev; - delta_base_cache_lru.prev->next = &ent->lru; - delta_base_cache_lru.prev = &ent->lru; -} - -static void *unpack_delta_entry(struct packed_git *p, - struct pack_window **w_curs, - off_t curpos, - unsigned long delta_size, - off_t obj_offset, - enum object_type *type, - unsigned long *sizep) -{ - void *delta_data, *result, *base; - unsigned long base_size; - off_t base_offset; - - base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset); - if (!base_offset) { - error("failed to validate delta base reference " - "at offset %"PRIuMAX" from %s", - (uintmax_t)curpos, p->pack_name); - return NULL; - } - unuse_pack(w_curs); - base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0); - if (!base) { - /* - * We're probably in deep shit, but let's try to fetch - * the required base anyway from another pack or loose. - * This is costly but should happen only in the presence - * of a corrupted pack, and is better than failing outright. - */ - struct revindex_entry *revidx = find_pack_revindex(p, base_offset); - const unsigned char *base_sha1 = - nth_packed_object_sha1(p, revidx->nr); - error("failed to read delta base object %s" - " at offset %"PRIuMAX" from %s", - sha1_to_hex(base_sha1), (uintmax_t)base_offset, - p->pack_name); - mark_bad_packed_object(p, base_sha1); - base = read_object(base_sha1, type, &base_size); - if (!base) - return NULL; - } - - delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size); - if (!delta_data) { - error("failed to unpack compressed delta " - "at offset %"PRIuMAX" from %s", - (uintmax_t)curpos, p->pack_name); - free(base); - return NULL; - } - result = patch_delta(base, base_size, - delta_data, delta_size, - sizep); - if (!result) - die("failed to apply delta"); - free(delta_data); - add_delta_base_cache(p, base_offset, base, base_size, *type); - return result; -} - -void *unpack_entry(struct packed_git *p, off_t obj_offset, - enum object_type *type, unsigned long *sizep) -{ - struct pack_window *w_curs = NULL; - off_t curpos = obj_offset; - void *data; - - *type = unpack_object_header(p, &w_curs, &curpos, sizep); - switch (*type) { - case OBJ_OFS_DELTA: - case OBJ_REF_DELTA: - data = unpack_delta_entry(p, &w_curs, curpos, *sizep, - obj_offset, type, sizep); - break; - case OBJ_COMMIT: - case OBJ_TREE: - case OBJ_BLOB: - case OBJ_TAG: - data = unpack_compressed_entry(p, &w_curs, curpos, *sizep); - break; - default: - data = NULL; - error("unknown object type %i at offset %"PRIuMAX" in %s", - *type, (uintmax_t)obj_offset, p->pack_name); - } - unuse_pack(&w_curs); - return data; -} - -const unsigned char *nth_packed_object_sha1(struct packed_git *p, - uint32_t n) -{ - const unsigned char *index = p->index_data; - if (!index) { - if (open_pack_index(p)) - return NULL; - index = p->index_data; - } - if (n >= p->num_objects) - return NULL; - index += 4 * 256; - if (p->index_version == 1) { - return index + 24 * n + 4; - } else { - index += 8; - return index + 20 * n; - } -} - -off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n) -{ - const unsigned char *index = p->index_data; - index += 4 * 256; - if (p->index_version == 1) { - return ntohl(*((uint32_t *)(index + 24 * n))); - } else { - uint32_t off; - index += 8 + p->num_objects * (20 + 4); - off = ntohl(*((uint32_t *)(index + 4 * n))); - if (!(off & 0x80000000)) - return off; - index += p->num_objects * 4 + (off & 0x7fffffff) * 8; - return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | - ntohl(*((uint32_t *)(index + 4))); - } -} - -off_t find_pack_entry_one(const unsigned char *sha1, - struct packed_git *p) -{ - const uint32_t *level1_ofs = p->index_data; - const unsigned char *index = p->index_data; - unsigned hi, lo, stride; - static int use_lookup = -1; - static int debug_lookup = -1; - - if (debug_lookup < 0) - debug_lookup = !!getenv("GIT_DEBUG_LOOKUP"); - - if (!index) { - if (open_pack_index(p)) - return 0; - level1_ofs = p->index_data; - index = p->index_data; - } - if (p->index_version > 1) { - level1_ofs += 2; - index += 8; - } - index += 4 * 256; - hi = ntohl(level1_ofs[*sha1]); - lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1])); - if (p->index_version > 1) { - stride = 20; - } else { - stride = 24; - index += 4; - } - - if (debug_lookup) - printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n", - sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects); - - if (use_lookup < 0) - use_lookup = !!getenv("GIT_USE_LOOKUP"); - if (use_lookup) { - int pos = sha1_entry_pos(index, stride, 0, - lo, hi, p->num_objects, sha1); - if (pos < 0) - return 0; - return nth_packed_object_offset(p, pos); - } - - do { - unsigned mi = (lo + hi) / 2; - int cmp = hashcmp(index + mi * stride, sha1); - - if (debug_lookup) - printf("lo %u hi %u rg %u mi %u\n", - lo, hi, hi - lo, mi); - if (!cmp) - return nth_packed_object_offset(p, mi); - if (cmp > 0) - hi = mi; - else - lo = mi+1; - } while (lo < hi); - return 0; -} - -int matches_pack_name(struct packed_git *p, const char *name) -{ - const char *last_c, *c; - - if (!strcmp(p->pack_name, name)) - return 1; - - for (c = p->pack_name, last_c = c; *c;) - if (*c == '/') - last_c = ++c; - else - ++c; - if (!strcmp(last_c, name)) - return 1; - - return 0; -} - -static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed) -{ - static struct packed_git *last_found = (void *)1; - struct packed_git *p; - off_t offset; - - prepare_packed_git(); - if (!packed_git) - return 0; - p = (last_found == (void *)1) ? packed_git : last_found; - - do { - if (ignore_packed) { - const char **ig; - for (ig = ignore_packed; *ig; ig++) - if (matches_pack_name(p, *ig)) - break; - if (*ig) - goto next; - } - - if (p->num_bad_objects) { - unsigned i; - for (i = 0; i < p->num_bad_objects; i++) - if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i)) - goto next; - } - - offset = find_pack_entry_one(sha1, p); - if (offset) { - /* - * We are about to tell the caller where they can - * locate the requested object. We better make - * sure the packfile is still here and can be - * accessed before supplying that answer, as - * it may have been deleted since the index - * was loaded! - */ - if (p->pack_fd == -1 && open_packed_git(p)) { - error("packfile %s cannot be accessed", p->pack_name); - goto next; - } - e->offset = offset; - e->p = p; - hashcpy(e->sha1, sha1); - last_found = p; - return 1; - } - - next: - if (p == last_found) - p = packed_git; - else - p = p->next; - if (p == last_found) - p = p->next; - } while (p); - return 0; -} - -struct packed_git *find_sha1_pack(const unsigned char *sha1, - struct packed_git *packs) -{ - struct packed_git *p; - - for (p = packs; p; p = p->next) { - if (find_pack_entry_one(sha1, p)) - return p; - } - return NULL; - -} - -static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep) -{ - int status; - unsigned long mapsize, size; - void *map; - z_stream stream; - char hdr[32]; - - map = map_sha1_file(sha1, &mapsize); - if (!map) - return error("unable to find %s", sha1_to_hex(sha1)); - if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) - status = error("unable to unpack %s header", - sha1_to_hex(sha1)); - else if ((status = parse_sha1_header(hdr, &size)) < 0) - status = error("unable to parse %s header", sha1_to_hex(sha1)); - else if (sizep) - *sizep = size; - inflateEnd(&stream); - munmap(map, mapsize); - return status; -} - -int sha1_object_info(const unsigned char *sha1, unsigned long *sizep) -{ - struct pack_entry e; - int status; - - if (!find_pack_entry(sha1, &e, NULL)) { - /* Most likely it's a loose object. */ - status = sha1_loose_object_info(sha1, sizep); - if (status >= 0) - return status; - - /* Not a loose object; someone else may have just packed it. */ - reprepare_packed_git(); - if (!find_pack_entry(sha1, &e, NULL)) - return status; - } - return packed_object_info(e.p, e.offset, sizep); -} - -static void *read_packed_sha1(const unsigned char *sha1, - enum object_type *type, unsigned long *size) -{ - struct pack_entry e; - void *data; - - if (!find_pack_entry(sha1, &e, NULL)) - return NULL; - data = cache_or_unpack_entry(e.p, e.offset, size, type, 1); - if (!data) { - /* - * We're probably in deep shit, but let's try to fetch - * the required object anyway from another pack or loose. - * This should happen only in the presence of a corrupted - * pack, and is better than failing outright. - */ - error("failed to read object %s at offset %"PRIuMAX" from %s", - sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name); - mark_bad_packed_object(e.p, sha1); - data = read_object(sha1, type, size); - } - return data; -} - -/* - * This is meant to hold a *small* number of objects that you would - * want read_sha1_file() to be able to return, but yet you do not want - * to write them into the object store (e.g. a browse-only - * application). - */ -static struct cached_object { - unsigned char sha1[20]; - enum object_type type; - void *buf; - unsigned long size; -} *cached_objects; -static int cached_object_nr, cached_object_alloc; - -static struct cached_object empty_tree = { - /* empty tree sha1: 4b825dc642cb6eb9a060e54bf8d69288fbee4904 */ - "\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60" - "\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04", - OBJ_TREE, - "", - 0 -}; - -static struct cached_object *find_cached_object(const unsigned char *sha1) -{ - int i; - struct cached_object *co = cached_objects; - - for (i = 0; i < cached_object_nr; i++, co++) { - if (!hashcmp(co->sha1, sha1)) - return co; - } - if (!hashcmp(sha1, empty_tree.sha1)) - return &empty_tree; - return NULL; -} - -int pretend_sha1_file(void *buf, unsigned long len, enum object_type type, - unsigned char *sha1) -{ - struct cached_object *co; - - hash_sha1_file(buf, len, typename(type), sha1); - if (has_sha1_file(sha1) || find_cached_object(sha1)) - return 0; - if (cached_object_alloc <= cached_object_nr) { - cached_object_alloc = alloc_nr(cached_object_alloc); - cached_objects = xrealloc(cached_objects, - sizeof(*cached_objects) * - cached_object_alloc); - } - co = &cached_objects[cached_object_nr++]; - co->size = len; - co->type = type; - co->buf = xmalloc(len); - memcpy(co->buf, buf, len); - hashcpy(co->sha1, sha1); - return 0; -} - -void *read_object(const unsigned char *sha1, enum object_type *type, - unsigned long *size) -{ - unsigned long mapsize; - void *map, *buf; - struct cached_object *co; - - co = find_cached_object(sha1); - if (co) { - *type = co->type; - *size = co->size; - return xmemdupz(co->buf, co->size); - } - - buf = read_packed_sha1(sha1, type, size); - if (buf) - return buf; - map = map_sha1_file(sha1, &mapsize); - if (map) { - buf = unpack_sha1_file(map, mapsize, type, size, sha1); - munmap(map, mapsize); - return buf; - } - reprepare_packed_git(); - return read_packed_sha1(sha1, type, size); -} - -void *read_sha1_file(const unsigned char *sha1, enum object_type *type, - unsigned long *size) -{ - void *data = read_object(sha1, type, size); - /* legacy behavior is to die on corrupted objects */ - if (!data && (has_loose_object(sha1) || has_packed_and_bad(sha1))) - die("object %s is corrupted", sha1_to_hex(sha1)); - return data; -} - -void *read_object_with_reference(const unsigned char *sha1, - const char *required_type_name, - unsigned long *size, - unsigned char *actual_sha1_return) -{ - enum object_type type, required_type; - void *buffer; - unsigned long isize; - unsigned char actual_sha1[20]; - - required_type = type_from_string(required_type_name); - hashcpy(actual_sha1, sha1); - while (1) { - int ref_length = -1; - const char *ref_type = NULL; - - buffer = read_sha1_file(actual_sha1, &type, &isize); - if (!buffer) - return NULL; - if (type == required_type) { - *size = isize; - if (actual_sha1_return) - hashcpy(actual_sha1_return, actual_sha1); - return buffer; - } - /* Handle references */ - else if (type == OBJ_COMMIT) - ref_type = "tree "; - else if (type == OBJ_TAG) - ref_type = "object "; - else { - free(buffer); - return NULL; - } - ref_length = strlen(ref_type); - - if (ref_length + 40 > isize || - memcmp(buffer, ref_type, ref_length) || - get_sha1_hex((char *) buffer + ref_length, actual_sha1)) { - free(buffer); - return NULL; - } - free(buffer); - /* Now we have the ID of the referred-to object in - * actual_sha1. Check again. */ - } -} - -static void write_sha1_file_prepare(const void *buf, unsigned long len, - const char *type, unsigned char *sha1, - char *hdr, int *hdrlen) -{ - SHA_CTX c; - - /* Generate the header */ - *hdrlen = sprintf(hdr, "%s %lu", type, len)+1; - - /* Sha1.. */ - SHA1_Init(&c); - SHA1_Update(&c, hdr, *hdrlen); - SHA1_Update(&c, buf, len); - SHA1_Final(sha1, &c); -} - -/* - * Move the just written object into its final resting place - */ -int move_temp_to_file(const char *tmpfile, const char *filename) -{ - int ret = link(tmpfile, filename); - - /* - * Coda hack - coda doesn't like cross-directory links, - * so we fall back to a rename, which will mean that it - * won't be able to check collisions, but that's not a - * big deal. - * - * The same holds for FAT formatted media. - * - * When this succeeds, we just return 0. We have nothing - * left to unlink. - */ - if (ret && ret != EEXIST) { - if (!rename(tmpfile, filename)) - return 0; - ret = errno; - } - unlink(tmpfile); - if (ret) { - if (ret != EEXIST) { - return error("unable to write sha1 filename %s: %s\n", filename, strerror(ret)); - } - /* FIXME!!! Collision check here ? */ - } - - return 0; -} - -static int write_buffer(int fd, const void *buf, size_t len) -{ - if (write_in_full(fd, buf, len) < 0) - return error("file write error (%s)", strerror(errno)); - return 0; -} - -int hash_sha1_file(const void *buf, unsigned long len, const char *type, - unsigned char *sha1) -{ - char hdr[32]; - int hdrlen; - write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); - return 0; -} - -/* Finalize a file on disk, and close it. */ -static void close_sha1_file(int fd) -{ - if (fsync_object_files) - fsync_or_die(fd, "sha1 file"); - fchmod(fd, 0444); - if (close(fd) != 0) - die("unable to write sha1 file"); -} - -/* Size of directory component, including the ending '/' */ -static inline int directory_size(const char *filename) -{ - const char *s = strrchr(filename, '/'); - if (!s) - return 0; - return s - filename + 1; -} - -/* - * This creates a temporary file in the same directory as the final - * 'filename' - * - * We want to avoid cross-directory filename renames, because those - * can have problems on various filesystems (FAT, NFS, Coda). - */ -static int create_tmpfile(char *buffer, size_t bufsiz, const char *filename) -{ - int fd, dirlen = directory_size(filename); - - if (dirlen + 20 > bufsiz) { - errno = ENAMETOOLONG; - return -1; - } - memcpy(buffer, filename, dirlen); - strcpy(buffer + dirlen, "tmp_obj_XXXXXX"); - fd = mkstemp(buffer); - if (fd < 0 && dirlen) { - /* Make sure the directory exists */ - memcpy(buffer, filename, dirlen); - buffer[dirlen-1] = 0; - if (mkdir(buffer, 0777) || adjust_shared_perm(buffer)) - return -1; - - /* Try again */ - strcpy(buffer + dirlen - 1, "/tmp_obj_XXXXXX"); - fd = mkstemp(buffer); - } - return fd; -} - -static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, - void *buf, unsigned long len, time_t mtime) -{ - int fd, size, ret; - unsigned char *compressed; - z_stream stream; - char *filename; - static char tmpfile[PATH_MAX]; - - filename = sha1_file_name(sha1); - fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename); - if (fd < 0) { - if (errno == EPERM) - return error("insufficient permission for adding an object to repository database %s\n", get_object_directory()); - else - return error("unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno)); - } - - /* Set it up */ - memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); - size = 8 + deflateBound(&stream, len+hdrlen); - compressed = xmalloc(size); - - /* Compress it */ - stream.next_out = compressed; - stream.avail_out = size; - - /* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; - while (deflate(&stream, 0) == Z_OK) - /* nothing */; - - /* Then the data itself.. */ - stream.next_in = buf; - stream.avail_in = len; - ret = deflate(&stream, Z_FINISH); - if (ret != Z_STREAM_END) - die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret); - - ret = deflateEnd(&stream); - if (ret != Z_OK) - die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret); - - size = stream.total_out; - - if (write_buffer(fd, compressed, size) < 0) - die("unable to write sha1 file"); - close_sha1_file(fd); - free(compressed); - - if (mtime) { - struct utimbuf utb; - utb.actime = mtime; - utb.modtime = mtime; - if (utime(tmpfile, &utb) < 0) - warning("failed utime() on %s: %s", - tmpfile, strerror(errno)); - } - - return move_temp_to_file(tmpfile, filename); -} - -int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1) -{ - unsigned char sha1[20]; - char hdr[32]; - int hdrlen; - - /* Normally if we have it in the pack then we do not bother writing - * it out into .git/objects/??/?{38} file. - */ - write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); - if (returnsha1) - hashcpy(returnsha1, sha1); - if (has_sha1_file(sha1)) - return 0; - return write_loose_object(sha1, hdr, hdrlen, buf, len, 0); -} - -int force_object_loose(const unsigned char *sha1, time_t mtime) -{ - void *buf; - unsigned long len; - enum object_type type; - char hdr[32]; - int hdrlen; - - if (has_loose_object(sha1)) - return 0; - buf = read_packed_sha1(sha1, &type, &len); - if (!buf) - return error("cannot read sha1_file for %s", sha1_to_hex(sha1)); - hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1; - return write_loose_object(sha1, hdr, hdrlen, buf, len, mtime); -} - -int has_pack_index(const unsigned char *sha1) -{ - struct stat st; - if (stat(sha1_pack_index_name(sha1), &st)) - return 0; - return 1; -} - -int has_pack_file(const unsigned char *sha1) -{ - struct stat st; - if (stat(sha1_pack_name(sha1), &st)) - return 0; - return 1; -} - -int has_sha1_pack(const unsigned char *sha1, const char **ignore_packed) -{ - struct pack_entry e; - return find_pack_entry(sha1, &e, ignore_packed); -} - -int has_sha1_file(const unsigned char *sha1) -{ - struct pack_entry e; - - if (find_pack_entry(sha1, &e, NULL)) - return 1; - return has_loose_object(sha1); -} - -int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object) -{ - struct strbuf buf; - int ret; - - strbuf_init(&buf, 0); - if (strbuf_read(&buf, fd, 4096) < 0) { - strbuf_release(&buf); - return -1; - } - - if (!type) - type = blob_type; - if (write_object) - ret = write_sha1_file(buf.buf, buf.len, type, sha1); - else - ret = hash_sha1_file(buf.buf, buf.len, type, sha1); - strbuf_release(&buf); - - return ret; -} - -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, - enum object_type type, const char *path) -{ - size_t size = xsize_t(st->st_size); - void *buf = NULL; - int ret, re_allocated = 0; - - if (size) - buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - - if (!type) - type = OBJ_BLOB; - - /* - * Convert blobs to git internal format - */ - if ((type == OBJ_BLOB) && S_ISREG(st->st_mode)) { - struct strbuf nbuf; - strbuf_init(&nbuf, 0); - if (convert_to_git(path, buf, size, &nbuf, - write_object ? safe_crlf : 0)) { - munmap(buf, size); - buf = strbuf_detach(&nbuf, &size); - re_allocated = 1; - } - } - - if (write_object) - ret = write_sha1_file(buf, size, typename(type), sha1); - else - ret = hash_sha1_file(buf, size, typename(type), sha1); - if (re_allocated) { - free(buf); - return ret; - } - if (size) - munmap(buf, size); - return ret; -} - -int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object) -{ - int fd; - char *target; - size_t len; - - switch (st->st_mode & S_IFMT) { - case S_IFREG: - fd = open(path, O_RDONLY); - if (fd < 0) - return error("open(\"%s\"): %s", path, - strerror(errno)); - if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path) < 0) - return error("%s: failed to insert into database", - path); - break; - case S_IFLNK: - len = xsize_t(st->st_size); - target = xmalloc(len + 1); - if (readlink(path, target, len + 1) != st->st_size) { - char *errstr = strerror(errno); - free(target); - return error("readlink(\"%s\"): %s", path, - errstr); - } - if (!write_object) - hash_sha1_file(target, len, blob_type, sha1); - else if (write_sha1_file(target, len, blob_type, sha1)) - return error("%s: failed to insert into database", - path); - free(target); - break; - case S_IFDIR: - return resolve_gitlink_ref(path, "HEAD", sha1); - default: - return error("%s: unsupported file type", path); - } - return 0; -} - -int read_pack_header(int fd, struct pack_header *header) -{ - if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header)) - /* "eof before pack header was fully read" */ - return PH_ERROR_EOF; - - if (header->hdr_signature != htonl(PACK_SIGNATURE)) - /* "protocol error (pack signature mismatch detected)" */ - return PH_ERROR_PACK_SIGNATURE; - if (!pack_version_ok(header->hdr_version)) - /* "protocol error (pack version unsupported)" */ - return PH_ERROR_PROTOCOL; - return 0; -} + +#include "cache.h" + +static unsigned int pack_open_windows; +static size_t pack_mapped; +struct packed_git *packed_git; + +static void scan_windows(struct packed_git *p, + struct packed_git **lru_p, + struct pack_window **lru_w, + struct pack_window **lru_l) +{ + struct pack_window *w, *w_l; + + for (w_l = NULL, w = p->windows; w; w = w->next) { + if (!w->inuse_cnt) { + if (!*lru_w || w->last_used < (*lru_w)->last_used) { + *lru_p = p; + *lru_w = w; + *lru_l = w_l; + } + } + w_l = w; + } +} + +static int unuse_one_window(struct packed_git *current, int keep_fd) +{ + struct packed_git *p, *lru_p = NULL; + struct pack_window *lru_w = NULL, *lru_l = NULL; + + if (current) + scan_windows(current, &lru_p, &lru_w, &lru_l); + for (p = packed_git; p; p = p->next) + scan_windows(p, &lru_p, &lru_w, &lru_l); + if (lru_p) { + munmap(lru_w->base, lru_w->len); + pack_mapped -= lru_w->len; + if (lru_l) + lru_l->next = lru_w->next; + else { + lru_p->windows = lru_w->next; + if (!lru_p->windows && lru_p->pack_fd != keep_fd) { + close(lru_p->pack_fd); + lru_p->pack_fd = -1; + } + } + free(lru_w); + pack_open_windows--; + return 1; + } + return 0; +} + +void release_pack_memory(size_t need, int fd) +{ + size_t cur = pack_mapped; + while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd)) + ; /* nothing */ +} diff --git a/strbuf.c b/strbuf.c index 720737d..c2563cc 100644 --- a/strbuf.c +++ b/strbuf.c @@ -92,7 +92,7 @@ void strbuf_ltrim(struct strbuf *sb) void strbuf_tolower(struct strbuf *sb) { - int i; + size_t i; for (i = 0; i < sb->len; i++) sb->buf[i] = tolower(sb->buf[i]); } @@ -204,12 +204,12 @@ void strbuf_addf(struct strbuf *sb, const char *fmt, ...) va_end(ap); if (len < 0) die("your vsnprintf is broken"); - if (len > strbuf_avail(sb)) { + if ((size_t)len > strbuf_avail(sb)) { strbuf_grow(sb, len); va_start(ap, fmt); len = vsnprintf(sb->buf + sb->len, sb->alloc - sb->len, fmt, ap); va_end(ap); - if (len > strbuf_avail(sb)) { + if ((size_t)len > strbuf_avail(sb)) { die("this should not happen, your snprintf is broken"); } } diff --git a/usage.c b/usage.c index a5fc4ec..500941d 100644 --- a/usage.c +++ b/usage.c @@ -36,17 +36,17 @@ static void warn_builtin(const char *warn, va_list params) /* If we are in a dlopen()ed .so write to a global variable would segfault * (ugh), so keep things static. */ -static void (*usage_routine)(const char *err) NORETURN = usage_builtin; -static void (*die_routine)(const char *err, va_list params) NORETURN = die_builtin; +static void (*usage_routine)(const char *err) __attribute__((__noreturn__)) = usage_builtin; +static void (*die_routine)(const char *err, va_list params) __attribute__((__noreturn__)) = die_builtin; static void (*error_routine)(const char *err, va_list params) = error_builtin; static void (*warn_routine)(const char *err, va_list params) = warn_builtin; -void set_usage_routine(void (*routine)(const char *err) NORETURN) +void set_usage_routine(void (*routine)(const char *err) __attribute__((__noreturn__))) { usage_routine = routine; } -void set_die_routine(void (*routine)(const char *err, va_list params) NORETURN) +void set_die_routine(void (*routine)(const char *err, va_list params) __attribute__((__noreturn__))) { die_routine = routine; } @@ -67,7 +67,7 @@ void usage(const char *err) usage_routine(err); } -void die(const char *err, ...) +NORETURN void die(const char *err, ...) { va_list params; -- 2.11.4.GIT