4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 ******************************************************************************
13 ** This file implements a VFS shim that allows an SQLite database to be
14 ** appended onto the end of some other file, such as an executable.
16 ** A special record must appear at the end of the file that identifies the
17 ** file as an appended database and provides an offset to page 1. For
18 ** best performance page 1 should be located at a disk page boundary, though
19 ** that is not required.
21 ** When opening a database using this VFS, the connection might treat
22 ** the file as an ordinary SQLite database, or it might treat is as a
23 ** database appended onto some other file. Here are the rules:
25 ** (1) When opening a new empty file, that file is treated as an ordinary
28 ** (2) When opening a file that begins with the standard SQLite prefix
29 ** string "SQLite format 3", that file is treated as an ordinary
32 ** (3) When opening a file that ends with the appendvfs trailer string
33 ** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
36 ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is
37 ** set, then a new database is appended to the already existing file.
39 ** (5) Otherwise, SQLITE_CANTOPEN is returned.
41 ** To avoid unnecessary complications with the PENDING_BYTE, the size of
42 ** the file containing the database is limited to 1GB. This VFS will refuse
43 ** to read or write past the 1GB mark. This restriction might be lifted in
44 ** future versions. For now, if you need a large database, then keep the
45 ** database in a separate file.
47 ** If the file being opened is not an appended database, then this shim is
48 ** a pass-through into the default underlying VFS.
50 #include "sqlite3ext.h"
51 SQLITE_EXTENSION_INIT1
55 /* The append mark at the end of the database is:
57 ** Start-Of-SQLite3-NNNNNNNN
58 ** 123456789 123456789 12345
60 ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
61 ** the offset to page 1.
63 #define APND_MARK_PREFIX "Start-Of-SQLite3-"
64 #define APND_MARK_PREFIX_SZ 17
65 #define APND_MARK_SIZE 25
68 ** Maximum size of the combined prefix + database + append-mark. This
69 ** must be less than 0x40000000 to avoid locking issues on Windows.
71 #define APND_MAX_SIZE (65536*15259)
74 ** Forward declaration of objects used by this utility
76 typedef struct sqlite3_vfs ApndVfs
;
77 typedef struct ApndFile ApndFile
;
79 /* Access to a lower-level VFS that (might) implement dynamic loading,
80 ** access to randomness, etc.
82 #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
83 #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
87 sqlite3_file base
; /* IO methods */
88 sqlite3_int64 iPgOne
; /* File offset to page 1 */
89 sqlite3_int64 iMark
; /* Start of the append-mark */
93 ** Methods for ApndFile
95 static int apndClose(sqlite3_file
*);
96 static int apndRead(sqlite3_file
*, void*, int iAmt
, sqlite3_int64 iOfst
);
97 static int apndWrite(sqlite3_file
*,const void*,int iAmt
, sqlite3_int64 iOfst
);
98 static int apndTruncate(sqlite3_file
*, sqlite3_int64 size
);
99 static int apndSync(sqlite3_file
*, int flags
);
100 static int apndFileSize(sqlite3_file
*, sqlite3_int64
*pSize
);
101 static int apndLock(sqlite3_file
*, int);
102 static int apndUnlock(sqlite3_file
*, int);
103 static int apndCheckReservedLock(sqlite3_file
*, int *pResOut
);
104 static int apndFileControl(sqlite3_file
*, int op
, void *pArg
);
105 static int apndSectorSize(sqlite3_file
*);
106 static int apndDeviceCharacteristics(sqlite3_file
*);
107 static int apndShmMap(sqlite3_file
*, int iPg
, int pgsz
, int, void volatile**);
108 static int apndShmLock(sqlite3_file
*, int offset
, int n
, int flags
);
109 static void apndShmBarrier(sqlite3_file
*);
110 static int apndShmUnmap(sqlite3_file
*, int deleteFlag
);
111 static int apndFetch(sqlite3_file
*, sqlite3_int64 iOfst
, int iAmt
, void **pp
);
112 static int apndUnfetch(sqlite3_file
*, sqlite3_int64 iOfst
, void *p
);
115 ** Methods for ApndVfs
117 static int apndOpen(sqlite3_vfs
*, const char *, sqlite3_file
*, int , int *);
118 static int apndDelete(sqlite3_vfs
*, const char *zName
, int syncDir
);
119 static int apndAccess(sqlite3_vfs
*, const char *zName
, int flags
, int *);
120 static int apndFullPathname(sqlite3_vfs
*, const char *zName
, int, char *zOut
);
121 static void *apndDlOpen(sqlite3_vfs
*, const char *zFilename
);
122 static void apndDlError(sqlite3_vfs
*, int nByte
, char *zErrMsg
);
123 static void (*apndDlSym(sqlite3_vfs
*pVfs
, void *p
, const char*zSym
))(void);
124 static void apndDlClose(sqlite3_vfs
*, void*);
125 static int apndRandomness(sqlite3_vfs
*, int nByte
, char *zOut
);
126 static int apndSleep(sqlite3_vfs
*, int microseconds
);
127 static int apndCurrentTime(sqlite3_vfs
*, double*);
128 static int apndGetLastError(sqlite3_vfs
*, int, char *);
129 static int apndCurrentTimeInt64(sqlite3_vfs
*, sqlite3_int64
*);
130 static int apndSetSystemCall(sqlite3_vfs
*, const char*,sqlite3_syscall_ptr
);
131 static sqlite3_syscall_ptr
apndGetSystemCall(sqlite3_vfs
*, const char *z
);
132 static const char *apndNextSystemCall(sqlite3_vfs
*, const char *zName
);
134 static sqlite3_vfs apnd_vfs
= {
135 3, /* iVersion (set when registered) */
136 0, /* szOsFile (set when registered) */
137 1024, /* mxPathname */
139 "apndvfs", /* zName */
140 0, /* pAppData (set when registered) */
141 apndOpen
, /* xOpen */
142 apndDelete
, /* xDelete */
143 apndAccess
, /* xAccess */
144 apndFullPathname
, /* xFullPathname */
145 apndDlOpen
, /* xDlOpen */
146 apndDlError
, /* xDlError */
147 apndDlSym
, /* xDlSym */
148 apndDlClose
, /* xDlClose */
149 apndRandomness
, /* xRandomness */
150 apndSleep
, /* xSleep */
151 apndCurrentTime
, /* xCurrentTime */
152 apndGetLastError
, /* xGetLastError */
153 apndCurrentTimeInt64
, /* xCurrentTimeInt64 */
154 apndSetSystemCall
, /* xSetSystemCall */
155 apndGetSystemCall
, /* xGetSystemCall */
156 apndNextSystemCall
/* xNextSystemCall */
159 static const sqlite3_io_methods apnd_io_methods
= {
161 apndClose
, /* xClose */
162 apndRead
, /* xRead */
163 apndWrite
, /* xWrite */
164 apndTruncate
, /* xTruncate */
165 apndSync
, /* xSync */
166 apndFileSize
, /* xFileSize */
167 apndLock
, /* xLock */
168 apndUnlock
, /* xUnlock */
169 apndCheckReservedLock
, /* xCheckReservedLock */
170 apndFileControl
, /* xFileControl */
171 apndSectorSize
, /* xSectorSize */
172 apndDeviceCharacteristics
, /* xDeviceCharacteristics */
173 apndShmMap
, /* xShmMap */
174 apndShmLock
, /* xShmLock */
175 apndShmBarrier
, /* xShmBarrier */
176 apndShmUnmap
, /* xShmUnmap */
177 apndFetch
, /* xFetch */
178 apndUnfetch
/* xUnfetch */
184 ** Close an apnd-file.
186 static int apndClose(sqlite3_file
*pFile
){
187 pFile
= ORIGFILE(pFile
);
188 return pFile
->pMethods
->xClose(pFile
);
192 ** Read data from an apnd-file.
200 ApndFile
*p
= (ApndFile
*)pFile
;
201 pFile
= ORIGFILE(pFile
);
202 return pFile
->pMethods
->xRead(pFile
, zBuf
, iAmt
, iOfst
+p
->iPgOne
);
206 ** Add the append-mark onto the end of the file.
208 static int apndWriteMark(ApndFile
*p
, sqlite3_file
*pFile
){
210 unsigned char a
[APND_MARK_SIZE
];
211 memcpy(a
, APND_MARK_PREFIX
, APND_MARK_PREFIX_SZ
);
213 a
[APND_MARK_PREFIX_SZ
+i
] = (p
->iPgOne
>> (56 - i
*8)) & 0xff;
215 return pFile
->pMethods
->xWrite(pFile
, a
, APND_MARK_SIZE
, p
->iMark
);
219 ** Write data to an apnd-file.
221 static int apndWrite(
228 ApndFile
*p
= (ApndFile
*)pFile
;
229 pFile
= ORIGFILE(pFile
);
230 if( iOfst
+iAmt
>=APND_MAX_SIZE
) return SQLITE_FULL
;
231 rc
= pFile
->pMethods
->xWrite(pFile
, zBuf
, iAmt
, iOfst
+p
->iPgOne
);
232 if( rc
==SQLITE_OK
&& iOfst
+ iAmt
+ p
->iPgOne
> p
->iMark
){
233 sqlite3_int64 sz
= 0;
234 rc
= pFile
->pMethods
->xFileSize(pFile
, &sz
);
236 p
->iMark
= sz
- APND_MARK_SIZE
;
237 if( iOfst
+ iAmt
+ p
->iPgOne
> p
->iMark
){
238 p
->iMark
= p
->iPgOne
+ iOfst
+ iAmt
;
239 rc
= apndWriteMark(p
, pFile
);
247 ** Truncate an apnd-file.
249 static int apndTruncate(sqlite3_file
*pFile
, sqlite_int64 size
){
251 ApndFile
*p
= (ApndFile
*)pFile
;
252 pFile
= ORIGFILE(pFile
);
253 rc
= pFile
->pMethods
->xTruncate(pFile
, size
+p
->iPgOne
+APND_MARK_SIZE
);
255 p
->iMark
= p
->iPgOne
+size
;
256 rc
= apndWriteMark(p
, pFile
);
262 ** Sync an apnd-file.
264 static int apndSync(sqlite3_file
*pFile
, int flags
){
265 pFile
= ORIGFILE(pFile
);
266 return pFile
->pMethods
->xSync(pFile
, flags
);
270 ** Return the current file-size of an apnd-file.
272 static int apndFileSize(sqlite3_file
*pFile
, sqlite_int64
*pSize
){
273 ApndFile
*p
= (ApndFile
*)pFile
;
276 rc
= pFile
->pMethods
->xFileSize(pFile
, pSize
);
277 if( rc
==SQLITE_OK
&& p
->iPgOne
){
278 *pSize
-= p
->iPgOne
+ APND_MARK_SIZE
;
284 ** Lock an apnd-file.
286 static int apndLock(sqlite3_file
*pFile
, int eLock
){
287 pFile
= ORIGFILE(pFile
);
288 return pFile
->pMethods
->xLock(pFile
, eLock
);
292 ** Unlock an apnd-file.
294 static int apndUnlock(sqlite3_file
*pFile
, int eLock
){
295 pFile
= ORIGFILE(pFile
);
296 return pFile
->pMethods
->xUnlock(pFile
, eLock
);
300 ** Check if another file-handle holds a RESERVED lock on an apnd-file.
302 static int apndCheckReservedLock(sqlite3_file
*pFile
, int *pResOut
){
303 pFile
= ORIGFILE(pFile
);
304 return pFile
->pMethods
->xCheckReservedLock(pFile
, pResOut
);
308 ** File control method. For custom operations on an apnd-file.
310 static int apndFileControl(sqlite3_file
*pFile
, int op
, void *pArg
){
311 ApndFile
*p
= (ApndFile
*)pFile
;
313 pFile
= ORIGFILE(pFile
);
314 rc
= pFile
->pMethods
->xFileControl(pFile
, op
, pArg
);
315 if( rc
==SQLITE_OK
&& op
==SQLITE_FCNTL_VFSNAME
){
316 *(char**)pArg
= sqlite3_mprintf("apnd(%lld)/%z", p
->iPgOne
, *(char**)pArg
);
322 ** Return the sector-size in bytes for an apnd-file.
324 static int apndSectorSize(sqlite3_file
*pFile
){
325 pFile
= ORIGFILE(pFile
);
326 return pFile
->pMethods
->xSectorSize(pFile
);
330 ** Return the device characteristic flags supported by an apnd-file.
332 static int apndDeviceCharacteristics(sqlite3_file
*pFile
){
333 pFile
= ORIGFILE(pFile
);
334 return pFile
->pMethods
->xDeviceCharacteristics(pFile
);
337 /* Create a shared memory file mapping */
338 static int apndShmMap(
345 pFile
= ORIGFILE(pFile
);
346 return pFile
->pMethods
->xShmMap(pFile
,iPg
,pgsz
,bExtend
,pp
);
349 /* Perform locking on a shared-memory segment */
350 static int apndShmLock(sqlite3_file
*pFile
, int offset
, int n
, int flags
){
351 pFile
= ORIGFILE(pFile
);
352 return pFile
->pMethods
->xShmLock(pFile
,offset
,n
,flags
);
355 /* Memory barrier operation on shared memory */
356 static void apndShmBarrier(sqlite3_file
*pFile
){
357 pFile
= ORIGFILE(pFile
);
358 pFile
->pMethods
->xShmBarrier(pFile
);
361 /* Unmap a shared memory segment */
362 static int apndShmUnmap(sqlite3_file
*pFile
, int deleteFlag
){
363 pFile
= ORIGFILE(pFile
);
364 return pFile
->pMethods
->xShmUnmap(pFile
,deleteFlag
);
367 /* Fetch a page of a memory-mapped file */
368 static int apndFetch(
374 ApndFile
*p
= (ApndFile
*)pFile
;
375 pFile
= ORIGFILE(pFile
);
376 return pFile
->pMethods
->xFetch(pFile
, iOfst
+p
->iPgOne
, iAmt
, pp
);
379 /* Release a memory-mapped page */
380 static int apndUnfetch(sqlite3_file
*pFile
, sqlite3_int64 iOfst
, void *pPage
){
381 ApndFile
*p
= (ApndFile
*)pFile
;
382 pFile
= ORIGFILE(pFile
);
383 return pFile
->pMethods
->xUnfetch(pFile
, iOfst
+p
->iPgOne
, pPage
);
387 ** Check to see if the file is an ordinary SQLite database file.
389 static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz
, sqlite3_file
*pFile
){
392 static const char aSqliteHdr
[] = "SQLite format 3";
393 if( sz
<512 ) return 0;
394 rc
= pFile
->pMethods
->xRead(pFile
, zHdr
, sizeof(zHdr
), 0);
396 return memcmp(zHdr
, aSqliteHdr
, sizeof(zHdr
))==0;
400 ** Try to read the append-mark off the end of a file. Return the
401 ** start of the appended database if the append-mark is present. If
402 ** there is no append-mark, return -1;
404 static sqlite3_int64
apndReadMark(sqlite3_int64 sz
, sqlite3_file
*pFile
){
407 unsigned char a
[APND_MARK_SIZE
];
409 if( sz
<=APND_MARK_SIZE
) return -1;
410 rc
= pFile
->pMethods
->xRead(pFile
, a
, APND_MARK_SIZE
, sz
-APND_MARK_SIZE
);
412 if( memcmp(a
, APND_MARK_PREFIX
, APND_MARK_PREFIX_SZ
)!=0 ) return -1;
413 iMark
= ((sqlite3_int64
)(a
[APND_MARK_PREFIX_SZ
]&0x7f))<<56;
415 iMark
+= (sqlite3_int64
)a
[APND_MARK_PREFIX_SZ
+i
]<<(56-8*i
);
421 ** Open an apnd file handle.
431 sqlite3_file
*pSubFile
;
432 sqlite3_vfs
*pSubVfs
;
435 pSubVfs
= ORIGVFS(pVfs
);
436 if( (flags
& SQLITE_OPEN_MAIN_DB
)==0 ){
437 return pSubVfs
->xOpen(pSubVfs
, zName
, pFile
, flags
, pOutFlags
);
439 p
= (ApndFile
*)pFile
;
440 memset(p
, 0, sizeof(*p
));
441 pSubFile
= ORIGFILE(pFile
);
442 p
->base
.pMethods
= &apnd_io_methods
;
443 rc
= pSubVfs
->xOpen(pSubVfs
, zName
, pSubFile
, flags
, pOutFlags
);
444 if( rc
) goto apnd_open_done
;
445 rc
= pSubFile
->pMethods
->xFileSize(pSubFile
, &sz
);
447 pSubFile
->pMethods
->xClose(pSubFile
);
450 if( apndIsOrdinaryDatabaseFile(sz
, pSubFile
) ){
451 memmove(pFile
, pSubFile
, pSubVfs
->szOsFile
);
455 p
->iPgOne
= apndReadMark(sz
, pFile
);
459 if( (flags
& SQLITE_OPEN_CREATE
)==0 ){
460 pSubFile
->pMethods
->xClose(pSubFile
);
461 rc
= SQLITE_CANTOPEN
;
463 p
->iPgOne
= (sz
+0xfff) & ~(sqlite3_int64
)0xfff;
465 if( rc
) pFile
->pMethods
= 0;
470 ** All other VFS methods are pass-thrus.
472 static int apndDelete(sqlite3_vfs
*pVfs
, const char *zPath
, int dirSync
){
473 return ORIGVFS(pVfs
)->xDelete(ORIGVFS(pVfs
), zPath
, dirSync
);
475 static int apndAccess(
481 return ORIGVFS(pVfs
)->xAccess(ORIGVFS(pVfs
), zPath
, flags
, pResOut
);
483 static int apndFullPathname(
489 return ORIGVFS(pVfs
)->xFullPathname(ORIGVFS(pVfs
),zPath
,nOut
,zOut
);
491 static void *apndDlOpen(sqlite3_vfs
*pVfs
, const char *zPath
){
492 return ORIGVFS(pVfs
)->xDlOpen(ORIGVFS(pVfs
), zPath
);
494 static void apndDlError(sqlite3_vfs
*pVfs
, int nByte
, char *zErrMsg
){
495 ORIGVFS(pVfs
)->xDlError(ORIGVFS(pVfs
), nByte
, zErrMsg
);
497 static void (*apndDlSym(sqlite3_vfs
*pVfs
, void *p
, const char *zSym
))(void){
498 return ORIGVFS(pVfs
)->xDlSym(ORIGVFS(pVfs
), p
, zSym
);
500 static void apndDlClose(sqlite3_vfs
*pVfs
, void *pHandle
){
501 ORIGVFS(pVfs
)->xDlClose(ORIGVFS(pVfs
), pHandle
);
503 static int apndRandomness(sqlite3_vfs
*pVfs
, int nByte
, char *zBufOut
){
504 return ORIGVFS(pVfs
)->xRandomness(ORIGVFS(pVfs
), nByte
, zBufOut
);
506 static int apndSleep(sqlite3_vfs
*pVfs
, int nMicro
){
507 return ORIGVFS(pVfs
)->xSleep(ORIGVFS(pVfs
), nMicro
);
509 static int apndCurrentTime(sqlite3_vfs
*pVfs
, double *pTimeOut
){
510 return ORIGVFS(pVfs
)->xCurrentTime(ORIGVFS(pVfs
), pTimeOut
);
512 static int apndGetLastError(sqlite3_vfs
*pVfs
, int a
, char *b
){
513 return ORIGVFS(pVfs
)->xGetLastError(ORIGVFS(pVfs
), a
, b
);
515 static int apndCurrentTimeInt64(sqlite3_vfs
*pVfs
, sqlite3_int64
*p
){
516 return ORIGVFS(pVfs
)->xCurrentTimeInt64(ORIGVFS(pVfs
), p
);
518 static int apndSetSystemCall(
521 sqlite3_syscall_ptr pCall
523 return ORIGVFS(pVfs
)->xSetSystemCall(ORIGVFS(pVfs
),zName
,pCall
);
525 static sqlite3_syscall_ptr
apndGetSystemCall(
529 return ORIGVFS(pVfs
)->xGetSystemCall(ORIGVFS(pVfs
),zName
);
531 static const char *apndNextSystemCall(sqlite3_vfs
*pVfs
, const char *zName
){
532 return ORIGVFS(pVfs
)->xNextSystemCall(ORIGVFS(pVfs
), zName
);
537 __declspec(dllexport
)
540 ** This routine is called when the extension is loaded.
541 ** Register the new VFS.
543 int sqlite3_appendvfs_init(
546 const sqlite3_api_routines
*pApi
550 SQLITE_EXTENSION_INIT2(pApi
);
553 pOrig
= sqlite3_vfs_find(0);
554 apnd_vfs
.iVersion
= pOrig
->iVersion
;
555 apnd_vfs
.pAppData
= pOrig
;
556 apnd_vfs
.szOsFile
= pOrig
->szOsFile
+ sizeof(ApndFile
);
557 rc
= sqlite3_vfs_register(&apnd_vfs
, 0);
558 #ifdef APPENDVFS_TEST
560 rc
= sqlite3_auto_extension((void(*)(void))apndvfsRegister
);
563 if( rc
==SQLITE_OK
) rc
= SQLITE_OK_LOAD_PERMANENTLY
;