Enhance the command-line completion extension to return the names of
[sqlite.git] / ext / misc / appendvfs.c
blobb224245f3d9e9cf5ed1f549ae59980ce8883d54c
1 /*
2 ** 2017-10-20
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 ******************************************************************************
13 ** This file implements a VFS shim that allows an SQLite database to be
14 ** appended onto the end of some other file, such as an executable.
16 ** A special record must appear at the end of the file that identifies the
17 ** file as an appended database and provides an offset to page 1. For
18 ** best performance page 1 should be located at a disk page boundary, though
19 ** that is not required.
21 ** When opening a database using this VFS, the connection might treat
22 ** the file as an ordinary SQLite database, or it might treat is as a
23 ** database appended onto some other file. Here are the rules:
25 ** (1) When opening a new empty file, that file is treated as an ordinary
26 ** database.
28 ** (2) When opening a file that begins with the standard SQLite prefix
29 ** string "SQLite format 3", that file is treated as an ordinary
30 ** database.
32 ** (3) When opening a file that ends with the appendvfs trailer string
33 ** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
34 ** database.
36 ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is
37 ** set, then a new database is appended to the already existing file.
39 ** (5) Otherwise, SQLITE_CANTOPEN is returned.
41 ** To avoid unnecessary complications with the PENDING_BYTE, the size of
42 ** the file containing the database is limited to 1GB. This VFS will refuse
43 ** to read or write past the 1GB mark. This restriction might be lifted in
44 ** future versions. For now, if you need a large database, then keep the
45 ** database in a separate file.
47 ** If the file being opened is not an appended database, then this shim is
48 ** a pass-through into the default underlying VFS.
49 **/
50 #include "sqlite3ext.h"
51 SQLITE_EXTENSION_INIT1
52 #include <string.h>
53 #include <assert.h>
55 /* The append mark at the end of the database is:
57 ** Start-Of-SQLite3-NNNNNNNN
58 ** 123456789 123456789 12345
60 ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
61 ** the offset to page 1.
63 #define APND_MARK_PREFIX "Start-Of-SQLite3-"
64 #define APND_MARK_PREFIX_SZ 17
65 #define APND_MARK_SIZE 25
68 ** Maximum size of the combined prefix + database + append-mark. This
69 ** must be less than 0x40000000 to avoid locking issues on Windows.
71 #define APND_MAX_SIZE (65536*15259)
74 ** Forward declaration of objects used by this utility
76 typedef struct sqlite3_vfs ApndVfs;
77 typedef struct ApndFile ApndFile;
79 /* Access to a lower-level VFS that (might) implement dynamic loading,
80 ** access to randomness, etc.
82 #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
83 #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
85 /* An open file */
86 struct ApndFile {
87 sqlite3_file base; /* IO methods */
88 sqlite3_int64 iPgOne; /* File offset to page 1 */
89 sqlite3_int64 iMark; /* Start of the append-mark */
93 ** Methods for ApndFile
95 static int apndClose(sqlite3_file*);
96 static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
97 static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
98 static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
99 static int apndSync(sqlite3_file*, int flags);
100 static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
101 static int apndLock(sqlite3_file*, int);
102 static int apndUnlock(sqlite3_file*, int);
103 static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
104 static int apndFileControl(sqlite3_file*, int op, void *pArg);
105 static int apndSectorSize(sqlite3_file*);
106 static int apndDeviceCharacteristics(sqlite3_file*);
107 static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
108 static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
109 static void apndShmBarrier(sqlite3_file*);
110 static int apndShmUnmap(sqlite3_file*, int deleteFlag);
111 static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
112 static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
115 ** Methods for ApndVfs
117 static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
118 static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
119 static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
120 static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
121 static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
122 static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
123 static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
124 static void apndDlClose(sqlite3_vfs*, void*);
125 static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
126 static int apndSleep(sqlite3_vfs*, int microseconds);
127 static int apndCurrentTime(sqlite3_vfs*, double*);
128 static int apndGetLastError(sqlite3_vfs*, int, char *);
129 static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
130 static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
131 static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
132 static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
134 static sqlite3_vfs apnd_vfs = {
135 3, /* iVersion (set when registered) */
136 0, /* szOsFile (set when registered) */
137 1024, /* mxPathname */
138 0, /* pNext */
139 "apndvfs", /* zName */
140 0, /* pAppData (set when registered) */
141 apndOpen, /* xOpen */
142 apndDelete, /* xDelete */
143 apndAccess, /* xAccess */
144 apndFullPathname, /* xFullPathname */
145 apndDlOpen, /* xDlOpen */
146 apndDlError, /* xDlError */
147 apndDlSym, /* xDlSym */
148 apndDlClose, /* xDlClose */
149 apndRandomness, /* xRandomness */
150 apndSleep, /* xSleep */
151 apndCurrentTime, /* xCurrentTime */
152 apndGetLastError, /* xGetLastError */
153 apndCurrentTimeInt64, /* xCurrentTimeInt64 */
154 apndSetSystemCall, /* xSetSystemCall */
155 apndGetSystemCall, /* xGetSystemCall */
156 apndNextSystemCall /* xNextSystemCall */
159 static const sqlite3_io_methods apnd_io_methods = {
160 3, /* iVersion */
161 apndClose, /* xClose */
162 apndRead, /* xRead */
163 apndWrite, /* xWrite */
164 apndTruncate, /* xTruncate */
165 apndSync, /* xSync */
166 apndFileSize, /* xFileSize */
167 apndLock, /* xLock */
168 apndUnlock, /* xUnlock */
169 apndCheckReservedLock, /* xCheckReservedLock */
170 apndFileControl, /* xFileControl */
171 apndSectorSize, /* xSectorSize */
172 apndDeviceCharacteristics, /* xDeviceCharacteristics */
173 apndShmMap, /* xShmMap */
174 apndShmLock, /* xShmLock */
175 apndShmBarrier, /* xShmBarrier */
176 apndShmUnmap, /* xShmUnmap */
177 apndFetch, /* xFetch */
178 apndUnfetch /* xUnfetch */
184 ** Close an apnd-file.
186 static int apndClose(sqlite3_file *pFile){
187 pFile = ORIGFILE(pFile);
188 return pFile->pMethods->xClose(pFile);
192 ** Read data from an apnd-file.
194 static int apndRead(
195 sqlite3_file *pFile,
196 void *zBuf,
197 int iAmt,
198 sqlite_int64 iOfst
200 ApndFile *p = (ApndFile *)pFile;
201 pFile = ORIGFILE(pFile);
202 return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
206 ** Add the append-mark onto the end of the file.
208 static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
209 int i;
210 unsigned char a[APND_MARK_SIZE];
211 memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
212 for(i=0; i<8; i++){
213 a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
215 return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
219 ** Write data to an apnd-file.
221 static int apndWrite(
222 sqlite3_file *pFile,
223 const void *zBuf,
224 int iAmt,
225 sqlite_int64 iOfst
227 int rc;
228 ApndFile *p = (ApndFile *)pFile;
229 pFile = ORIGFILE(pFile);
230 if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
231 rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
232 if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){
233 sqlite3_int64 sz = 0;
234 rc = pFile->pMethods->xFileSize(pFile, &sz);
235 if( rc==SQLITE_OK ){
236 p->iMark = sz - APND_MARK_SIZE;
237 if( iOfst + iAmt + p->iPgOne > p->iMark ){
238 p->iMark = p->iPgOne + iOfst + iAmt;
239 rc = apndWriteMark(p, pFile);
243 return rc;
247 ** Truncate an apnd-file.
249 static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
250 int rc;
251 ApndFile *p = (ApndFile *)pFile;
252 pFile = ORIGFILE(pFile);
253 rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
254 if( rc==SQLITE_OK ){
255 p->iMark = p->iPgOne+size;
256 rc = apndWriteMark(p, pFile);
258 return rc;
262 ** Sync an apnd-file.
264 static int apndSync(sqlite3_file *pFile, int flags){
265 pFile = ORIGFILE(pFile);
266 return pFile->pMethods->xSync(pFile, flags);
270 ** Return the current file-size of an apnd-file.
272 static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
273 ApndFile *p = (ApndFile *)pFile;
274 int rc;
275 pFile = ORIGFILE(p);
276 rc = pFile->pMethods->xFileSize(pFile, pSize);
277 if( rc==SQLITE_OK && p->iPgOne ){
278 *pSize -= p->iPgOne + APND_MARK_SIZE;
280 return rc;
284 ** Lock an apnd-file.
286 static int apndLock(sqlite3_file *pFile, int eLock){
287 pFile = ORIGFILE(pFile);
288 return pFile->pMethods->xLock(pFile, eLock);
292 ** Unlock an apnd-file.
294 static int apndUnlock(sqlite3_file *pFile, int eLock){
295 pFile = ORIGFILE(pFile);
296 return pFile->pMethods->xUnlock(pFile, eLock);
300 ** Check if another file-handle holds a RESERVED lock on an apnd-file.
302 static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
303 pFile = ORIGFILE(pFile);
304 return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
308 ** File control method. For custom operations on an apnd-file.
310 static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
311 ApndFile *p = (ApndFile *)pFile;
312 int rc;
313 pFile = ORIGFILE(pFile);
314 rc = pFile->pMethods->xFileControl(pFile, op, pArg);
315 if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
316 *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
318 return rc;
322 ** Return the sector-size in bytes for an apnd-file.
324 static int apndSectorSize(sqlite3_file *pFile){
325 pFile = ORIGFILE(pFile);
326 return pFile->pMethods->xSectorSize(pFile);
330 ** Return the device characteristic flags supported by an apnd-file.
332 static int apndDeviceCharacteristics(sqlite3_file *pFile){
333 pFile = ORIGFILE(pFile);
334 return pFile->pMethods->xDeviceCharacteristics(pFile);
337 /* Create a shared memory file mapping */
338 static int apndShmMap(
339 sqlite3_file *pFile,
340 int iPg,
341 int pgsz,
342 int bExtend,
343 void volatile **pp
345 pFile = ORIGFILE(pFile);
346 return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
349 /* Perform locking on a shared-memory segment */
350 static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
351 pFile = ORIGFILE(pFile);
352 return pFile->pMethods->xShmLock(pFile,offset,n,flags);
355 /* Memory barrier operation on shared memory */
356 static void apndShmBarrier(sqlite3_file *pFile){
357 pFile = ORIGFILE(pFile);
358 pFile->pMethods->xShmBarrier(pFile);
361 /* Unmap a shared memory segment */
362 static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
363 pFile = ORIGFILE(pFile);
364 return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
367 /* Fetch a page of a memory-mapped file */
368 static int apndFetch(
369 sqlite3_file *pFile,
370 sqlite3_int64 iOfst,
371 int iAmt,
372 void **pp
374 ApndFile *p = (ApndFile *)pFile;
375 pFile = ORIGFILE(pFile);
376 return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
379 /* Release a memory-mapped page */
380 static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
381 ApndFile *p = (ApndFile *)pFile;
382 pFile = ORIGFILE(pFile);
383 return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
387 ** Check to see if the file is an ordinary SQLite database file.
389 static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
390 int rc;
391 char zHdr[16];
392 static const char aSqliteHdr[] = "SQLite format 3";
393 if( sz<512 ) return 0;
394 rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
395 if( rc ) return 0;
396 return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
400 ** Try to read the append-mark off the end of a file. Return the
401 ** start of the appended database if the append-mark is present. If
402 ** there is no append-mark, return -1;
404 static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
405 int rc, i;
406 sqlite3_int64 iMark;
407 unsigned char a[APND_MARK_SIZE];
409 if( sz<=APND_MARK_SIZE ) return -1;
410 rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
411 if( rc ) return -1;
412 if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
413 iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
414 for(i=1; i<8; i++){
415 iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
417 return iMark;
421 ** Open an apnd file handle.
423 static int apndOpen(
424 sqlite3_vfs *pVfs,
425 const char *zName,
426 sqlite3_file *pFile,
427 int flags,
428 int *pOutFlags
430 ApndFile *p;
431 sqlite3_file *pSubFile;
432 sqlite3_vfs *pSubVfs;
433 int rc;
434 sqlite3_int64 sz;
435 pSubVfs = ORIGVFS(pVfs);
436 if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
437 return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
439 p = (ApndFile*)pFile;
440 memset(p, 0, sizeof(*p));
441 pSubFile = ORIGFILE(pFile);
442 p->base.pMethods = &apnd_io_methods;
443 rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
444 if( rc ) goto apnd_open_done;
445 rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
446 if( rc ){
447 pSubFile->pMethods->xClose(pSubFile);
448 goto apnd_open_done;
450 if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
451 memmove(pFile, pSubFile, pSubVfs->szOsFile);
452 return SQLITE_OK;
454 p->iMark = 0;
455 p->iPgOne = apndReadMark(sz, pFile);
456 if( p->iPgOne>0 ){
457 return SQLITE_OK;
459 if( (flags & SQLITE_OPEN_CREATE)==0 ){
460 pSubFile->pMethods->xClose(pSubFile);
461 rc = SQLITE_CANTOPEN;
463 p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
464 apnd_open_done:
465 if( rc ) pFile->pMethods = 0;
466 return rc;
470 ** All other VFS methods are pass-thrus.
472 static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
473 return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
475 static int apndAccess(
476 sqlite3_vfs *pVfs,
477 const char *zPath,
478 int flags,
479 int *pResOut
481 return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
483 static int apndFullPathname(
484 sqlite3_vfs *pVfs,
485 const char *zPath,
486 int nOut,
487 char *zOut
489 return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
491 static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
492 return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
494 static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
495 ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
497 static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
498 return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
500 static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
501 ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
503 static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
504 return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
506 static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
507 return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
509 static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
510 return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
512 static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
513 return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
515 static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
516 return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
518 static int apndSetSystemCall(
519 sqlite3_vfs *pVfs,
520 const char *zName,
521 sqlite3_syscall_ptr pCall
523 return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
525 static sqlite3_syscall_ptr apndGetSystemCall(
526 sqlite3_vfs *pVfs,
527 const char *zName
529 return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
531 static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
532 return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
536 #ifdef _WIN32
537 __declspec(dllexport)
538 #endif
540 ** This routine is called when the extension is loaded.
541 ** Register the new VFS.
543 int sqlite3_appendvfs_init(
544 sqlite3 *db,
545 char **pzErrMsg,
546 const sqlite3_api_routines *pApi
548 int rc = SQLITE_OK;
549 sqlite3_vfs *pOrig;
550 SQLITE_EXTENSION_INIT2(pApi);
551 (void)pzErrMsg;
552 (void)db;
553 pOrig = sqlite3_vfs_find(0);
554 apnd_vfs.iVersion = pOrig->iVersion;
555 apnd_vfs.pAppData = pOrig;
556 apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
557 rc = sqlite3_vfs_register(&apnd_vfs, 0);
558 #ifdef APPENDVFS_TEST
559 if( rc==SQLITE_OK ){
560 rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
562 #endif
563 if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
564 return rc;