Poor mans workaround for hanging SDL app/sim on shutdown.
[maemo-rb.git] / apps / tagcache.c
blob6dfc88bba483821e5b9ff7979ca1cd41b468084e
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
10 * Copyright (C) 2005 by Miika Pekkarinen
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
20 ****************************************************************************/
23 * TagCache API
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
58 /*#define LOGF_ENABLE*/
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #ifdef APPLICATION
64 #include <unistd.h> /* readlink() */
65 #include <limits.h> /* PATH_MAX */
66 #endif
67 #include "config.h"
68 #include "ata_idle_notify.h"
69 #include "thread.h"
70 #include "kernel.h"
71 #include "system.h"
72 #include "logf.h"
73 #include "string-extra.h"
74 #include "usb.h"
75 #include "metadata.h"
76 #include "tagcache.h"
77 #include "buffer.h"
78 #include "crc32.h"
79 #include "misc.h"
80 #include "settings.h"
81 #include "dir.h"
82 #include "filefuncs.h"
83 #include "structec.h"
84 #include "debug.h"
86 #ifndef __PCTOOL__
87 #include "lang.h"
88 #include "eeprom_settings.h"
89 #endif
91 #ifdef __PCTOOL__
92 #define yield() do { } while(0)
93 #define sim_sleep(timeout) do { } while(0)
94 #define do_timed_yield() do { } while(0)
95 #endif
97 #ifndef __PCTOOL__
98 /* Tag Cache thread. */
99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
101 static const char tagcache_thread_name[] = "tagcache";
102 #endif
104 /* Previous path when scanning directory tree recursively. */
105 static char curpath[TAG_MAXLEN+32];
107 /* Used when removing duplicates. */
108 static char *tempbuf; /* Allocated when needed. */
109 static long tempbufidx; /* Current location in buffer. */
110 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
111 static long tempbuf_left; /* Buffer space left. */
112 static long tempbuf_pos;
114 #define SORTED_TAGS_COUNT 8
115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
118 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
119 /* Tags we want to get sorted (loaded to the tempbuf). */
120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
121 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
122 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
126 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
127 (1LU << tag_albumartist) | (1LU << tag_grouping))
129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
130 static const char *tags_str[] = { "artist", "album", "genre", "title",
131 "filename", "composer", "comment", "albumartist", "grouping", "year",
132 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
133 "playtime", "lastplayed", "commitid", "mtime" };
135 /* Status information of the tagcache. */
136 static struct tagcache_stat tc_stat;
138 /* Queue commands. */
139 enum tagcache_queue {
140 Q_STOP_SCAN = 0,
141 Q_START_SCAN,
142 Q_IMPORT_CHANGELOG,
143 Q_UPDATE,
144 Q_REBUILD,
146 /* Internal tagcache command queue. */
147 CMD_UPDATE_MASTER_HEADER,
148 CMD_UPDATE_NUMERIC,
151 struct tagcache_command_entry {
152 int32_t command;
153 int32_t idx_id;
154 int32_t tag;
155 int32_t data;
158 #ifndef __PCTOOL__
159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
160 static volatile int command_queue_widx = 0;
161 static volatile int command_queue_ridx = 0;
162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
163 #endif
165 /* Tag database structures. */
167 /* Variable-length tag entry in tag files. */
168 struct tagfile_entry {
169 int32_t tag_length; /* Length of the data in bytes including '\0' */
170 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
171 char tag_data[0]; /* Begin of the tag data */
174 /* Fixed-size tag entry in master db index. */
175 struct index_entry {
176 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
177 int32_t flag; /* Status flags */
180 /* Header is the same in every file. */
181 struct tagcache_header {
182 int32_t magic; /* Header version number */
183 int32_t datasize; /* Data size in bytes */
184 int32_t entry_count; /* Number of entries in this file */
187 struct master_header {
188 struct tagcache_header tch;
189 int32_t serial; /* Increasing counting number */
190 int32_t commitid; /* Number of commits so far */
191 int32_t dirty;
194 /* For the endianess correction */
195 static const char *tagfile_entry_ec = "ll";
197 Note: This should be (1 + TAG_COUNT) amount of l's.
199 static const char *index_entry_ec = "llllllllllllllllllllll";
201 static const char *tagcache_header_ec = "lll";
202 static const char *master_header_ec = "llllll";
204 static struct master_header current_tcmh;
206 #ifdef HAVE_TC_RAMCACHE
207 /* Header is created when loading database to ram. */
208 struct ramcache_header {
209 struct master_header h; /* Header from the master index */
210 struct index_entry *indices; /* Master index file content */
211 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
212 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
215 # ifdef HAVE_EEPROM_SETTINGS
216 struct statefile_header {
217 struct ramcache_header *hdr;
218 struct tagcache_stat tc_stat;
220 # endif
222 /* Pointer to allocated ramcache_header */
223 static struct ramcache_header *hdr;
224 #endif
226 /**
227 * Full tag entries stored in a temporary file waiting
228 * for commit to the cache. */
229 struct temp_file_entry {
230 long tag_offset[TAG_COUNT];
231 short tag_length[TAG_COUNT];
232 long flag;
234 long data_length;
237 struct tempbuf_id_list {
238 long id;
239 struct tempbuf_id_list *next;
242 struct tempbuf_searchidx {
243 long idx_id;
244 char *str;
245 int seek;
246 struct tempbuf_id_list idlist;
249 /* Lookup buffer for fixing messed up index while after sorting. */
250 static long commit_entry_count;
251 static long lookup_buffer_depth;
252 static struct tempbuf_searchidx **lookup;
254 /* Used when building the temporary file. */
255 static int cachefd = -1, filenametag_fd;
256 static int total_entry_count = 0;
257 static int data_size = 0;
258 static int processed_dir_count;
260 /* Thread safe locking */
261 static volatile int write_lock;
262 static volatile int read_lock;
264 static bool delete_entry(long idx_id);
266 const char* tagcache_tag_to_str(int tag)
268 return tags_str[tag];
271 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
272 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
274 return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
277 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
279 return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
282 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
284 return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
287 #ifdef HAVE_DIRCACHE
289 * Returns true if specified flag is still present, i.e., dircache
290 * has not been reloaded.
292 static bool is_dircache_intact(void)
294 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
296 #endif
298 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
300 int fd;
301 char buf[MAX_PATH];
302 int rc;
304 if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
305 return -1;
307 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
309 fd = open(buf, write ? O_RDWR : O_RDONLY);
310 if (fd < 0)
312 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
313 tc_stat.ready = false;
314 return fd;
317 /* Check the header. */
318 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
319 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
321 logf("header error");
322 tc_stat.ready = false;
323 close(fd);
324 return -2;
327 return fd;
330 static int open_master_fd(struct master_header *hdr, bool write)
332 int fd;
333 int rc;
335 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
336 if (fd < 0)
338 logf("master file open failed for R/W");
339 tc_stat.ready = false;
340 return fd;
343 tc_stat.econ = false;
345 /* Check the header. */
346 rc = read(fd, hdr, sizeof(struct master_header));
347 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
349 /* Success. */
350 return fd;
353 /* Trying to read again, this time with endianess correction enabled. */
354 lseek(fd, 0, SEEK_SET);
356 rc = ecread(fd, hdr, 1, master_header_ec, true);
357 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
359 logf("header error");
360 tc_stat.ready = false;
361 close(fd);
362 return -2;
365 tc_stat.econ = true;
367 return fd;
370 #ifndef __PCTOOL__
371 static bool do_timed_yield(void)
373 /* Sorting can lock up for quite a while, so yield occasionally */
374 static long wakeup_tick = 0;
375 if (TIME_AFTER(current_tick, wakeup_tick))
377 wakeup_tick = current_tick + (HZ/4);
378 yield();
379 return true;
381 return false;
383 #endif
385 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
386 static long find_entry_ram(const char *filename,
387 const struct dircache_entry *dc)
389 static long last_pos = 0;
390 int i;
392 /* Check if tagcache is loaded into ram. */
393 if (!tc_stat.ramcache)
394 return -1;
396 if (dc == NULL)
397 dc = dircache_get_entry_ptr(filename);
399 if (dc == NULL)
401 logf("tagcache: file not found.");
402 return -1;
405 try_again:
407 if (last_pos > 0)
408 i = last_pos;
409 else
410 i = 0;
412 for (; i < hdr->h.tch.entry_count; i++)
414 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
416 last_pos = MAX(0, i - 3);
417 return i;
420 do_timed_yield();
423 if (last_pos > 0)
425 last_pos = 0;
426 goto try_again;
429 return -1;
431 #endif
433 static long find_entry_disk(const char *filename_raw, bool localfd)
435 struct tagcache_header tch;
436 static long last_pos = -1;
437 long pos_history[POS_HISTORY_COUNT];
438 long pos_history_idx = 0;
439 bool found = false;
440 struct tagfile_entry tfe;
441 int fd;
442 char buf[TAG_MAXLEN+32];
443 int i;
444 int pos = -1;
446 const char *filename = filename_raw;
447 #ifdef APPLICATION
448 char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
449 if (realpath(filename, pathbuf) == pathbuf)
450 filename = pathbuf;
451 #endif
453 if (!tc_stat.ready)
454 return -2;
456 fd = filenametag_fd;
457 if (fd < 0 || localfd)
459 last_pos = -1;
460 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
461 return -1;
464 check_again:
466 if (last_pos > 0)
467 lseek(fd, last_pos, SEEK_SET);
468 else
469 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
471 while (true)
473 pos = lseek(fd, 0, SEEK_CUR);
474 for (i = pos_history_idx-1; i >= 0; i--)
475 pos_history[i+1] = pos_history[i];
476 pos_history[0] = pos;
478 if (ecread_tagfile_entry(fd, &tfe)
479 != sizeof(struct tagfile_entry))
481 break ;
484 if (tfe.tag_length >= (long)sizeof(buf))
486 logf("too long tag #1");
487 close(fd);
488 if (!localfd)
489 filenametag_fd = -1;
490 last_pos = -1;
491 return -2;
494 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
496 logf("read error #2");
497 close(fd);
498 if (!localfd)
499 filenametag_fd = -1;
500 last_pos = -1;
501 return -3;
504 if (!strcasecmp(filename, buf))
506 last_pos = pos_history[pos_history_idx];
507 found = true;
508 break ;
511 if (pos_history_idx < POS_HISTORY_COUNT - 1)
512 pos_history_idx++;
515 /* Not found? */
516 if (!found)
518 if (last_pos > 0)
520 last_pos = -1;
521 logf("seek again");
522 goto check_again;
525 if (fd != filenametag_fd || localfd)
526 close(fd);
527 return -4;
530 if (fd != filenametag_fd || localfd)
531 close(fd);
533 return tfe.idx_id;
536 static int find_index(const char *filename)
538 long idx_id = -1;
540 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
541 if (tc_stat.ramcache && is_dircache_intact())
542 idx_id = find_entry_ram(filename, NULL);
543 #endif
545 if (idx_id < 0)
546 idx_id = find_entry_disk(filename, true);
548 return idx_id;
551 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
553 int idx_id;
555 if (!tc_stat.ready)
556 return false;
558 idx_id = find_index(filename);
559 if (idx_id < 0)
560 return false;
562 if (!tagcache_search(tcs, tag_filename))
563 return false;
565 tcs->entry_count = 0;
566 tcs->idx_id = idx_id;
568 return true;
571 static bool get_index(int masterfd, int idxid,
572 struct index_entry *idx, bool use_ram)
574 bool localfd = false;
576 if (idxid < 0)
578 logf("Incorrect idxid: %d", idxid);
579 return false;
582 #ifdef HAVE_TC_RAMCACHE
583 if (tc_stat.ramcache && use_ram)
585 if (hdr->indices[idxid].flag & FLAG_DELETED)
586 return false;
588 # ifdef HAVE_DIRCACHE
589 if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
590 || is_dircache_intact())
591 #endif
593 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
594 return true;
597 #else
598 (void)use_ram;
599 #endif
601 if (masterfd < 0)
603 struct master_header tcmh;
605 localfd = true;
606 masterfd = open_master_fd(&tcmh, false);
607 if (masterfd < 0)
608 return false;
611 lseek(masterfd, idxid * sizeof(struct index_entry)
612 + sizeof(struct master_header), SEEK_SET);
613 if (ecread_index_entry(masterfd, idx)
614 != sizeof(struct index_entry))
616 logf("read error #3");
617 if (localfd)
618 close(masterfd);
620 return false;
623 if (localfd)
624 close(masterfd);
626 if (idx->flag & FLAG_DELETED)
627 return false;
629 return true;
632 #ifndef __PCTOOL__
634 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
636 /* We need to exclude all memory only flags & tags when writing to disk. */
637 if (idx->flag & FLAG_DIRCACHE)
639 logf("memory only flags!");
640 return false;
643 #ifdef HAVE_TC_RAMCACHE
644 /* Only update numeric data. Writing the whole index to RAM by memcpy
645 * destroys dircache pointers!
647 if (tc_stat.ramcache)
649 int tag;
650 struct index_entry *idx_ram = &hdr->indices[idxid];
652 for (tag = 0; tag < TAG_COUNT; tag++)
654 if (TAGCACHE_IS_NUMERIC(tag))
656 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
660 /* Don't touch the dircache flag or attributes. */
661 idx_ram->flag = (idx->flag & 0x0000ffff)
662 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
664 #endif
666 lseek(masterfd, idxid * sizeof(struct index_entry)
667 + sizeof(struct master_header), SEEK_SET);
668 if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
670 logf("write error #3");
671 logf("idxid: %d", idxid);
672 return false;
675 return true;
678 #endif /* !__PCTOOL__ */
680 static bool open_files(struct tagcache_search *tcs, int tag)
682 if (tcs->idxfd[tag] < 0)
684 char fn[MAX_PATH];
686 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
687 tcs->idxfd[tag] = open(fn, O_RDONLY);
690 if (tcs->idxfd[tag] < 0)
692 logf("File not open!");
693 return false;
696 return true;
699 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
700 int tag, char *buf, long size)
702 struct tagfile_entry tfe;
703 long seek;
705 *buf = '\0';
707 if (TAGCACHE_IS_NUMERIC(tag))
708 return false;
710 seek = idx->tag_seek[tag];
711 if (seek < 0)
713 logf("Retrieve failed");
714 return false;
717 #ifdef HAVE_TC_RAMCACHE
718 if (tcs->ramsearch)
720 struct tagfile_entry *ep;
722 # ifdef HAVE_DIRCACHE
723 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
724 && is_dircache_intact())
726 dircache_copy_path((struct dircache_entry *)seek,
727 buf, size);
728 return true;
730 else
731 # endif
732 if (tag != tag_filename)
734 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
735 strlcpy(buf, ep->tag_data, size);
737 return true;
740 #endif
742 if (!open_files(tcs, tag))
743 return false;
745 lseek(tcs->idxfd[tag], seek, SEEK_SET);
746 if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
747 != sizeof(struct tagfile_entry))
749 logf("read error #5");
750 return false;
753 if (tfe.tag_length >= size)
755 logf("too small buffer");
756 return false;
759 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
760 tfe.tag_length)
762 logf("read error #6");
763 return false;
766 buf[tfe.tag_length] = '\0';
768 return true;
771 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
773 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
775 #ifndef __PCTOOL__
776 if (! COMMAND_QUEUE_IS_EMPTY)
778 /* Attempt to find tag data through store-to-load forwarding in
779 command queue */
780 long result = -1;
782 mutex_lock(&command_queue_mutex);
784 int ridx = command_queue_widx;
786 while (ridx != command_queue_ridx)
788 if (--ridx < 0)
789 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
791 if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
792 && command_queue[ridx].idx_id == idx_id
793 && command_queue[ridx].tag == tag)
795 result = command_queue[ridx].data;
796 break;
800 mutex_unlock(&command_queue_mutex);
802 if (result >= 0)
804 logf("read_numeric_tag: "
805 "Recovered tag %d value %lX from write queue",
806 tag, result);
807 return result;
810 #endif
812 return idx->tag_seek[tag];
816 static long check_virtual_tags(int tag, int idx_id,
817 const struct index_entry *idx)
819 long data = 0;
821 switch (tag)
823 case tag_virt_length_sec:
824 data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
825 break;
827 case tag_virt_length_min:
828 data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
829 break;
831 case tag_virt_playtime_sec:
832 data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
833 break;
835 case tag_virt_playtime_min:
836 data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
837 break;
839 case tag_virt_autoscore:
840 if (read_numeric_tag(tag_length, idx_id, idx) == 0
841 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
843 data = 0;
845 else
847 /* A straight calculus gives:
848 autoscore = 100 * playtime / length / playcout (1)
849 Now, consider the euclidian division of playtime by length:
850 playtime = alpha * length + beta
851 With:
852 0 <= beta < length
853 Now, (1) becomes:
854 autoscore = 100 * (alpha / playcout + beta / length / playcount)
855 Both terms should be small enough to avoid any overflow
857 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
858 / read_numeric_tag(tag_length, idx_id, idx))
859 + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
860 % read_numeric_tag(tag_length, idx_id, idx)))
861 / read_numeric_tag(tag_length, idx_id, idx);
862 data /= read_numeric_tag(tag_playcount, idx_id, idx);
864 break;
866 /* How many commits before the file has been added to the DB. */
867 case tag_virt_entryage:
868 data = current_tcmh.commitid
869 - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
870 break;
872 default:
873 data = read_numeric_tag(tag, idx_id, idx);
876 return data;
879 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
881 struct index_entry idx;
883 if (!tc_stat.ready)
884 return false;
886 if (!TAGCACHE_IS_NUMERIC(tag))
887 return -1;
889 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
890 return -2;
892 return check_virtual_tags(tag, tcs->idx_id, &idx);
895 inline static bool str_ends_with(const char *str1, const char *str2)
897 int str_len = strlen(str1);
898 int clause_len = strlen(str2);
900 if (clause_len > str_len)
901 return false;
903 return !strcasecmp(&str1[str_len - clause_len], str2);
906 inline static bool str_oneof(const char *str, const char *list)
908 const char *sep;
909 int l, len = strlen(str);
911 while (*list)
913 sep = strchr(list, '|');
914 l = sep ? (long)sep - (long)list : (int)strlen(list);
915 if ((l==len) && !strncasecmp(str, list, len))
916 return true;
917 list += sep ? l + 1 : l;
920 return false;
923 static bool check_against_clause(long numeric, const char *str,
924 const struct tagcache_search_clause *clause)
926 if (clause->numeric)
928 switch (clause->type)
930 case clause_is:
931 return numeric == clause->numeric_data;
932 case clause_is_not:
933 return numeric != clause->numeric_data;
934 case clause_gt:
935 return numeric > clause->numeric_data;
936 case clause_gteq:
937 return numeric >= clause->numeric_data;
938 case clause_lt:
939 return numeric < clause->numeric_data;
940 case clause_lteq:
941 return numeric <= clause->numeric_data;
942 default:
943 logf("Incorrect numeric tag: %d", clause->type);
946 else
948 switch (clause->type)
950 case clause_is:
951 return !strcasecmp(clause->str, str);
952 case clause_is_not:
953 return strcasecmp(clause->str, str);
954 case clause_gt:
955 return 0>strcasecmp(clause->str, str);
956 case clause_gteq:
957 return 0>=strcasecmp(clause->str, str);
958 case clause_lt:
959 return 0<strcasecmp(clause->str, str);
960 case clause_lteq:
961 return 0<=strcasecmp(clause->str, str);
962 case clause_contains:
963 return (strcasestr(str, clause->str) != NULL);
964 case clause_not_contains:
965 return (strcasestr(str, clause->str) == NULL);
966 case clause_begins_with:
967 return (strcasestr(str, clause->str) == str);
968 case clause_not_begins_with:
969 return (strcasestr(str, clause->str) != str);
970 case clause_ends_with:
971 return str_ends_with(str, clause->str);
972 case clause_not_ends_with:
973 return !str_ends_with(str, clause->str);
974 case clause_oneof:
975 return str_oneof(str, clause->str);
977 default:
978 logf("Incorrect tag: %d", clause->type);
982 return false;
985 static bool check_clauses(struct tagcache_search *tcs,
986 struct index_entry *idx,
987 struct tagcache_search_clause **clause, int count)
989 int i;
991 #ifdef HAVE_TC_RAMCACHE
992 if (tcs->ramsearch)
994 /* Go through all conditional clauses. */
995 for (i = 0; i < count; i++)
997 struct tagfile_entry *tfe;
998 int seek;
999 char buf[256];
1000 char *str = NULL;
1002 seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1004 if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1006 if (clause[i]->tag == tag_filename)
1008 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1009 str = buf;
1011 else
1013 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1014 str = tfe->tag_data;
1018 if (!check_against_clause(seek, str, clause[i]))
1019 return false;
1022 else
1023 #endif
1025 /* Check for conditions. */
1026 for (i = 0; i < count; i++)
1028 struct tagfile_entry tfe;
1029 int seek;
1030 char str[256];
1032 seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1034 memset(str, 0, sizeof str);
1035 if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1037 int fd = tcs->idxfd[clause[i]->tag];
1038 lseek(fd, seek, SEEK_SET);
1039 ecread_tagfile_entry(fd, &tfe);
1040 if (tfe.tag_length >= (int)sizeof(str))
1042 logf("Too long tag read!");
1043 break ;
1046 read(fd, str, tfe.tag_length);
1048 /* Check if entry has been deleted. */
1049 if (str[0] == '\0')
1050 break;
1053 if (!check_against_clause(seek, str, clause[i]))
1054 return false;
1058 return true;
1061 bool tagcache_check_clauses(struct tagcache_search *tcs,
1062 struct tagcache_search_clause **clause, int count)
1064 struct index_entry idx;
1066 if (count == 0)
1067 return true;
1069 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1070 return false;
1072 return check_clauses(tcs, &idx, clause, count);
1075 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1077 int i;
1079 /* If uniq buffer is not defined we must return true for search to work. */
1080 if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1081 && !TAGCACHE_IS_NUMERIC(tcs->type)))
1083 return true;
1086 for (i = 0; i < tcs->unique_list_count; i++)
1088 /* Return false if entry is found. */
1089 if (tcs->unique_list[i] == id)
1090 return false;
1093 if (tcs->unique_list_count < tcs->unique_list_capacity)
1095 tcs->unique_list[i] = id;
1096 tcs->unique_list_count++;
1099 return true;
1102 static bool build_lookup_list(struct tagcache_search *tcs)
1104 struct index_entry entry;
1105 int i, j;
1107 tcs->seek_list_count = 0;
1109 #ifdef HAVE_TC_RAMCACHE
1110 if (tcs->ramsearch
1111 # ifdef HAVE_DIRCACHE
1112 && (tcs->type != tag_filename || is_dircache_intact())
1113 # endif
1116 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1118 struct tagcache_seeklist_entry *seeklist;
1119 struct index_entry *idx = &hdr->indices[i];
1120 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1121 break ;
1123 /* Skip deleted files. */
1124 if (idx->flag & FLAG_DELETED)
1125 continue;
1127 /* Go through all filters.. */
1128 for (j = 0; j < tcs->filter_count; j++)
1130 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1132 break ;
1136 if (j < tcs->filter_count)
1137 continue ;
1139 /* Check for conditions. */
1140 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1141 continue;
1143 /* Add to the seek list if not already in uniq buffer. */
1144 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1145 continue;
1147 /* Lets add it. */
1148 seeklist = &tcs->seeklist[tcs->seek_list_count];
1149 seeklist->seek = idx->tag_seek[tcs->type];
1150 seeklist->flag = idx->flag;
1151 seeklist->idx_id = i;
1152 tcs->seek_list_count++;
1155 tcs->seek_pos = i;
1157 return tcs->seek_list_count > 0;
1159 #endif
1161 if (tcs->masterfd < 0)
1163 struct master_header tcmh;
1164 tcs->masterfd = open_master_fd(&tcmh, false);
1167 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1168 sizeof(struct master_header), SEEK_SET);
1170 while (ecread_index_entry(tcs->masterfd, &entry)
1171 == sizeof(struct index_entry))
1173 struct tagcache_seeklist_entry *seeklist;
1175 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1176 break ;
1178 i = tcs->seek_pos;
1179 tcs->seek_pos++;
1181 /* Check if entry has been deleted. */
1182 if (entry.flag & FLAG_DELETED)
1183 continue;
1185 /* Go through all filters.. */
1186 for (j = 0; j < tcs->filter_count; j++)
1188 if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1189 break ;
1192 if (j < tcs->filter_count)
1193 continue ;
1195 /* Check for conditions. */
1196 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1197 continue;
1199 /* Add to the seek list if not already in uniq buffer. */
1200 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1201 continue;
1203 /* Lets add it. */
1204 seeklist = &tcs->seeklist[tcs->seek_list_count];
1205 seeklist->seek = entry.tag_seek[tcs->type];
1206 seeklist->flag = entry.flag;
1207 seeklist->idx_id = i;
1208 tcs->seek_list_count++;
1210 yield();
1213 return tcs->seek_list_count > 0;
1217 static void remove_files(void)
1219 int i;
1220 char buf[MAX_PATH];
1222 tc_stat.ready = false;
1223 tc_stat.ramcache = false;
1224 tc_stat.econ = false;
1225 remove(TAGCACHE_FILE_MASTER);
1226 for (i = 0; i < TAG_COUNT; i++)
1228 if (TAGCACHE_IS_NUMERIC(i))
1229 continue;
1231 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1232 remove(buf);
1237 static bool check_all_headers(void)
1239 struct master_header myhdr;
1240 struct tagcache_header tch;
1241 int tag;
1242 int fd;
1244 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1245 return false;
1247 close(fd);
1248 if (myhdr.dirty)
1250 logf("tagcache is dirty!");
1251 return false;
1254 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1256 for (tag = 0; tag < TAG_COUNT; tag++)
1258 if (TAGCACHE_IS_NUMERIC(tag))
1259 continue;
1261 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1262 return false;
1264 close(fd);
1267 return true;
1270 bool tagcache_is_busy(void)
1272 return read_lock || write_lock;
1275 bool tagcache_search(struct tagcache_search *tcs, int tag)
1277 struct tagcache_header tag_hdr;
1278 struct master_header master_hdr;
1279 int i;
1281 while (read_lock)
1282 sleep(1);
1284 memset(tcs, 0, sizeof(struct tagcache_search));
1285 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1286 return false;
1288 tcs->position = sizeof(struct tagcache_header);
1289 tcs->type = tag;
1290 tcs->seek_pos = 0;
1291 tcs->list_position = 0;
1292 tcs->seek_list_count = 0;
1293 tcs->filter_count = 0;
1294 tcs->masterfd = -1;
1296 for (i = 0; i < TAG_COUNT; i++)
1297 tcs->idxfd[i] = -1;
1299 #ifndef HAVE_TC_RAMCACHE
1300 tcs->ramsearch = false;
1301 #else
1302 tcs->ramsearch = tc_stat.ramcache;
1303 if (tcs->ramsearch)
1305 tcs->entry_count = hdr->entry_count[tcs->type];
1307 else
1308 #endif
1310 /* Always open as R/W so we can pass tcs to functions that modify data also
1311 * without failing. */
1312 tcs->masterfd = open_master_fd(&master_hdr, true);
1313 if (tcs->masterfd < 0)
1314 return false;
1316 if (!TAGCACHE_IS_NUMERIC(tcs->type))
1318 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1319 if (tcs->idxfd[tcs->type] < 0)
1320 return false;
1322 tcs->entry_count = tag_hdr.entry_count;
1324 else
1326 tcs->entry_count = master_hdr.tch.entry_count;
1330 tcs->valid = true;
1331 tcs->initialized = true;
1332 write_lock++;
1334 return true;
1337 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1338 void *buffer, long length)
1340 tcs->unique_list = (unsigned long *)buffer;
1341 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1342 tcs->unique_list_count = 0;
1345 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1346 int tag, int seek)
1348 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1349 return false;
1351 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1352 return false;
1354 tcs->filter_tag[tcs->filter_count] = tag;
1355 tcs->filter_seek[tcs->filter_count] = seek;
1356 tcs->filter_count++;
1358 return true;
1361 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1362 struct tagcache_search_clause *clause)
1364 int i;
1366 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1368 logf("Too many clauses");
1369 return false;
1372 /* Check if there is already a similar filter in present (filters are
1373 * much faster than clauses).
1375 for (i = 0; i < tcs->filter_count; i++)
1377 if (tcs->filter_tag[i] == clause->tag)
1378 return true;
1381 if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1383 char buf[MAX_PATH];
1385 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1386 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1389 tcs->clause[tcs->clause_count] = clause;
1390 tcs->clause_count++;
1392 return true;
1395 static bool get_next(struct tagcache_search *tcs)
1397 static char buf[TAG_MAXLEN+32];
1398 struct tagfile_entry entry;
1399 long flag = 0;
1401 if (!tcs->valid || !tc_stat.ready)
1402 return false;
1404 if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1405 #ifdef HAVE_TC_RAMCACHE
1406 && !tcs->ramsearch
1407 #endif
1409 return false;
1411 /* Relative fetch. */
1412 if (tcs->filter_count > 0 || tcs->clause_count > 0
1413 || TAGCACHE_IS_NUMERIC(tcs->type)
1414 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1415 /* We need to retrieve flag status for dircache. */
1416 || (tcs->ramsearch && tcs->type == tag_filename)
1417 #endif
1420 struct tagcache_seeklist_entry *seeklist;
1422 /* Check for end of list. */
1423 if (tcs->list_position == tcs->seek_list_count)
1425 tcs->list_position = 0;
1427 /* Try to fetch more. */
1428 if (!build_lookup_list(tcs))
1430 tcs->valid = false;
1431 return false;
1435 seeklist = &tcs->seeklist[tcs->list_position];
1436 flag = seeklist->flag;
1437 tcs->position = seeklist->seek;
1438 tcs->idx_id = seeklist->idx_id;
1439 tcs->list_position++;
1441 else
1443 if (tcs->entry_count == 0)
1445 tcs->valid = false;
1446 return false;
1449 tcs->entry_count--;
1452 tcs->result_seek = tcs->position;
1454 if (TAGCACHE_IS_NUMERIC(tcs->type))
1456 snprintf(buf, sizeof(buf), "%ld", tcs->position);
1457 tcs->result = buf;
1458 tcs->result_len = strlen(buf) + 1;
1459 return true;
1462 /* Direct fetch. */
1463 #ifdef HAVE_TC_RAMCACHE
1464 if (tcs->ramsearch)
1467 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1468 if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1469 && is_dircache_intact())
1471 dircache_copy_path((struct dircache_entry *)tcs->position,
1472 buf, sizeof buf);
1473 tcs->result = buf;
1474 tcs->result_len = strlen(buf) + 1;
1475 tcs->ramresult = false;
1477 return true;
1479 else
1480 #endif
1481 if (tcs->type != tag_filename)
1483 struct tagfile_entry *ep;
1485 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1486 tcs->result = ep->tag_data;
1487 tcs->result_len = strlen(tcs->result) + 1;
1488 tcs->idx_id = ep->idx_id;
1489 tcs->ramresult = true;
1491 /* Increase position for the next run. This may get overwritten. */
1492 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1494 return true;
1497 #endif
1499 if (!open_files(tcs, tcs->type))
1501 tcs->valid = false;
1502 return false;
1505 /* Seek stream to the correct position and continue to direct fetch. */
1506 lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1508 if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1510 logf("read error #5");
1511 tcs->valid = false;
1512 return false;
1515 if (entry.tag_length > (long)sizeof(buf))
1517 tcs->valid = false;
1518 logf("too long tag #2");
1519 logf("P:%lX/%lX", tcs->position, entry.tag_length);
1520 return false;
1523 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1525 tcs->valid = false;
1526 logf("read error #4");
1527 return false;
1531 Update the position for the next read (this may be overridden
1532 if filters or clauses are being used).
1534 tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1535 tcs->result = buf;
1536 tcs->result_len = strlen(tcs->result) + 1;
1537 tcs->idx_id = entry.idx_id;
1538 tcs->ramresult = false;
1540 return true;
1543 bool tagcache_get_next(struct tagcache_search *tcs)
1545 while (get_next(tcs))
1547 if (tcs->result_len > 1)
1548 return true;
1551 return false;
1554 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1555 int tag, char *buf, long size)
1557 struct index_entry idx;
1559 *buf = '\0';
1560 if (!get_index(tcs->masterfd, idxid, &idx, true))
1561 return false;
1563 return retrieve(tcs, &idx, tag, buf, size);
1566 static bool update_master_header(void)
1568 struct master_header myhdr;
1569 int fd;
1571 if (!tc_stat.ready)
1572 return false;
1574 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1575 return false;
1577 myhdr.serial = current_tcmh.serial;
1578 myhdr.commitid = current_tcmh.commitid;
1579 myhdr.dirty = current_tcmh.dirty;
1581 /* Write it back */
1582 lseek(fd, 0, SEEK_SET);
1583 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1584 close(fd);
1586 #ifdef HAVE_TC_RAMCACHE
1587 if (hdr)
1589 hdr->h.serial = current_tcmh.serial;
1590 hdr->h.commitid = current_tcmh.commitid;
1591 hdr->h.dirty = current_tcmh.dirty;
1593 #endif
1595 return true;
1598 #if 0
1600 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1602 struct tagentry *entry;
1604 if (tcs->type != tag_title)
1605 return ;
1607 /* We will need reserve buffer for this. */
1608 if (tcs->ramcache)
1610 struct tagfile_entry *ep;
1612 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1613 tcs->seek_list[tcs->seek_list_count];
1616 entry = find_entry_ram();
1619 #endif
1621 void tagcache_search_finish(struct tagcache_search *tcs)
1623 int i;
1625 if (!tcs->initialized)
1626 return;
1628 if (tcs->masterfd >= 0)
1630 close(tcs->masterfd);
1631 tcs->masterfd = -1;
1634 for (i = 0; i < TAG_COUNT; i++)
1636 if (tcs->idxfd[i] >= 0)
1638 close(tcs->idxfd[i]);
1639 tcs->idxfd[i] = -1;
1643 tcs->ramsearch = false;
1644 tcs->valid = false;
1645 tcs->initialized = 0;
1646 if (write_lock > 0)
1647 write_lock--;
1650 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1651 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1653 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1656 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1658 return check_virtual_tags(tag, idx_id, entry);
1661 static char* get_tag_string(const struct index_entry *entry, int tag)
1663 char* s = get_tag(entry, tag)->tag_data;
1664 return strcmp(s, UNTAGGED) ? s : NULL;
1667 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1669 struct index_entry *entry;
1670 int idx_id;
1672 if (!tc_stat.ready || !tc_stat.ramcache)
1673 return false;
1675 /* Find the corresponding entry in tagcache. */
1676 idx_id = find_entry_ram(filename, NULL);
1677 if (idx_id < 0)
1678 return false;
1680 entry = &hdr->indices[idx_id];
1682 memset(id3, 0, sizeof(struct mp3entry));
1684 id3->title = get_tag_string(entry, tag_title);
1685 id3->artist = get_tag_string(entry, tag_artist);
1686 id3->album = get_tag_string(entry, tag_album);
1687 id3->genre_string = get_tag_string(entry, tag_genre);
1688 id3->composer = get_tag_string(entry, tag_composer);
1689 id3->comment = get_tag_string(entry, tag_comment);
1690 id3->albumartist = get_tag_string(entry, tag_albumartist);
1691 id3->grouping = get_tag_string(entry, tag_grouping);
1693 id3->length = get_tag_numeric(entry, tag_length, idx_id);
1694 id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id);
1695 id3->rating = get_tag_numeric(entry, tag_rating, idx_id);
1696 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1697 id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1698 id3->year = get_tag_numeric(entry, tag_year, idx_id);
1700 id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1701 id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1702 id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1703 if (id3->bitrate == 0)
1704 id3->bitrate = 1;
1706 #if CONFIG_CODEC == SWCODEC
1707 if (global_settings.autoresume_enable)
1709 id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1710 logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1711 id3->title, id3->offset);
1713 #endif
1715 return true;
1717 #endif
1719 static inline void write_item(const char *item)
1721 int len = strlen(item) + 1;
1723 data_size += len;
1724 write(cachefd, item, len);
1727 static int check_if_empty(char **tag)
1729 int length;
1731 if (*tag == NULL || **tag == '\0')
1733 *tag = UNTAGGED;
1734 return sizeof(UNTAGGED); /* Tag length */
1737 length = strlen(*tag);
1738 if (length > TAG_MAXLEN)
1740 logf("over length tag: %s", *tag);
1741 length = TAG_MAXLEN;
1742 (*tag)[length] = '\0';
1745 return length + 1;
1748 #define ADD_TAG(entry,tag,data) \
1749 /* Adding tag */ \
1750 entry.tag_offset[tag] = offset; \
1751 entry.tag_length[tag] = check_if_empty(data); \
1752 offset += entry.tag_length[tag]
1753 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1754 * idea, as it uses lots of stack and is called from a recursive function
1755 * (check_dir).
1757 static void __attribute__ ((noinline)) add_tagcache(char *path,
1758 unsigned long mtime
1759 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1760 ,const struct dircache_entry *dc
1761 #endif
1764 struct mp3entry id3;
1765 struct temp_file_entry entry;
1766 bool ret;
1767 int fd;
1768 int idx_id = -1;
1769 char tracknumfix[3];
1770 int offset = 0;
1771 int path_length = strlen(path);
1772 bool has_albumartist;
1773 bool has_grouping;
1775 #ifdef SIMULATOR
1776 /* Crude logging for the sim - to aid in debugging */
1777 int logfd = open(ROCKBOX_DIR "/database.log",
1778 O_WRONLY | O_APPEND | O_CREAT, 0666);
1779 if (logfd >= 0) {
1780 write(logfd, path, strlen(path));
1781 write(logfd, "\n", 1);
1782 close(logfd);
1784 #endif
1786 if (cachefd < 0)
1787 return ;
1789 /* Check for overlength file path. */
1790 if (path_length > TAG_MAXLEN)
1792 /* Path can't be shortened. */
1793 logf("Too long path: %s", path);
1794 return ;
1797 /* Check if the file is supported. */
1798 if (probe_file_format(path) == AFMT_UNKNOWN)
1799 return ;
1801 /* Check if the file is already cached. */
1802 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1803 if (tc_stat.ramcache && is_dircache_intact())
1805 idx_id = find_entry_ram(path, dc);
1807 #endif
1809 /* Be sure the entry doesn't exist. */
1810 if (filenametag_fd >= 0 && idx_id < 0)
1811 idx_id = find_entry_disk(path, false);
1813 /* Check if file has been modified. */
1814 if (idx_id >= 0)
1816 struct index_entry idx;
1818 /* TODO: Mark that the index exists (for fast reverse scan) */
1819 //found_idx[idx_id/8] |= idx_id%8;
1821 if (!get_index(-1, idx_id, &idx, true))
1823 logf("failed to retrieve index entry");
1824 return ;
1827 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1829 /* No changes to file. */
1830 return ;
1833 /* Metadata might have been changed. Delete the entry. */
1834 logf("Re-adding: %s", path);
1835 if (!delete_entry(idx_id))
1837 logf("delete_entry failed: %d", idx_id);
1838 return ;
1842 fd = open(path, O_RDONLY);
1843 if (fd < 0)
1845 logf("open fail: %s", path);
1846 return ;
1849 memset(&id3, 0, sizeof(struct mp3entry));
1850 memset(&entry, 0, sizeof(struct temp_file_entry));
1851 memset(&tracknumfix, 0, sizeof(tracknumfix));
1852 ret = get_metadata(&id3, fd, path);
1853 close(fd);
1855 if (!ret)
1856 return ;
1858 logf("-> %s", path);
1860 /* Generate track number if missing. */
1861 if (id3.tracknum <= 0)
1863 const char *p = strrchr(path, '.');
1865 if (p == NULL)
1866 p = &path[strlen(path)-1];
1868 while (*p != '/')
1870 if (isdigit(*p) && isdigit(*(p-1)))
1872 tracknumfix[1] = *p--;
1873 tracknumfix[0] = *p;
1874 break;
1876 p--;
1879 if (tracknumfix[0] != '\0')
1881 id3.tracknum = atoi(tracknumfix);
1882 /* Set a flag to indicate track number has been generated. */
1883 entry.flag |= FLAG_TRKNUMGEN;
1885 else
1887 /* Unable to generate track number. */
1888 id3.tracknum = -1;
1892 /* Numeric tags */
1893 entry.tag_offset[tag_year] = id3.year;
1894 entry.tag_offset[tag_discnumber] = id3.discnum;
1895 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1896 entry.tag_offset[tag_length] = id3.length;
1897 entry.tag_offset[tag_bitrate] = id3.bitrate;
1898 entry.tag_offset[tag_mtime] = mtime;
1900 /* String tags. */
1901 has_albumartist = id3.albumartist != NULL
1902 && strlen(id3.albumartist) > 0;
1903 has_grouping = id3.grouping != NULL
1904 && strlen(id3.grouping) > 0;
1906 ADD_TAG(entry, tag_filename, &path);
1907 ADD_TAG(entry, tag_title, &id3.title);
1908 ADD_TAG(entry, tag_artist, &id3.artist);
1909 ADD_TAG(entry, tag_album, &id3.album);
1910 ADD_TAG(entry, tag_genre, &id3.genre_string);
1911 ADD_TAG(entry, tag_composer, &id3.composer);
1912 ADD_TAG(entry, tag_comment, &id3.comment);
1913 if (has_albumartist)
1915 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1917 else
1919 ADD_TAG(entry, tag_albumartist, &id3.artist);
1921 if (has_grouping)
1923 ADD_TAG(entry, tag_grouping, &id3.grouping);
1925 else
1927 ADD_TAG(entry, tag_grouping, &id3.title);
1929 entry.data_length = offset;
1931 /* Write the header */
1932 write(cachefd, &entry, sizeof(struct temp_file_entry));
1934 /* And tags also... Correct order is critical */
1935 write_item(path);
1936 write_item(id3.title);
1937 write_item(id3.artist);
1938 write_item(id3.album);
1939 write_item(id3.genre_string);
1940 write_item(id3.composer);
1941 write_item(id3.comment);
1942 if (has_albumartist)
1944 write_item(id3.albumartist);
1946 else
1948 write_item(id3.artist);
1950 if (has_grouping)
1952 write_item(id3.grouping);
1954 else
1956 write_item(id3.title);
1958 total_entry_count++;
1961 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1963 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1964 int len = strlen(str)+1;
1965 int i;
1966 unsigned crc32;
1967 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1968 char buf[TAG_MAXLEN+32];
1970 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1971 buf[i] = tolower(str[i]);
1972 buf[i] = '\0';
1974 crc32 = crc_32(buf, i, 0xffffffff);
1976 if (unique)
1978 /* Check if the crc does not exist -> entry does not exist for sure. */
1979 for (i = 0; i < tempbufidx; i++)
1981 if (crcbuf[-i] != crc32)
1982 continue;
1984 if (!strcasecmp(str, index[i].str))
1986 if (id < 0 || id >= lookup_buffer_depth)
1988 logf("lookup buf overf.: %d", id);
1989 return false;
1992 lookup[id] = &index[i];
1993 return true;
1998 /* Insert to CRC buffer. */
1999 crcbuf[-tempbufidx] = crc32;
2000 tempbuf_left -= 4;
2002 /* Insert it to the buffer. */
2003 tempbuf_left -= len;
2004 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
2005 return false;
2007 if (id >= lookup_buffer_depth)
2009 logf("lookup buf overf. #2: %d", id);
2010 return false;
2013 if (id >= 0)
2015 lookup[id] = &index[tempbufidx];
2016 index[tempbufidx].idlist.id = id;
2018 else
2019 index[tempbufidx].idlist.id = -1;
2021 index[tempbufidx].idlist.next = NULL;
2022 index[tempbufidx].idx_id = idx_id;
2023 index[tempbufidx].seek = -1;
2024 index[tempbufidx].str = &tempbuf[tempbuf_pos];
2025 memcpy(index[tempbufidx].str, str, len);
2026 tempbuf_pos += len;
2027 tempbufidx++;
2029 return true;
2032 static int compare(const void *p1, const void *p2)
2034 do_timed_yield();
2036 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2037 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2039 if (strcmp(e1->str, UNTAGGED) == 0)
2041 if (strcmp(e2->str, UNTAGGED) == 0)
2042 return 0;
2043 return -1;
2045 else if (strcmp(e2->str, UNTAGGED) == 0)
2046 return 1;
2048 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2051 static int tempbuf_sort(int fd)
2053 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2054 struct tagfile_entry fe;
2055 int i;
2056 int length;
2058 /* Generate reverse lookup entries. */
2059 for (i = 0; i < lookup_buffer_depth; i++)
2061 struct tempbuf_id_list *idlist;
2063 if (!lookup[i])
2064 continue;
2066 if (lookup[i]->idlist.id == i)
2067 continue;
2069 idlist = &lookup[i]->idlist;
2070 while (idlist->next != NULL)
2071 idlist = idlist->next;
2073 tempbuf_left -= sizeof(struct tempbuf_id_list);
2074 if (tempbuf_left - 4 < 0)
2075 return -1;
2077 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2078 if (tempbuf_pos & 0x03)
2080 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2081 tempbuf_left -= 3;
2082 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2084 tempbuf_pos += sizeof(struct tempbuf_id_list);
2086 idlist = idlist->next;
2087 idlist->id = i;
2088 idlist->next = NULL;
2090 do_timed_yield();
2093 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2094 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2096 for (i = 0; i < tempbufidx; i++)
2098 struct tempbuf_id_list *idlist = &index[i].idlist;
2100 /* Fix the lookup list. */
2101 while (idlist != NULL)
2103 if (idlist->id >= 0)
2104 lookup[idlist->id] = &index[i];
2105 idlist = idlist->next;
2108 index[i].seek = lseek(fd, 0, SEEK_CUR);
2109 length = strlen(index[i].str) + 1;
2110 fe.tag_length = length;
2111 fe.idx_id = index[i].idx_id;
2113 /* Check the chunk alignment. */
2114 if ((fe.tag_length + sizeof(struct tagfile_entry))
2115 % TAGFILE_ENTRY_CHUNK_LENGTH)
2117 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2118 ((fe.tag_length + sizeof(struct tagfile_entry))
2119 % TAGFILE_ENTRY_CHUNK_LENGTH);
2122 #ifdef TAGCACHE_STRICT_ALIGN
2123 /* Make sure the entry is long aligned. */
2124 if (index[i].seek & 0x03)
2126 logf("tempbuf_sort: alignment error!");
2127 return -3;
2129 #endif
2131 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2132 sizeof(struct tagfile_entry))
2134 logf("tempbuf_sort: write error #1");
2135 return -1;
2138 if (write(fd, index[i].str, length) != length)
2140 logf("tempbuf_sort: write error #2");
2141 return -2;
2144 /* Write some padding. */
2145 if (fe.tag_length - length > 0)
2146 write(fd, "XXXXXXXX", fe.tag_length - length);
2149 return i;
2152 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2154 if (id < 0 || id >= lookup_buffer_depth)
2155 return NULL;
2157 return lookup[id];
2161 inline static int tempbuf_find_location(int id)
2163 struct tempbuf_searchidx *entry;
2165 entry = tempbuf_locate(id);
2166 if (entry == NULL)
2167 return -1;
2169 return entry->seek;
2172 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2174 struct master_header tcmh;
2175 struct index_entry idx;
2176 int masterfd;
2177 int masterfd_pos;
2178 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2179 int max_entries;
2180 int entries_processed = 0;
2181 int i, j;
2182 char buf[TAG_MAXLEN];
2184 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2186 logf("Building numeric indices...");
2187 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2189 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2190 return false;
2192 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2193 SEEK_CUR);
2194 if (masterfd_pos == filesize(masterfd))
2196 logf("we can't append!");
2197 close(masterfd);
2198 return false;
2201 while (entries_processed < h->entry_count)
2203 int count = MIN(h->entry_count - entries_processed, max_entries);
2205 /* Read in as many entries as possible. */
2206 for (i = 0; i < count; i++)
2208 struct temp_file_entry *tfe = &entrybuf[i];
2209 int datastart;
2211 /* Read in numeric data. */
2212 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2213 sizeof(struct temp_file_entry))
2215 logf("read fail #1");
2216 close(masterfd);
2217 return false;
2220 datastart = lseek(tmpfd, 0, SEEK_CUR);
2223 * Read string data from the following tags:
2224 * - tag_filename
2225 * - tag_artist
2226 * - tag_album
2227 * - tag_title
2229 * A crc32 hash is calculated from the read data
2230 * and stored back to the data offset field kept in memory.
2232 #define tmpdb_read_string_tag(tag) \
2233 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2234 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2236 logf("read fail: buffer overflow"); \
2237 close(masterfd); \
2238 return false; \
2241 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2242 tfe->tag_length[tag]) \
2244 logf("read fail #2"); \
2245 close(masterfd); \
2246 return false; \
2249 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2250 lseek(tmpfd, datastart, SEEK_SET)
2252 tmpdb_read_string_tag(tag_filename);
2253 tmpdb_read_string_tag(tag_artist);
2254 tmpdb_read_string_tag(tag_album);
2255 tmpdb_read_string_tag(tag_title);
2257 /* Seek to the end of the string data. */
2258 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2261 /* Backup the master index position. */
2262 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2263 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2265 /* Check if we can resurrect some deleted runtime statistics data. */
2266 for (i = 0; i < tcmh.tch.entry_count; i++)
2268 /* Read the index entry. */
2269 if (ecread_index_entry(masterfd, &idx)
2270 != sizeof(struct index_entry))
2272 logf("read fail #3");
2273 close(masterfd);
2274 return false;
2278 * Skip unless the entry is marked as being deleted
2279 * or the data has already been resurrected.
2281 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2282 continue;
2284 /* Now try to match the entry. */
2286 * To succesfully match a song, the following conditions
2287 * must apply:
2289 * For numeric fields: tag_length
2290 * - Full identical match is required
2292 * If tag_filename matches, no further checking necessary.
2294 * For string hashes: tag_artist, tag_album, tag_title
2295 * - Two of these must match
2297 for (j = 0; j < count; j++)
2299 struct temp_file_entry *tfe = &entrybuf[j];
2301 /* Try to match numeric fields first. */
2302 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2303 continue;
2305 /* Now it's time to do the hash matching. */
2306 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2308 int match_count = 0;
2310 /* No filename match, check if we can match two other tags. */
2311 #define tmpdb_match(tag) \
2312 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2313 match_count++
2315 tmpdb_match(tag_artist);
2316 tmpdb_match(tag_album);
2317 tmpdb_match(tag_title);
2319 if (match_count < 2)
2321 /* Still no match found, give up. */
2322 continue;
2326 /* A match found, now copy & resurrect the statistical data. */
2327 #define tmpdb_copy_tag(tag) \
2328 tfe->tag_offset[tag] = idx.tag_seek[tag]
2330 tmpdb_copy_tag(tag_playcount);
2331 tmpdb_copy_tag(tag_rating);
2332 tmpdb_copy_tag(tag_playtime);
2333 tmpdb_copy_tag(tag_lastplayed);
2334 tmpdb_copy_tag(tag_commitid);
2335 tmpdb_copy_tag(tag_lastoffset);
2337 /* Avoid processing this entry again. */
2338 idx.flag |= FLAG_RESURRECTED;
2340 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2341 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2343 logf("masterfd writeback fail #1");
2344 close(masterfd);
2345 return false;
2348 logf("Entry resurrected");
2353 /* Restore the master index position. */
2354 lseek(masterfd, masterfd_pos, SEEK_SET);
2356 /* Commit the data to the index. */
2357 for (i = 0; i < count; i++)
2359 int loc = lseek(masterfd, 0, SEEK_CUR);
2361 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2363 logf("read fail #3");
2364 close(masterfd);
2365 return false;
2368 for (j = 0; j < TAG_COUNT; j++)
2370 if (!TAGCACHE_IS_NUMERIC(j))
2371 continue;
2373 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2375 idx.flag = entrybuf[i].flag;
2377 if (idx.tag_seek[tag_commitid])
2379 /* Data has been resurrected. */
2380 idx.flag |= FLAG_DIRTYNUM;
2382 else if (tc_stat.ready && current_tcmh.commitid > 0)
2384 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2385 idx.flag |= FLAG_DIRTYNUM;
2388 /* Write back the updated index. */
2389 lseek(masterfd, loc, SEEK_SET);
2390 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2392 logf("write fail");
2393 close(masterfd);
2394 return false;
2398 entries_processed += count;
2399 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2402 close(masterfd);
2404 return true;
2408 * Return values:
2409 * > 0 success
2410 * == 0 temporary failure
2411 * < 0 fatal error
2413 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2415 int i;
2416 struct tagcache_header tch;
2417 struct master_header tcmh;
2418 struct index_entry idxbuf[IDX_BUF_DEPTH];
2419 int idxbuf_pos;
2420 char buf[TAG_MAXLEN+32];
2421 int fd = -1, masterfd;
2422 bool error = false;
2423 int init;
2424 int masterfd_pos;
2426 logf("Building index: %d", index_type);
2428 /* Check the number of entries we need to allocate ram for. */
2429 commit_entry_count = h->entry_count + 1;
2431 masterfd = open_master_fd(&tcmh, false);
2432 if (masterfd >= 0)
2434 commit_entry_count += tcmh.tch.entry_count;
2435 close(masterfd);
2437 else
2438 remove_files(); /* Just to be sure we are clean. */
2440 /* Open the index file, which contains the tag names. */
2441 fd = open_tag_fd(&tch, index_type, true);
2442 if (fd >= 0)
2444 logf("tch.datasize=%ld", tch.datasize);
2445 lookup_buffer_depth = 1 +
2446 /* First part */ commit_entry_count +
2447 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2449 else
2451 lookup_buffer_depth = 1 +
2452 /* First part */ commit_entry_count +
2453 /* Second part */ 0;
2456 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2457 logf("commit_entry_count=%ld", commit_entry_count);
2459 /* Allocate buffer for all index entries from both old and new
2460 * tag files. */
2461 tempbufidx = 0;
2462 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2464 /* Allocate lookup buffer. The first portion of commit_entry_count
2465 * contains the new tags in the temporary file and the second
2466 * part for locating entries already in the db.
2468 * New tags Old tags
2469 * +---------+---------------------------+
2470 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2471 * +---------+---------------------------+
2473 * Old tags are inserted to a temporary buffer with position:
2474 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2475 * And new tags with index:
2476 * tempbuf_insert(idx, ...);
2478 * The buffer is sorted and written into tag file:
2479 * tempbuf_sort(...);
2480 * leaving master index locations messed up.
2482 * That is fixed using the lookup buffer for old tags:
2483 * new_seek = tempbuf_find_location(old_seek, ...);
2484 * and for new tags:
2485 * new_seek = tempbuf_find_location(idx);
2487 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2488 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2489 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2491 /* And calculate the remaining data space used mainly for storing
2492 * tag data (strings). */
2493 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2494 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2496 logf("Buffer way too small!");
2497 return 0;
2500 if (fd >= 0)
2503 * If tag file contains unique tags (sorted index), we will load
2504 * it entirely into memory so we can resort it later for use with
2505 * chunked browsing.
2507 if (TAGCACHE_IS_SORTED(index_type))
2509 logf("loading tags...");
2510 for (i = 0; i < tch.entry_count; i++)
2512 struct tagfile_entry entry;
2513 int loc = lseek(fd, 0, SEEK_CUR);
2514 bool ret;
2516 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2518 logf("read error #7");
2519 close(fd);
2520 return -2;
2523 if (entry.tag_length >= (int)sizeof(buf))
2525 logf("too long tag #3");
2526 close(fd);
2527 return -2;
2530 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2532 logf("read error #8");
2533 close(fd);
2534 return -2;
2537 /* Skip deleted entries. */
2538 if (buf[0] == '\0')
2539 continue;
2542 * Save the tag and tag id in the memory buffer. Tag id
2543 * is saved so we can later reindex the master lookup
2544 * table when the index gets resorted.
2546 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2547 + commit_entry_count, entry.idx_id,
2548 TAGCACHE_IS_UNIQUE(index_type));
2549 if (!ret)
2551 close(fd);
2552 return -3;
2554 do_timed_yield();
2556 logf("done");
2558 else
2559 tempbufidx = tch.entry_count;
2561 else
2564 * Creating new index file to store the tags. No need to preload
2565 * anything whether the index type is sorted or not.
2567 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2568 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2569 if (fd < 0)
2571 logf("%s open fail", buf);
2572 return -2;
2575 tch.magic = TAGCACHE_MAGIC;
2576 tch.entry_count = 0;
2577 tch.datasize = 0;
2579 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2580 != sizeof(struct tagcache_header))
2582 logf("header write failed");
2583 close(fd);
2584 return -2;
2588 /* Loading the tag lookup file as "master file". */
2589 logf("Loading index file");
2590 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2592 if (masterfd < 0)
2594 logf("Creating new DB");
2595 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2597 if (masterfd < 0)
2599 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2600 close(fd);
2601 return -2;
2604 /* Write the header (write real values later). */
2605 memset(&tcmh, 0, sizeof(struct master_header));
2606 tcmh.tch = *h;
2607 tcmh.tch.entry_count = 0;
2608 tcmh.tch.datasize = 0;
2609 tcmh.dirty = true;
2610 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2611 init = true;
2612 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2614 else
2617 * Master file already exists so we need to process the current
2618 * file first.
2620 init = false;
2622 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2623 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2625 logf("header error");
2626 close(fd);
2627 close(masterfd);
2628 return -2;
2632 * If we reach end of the master file, we need to expand it to
2633 * hold new tags. If the current index is not sorted, we can
2634 * simply append new data to end of the file.
2635 * However, if the index is sorted, we need to update all tag
2636 * pointers in the master file for the current index.
2638 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2639 SEEK_CUR);
2640 if (masterfd_pos == filesize(masterfd))
2642 logf("appending...");
2643 init = true;
2648 * Load new unique tags in memory to be sorted later and added
2649 * to the master lookup file.
2651 if (TAGCACHE_IS_SORTED(index_type))
2653 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2654 /* h is the header of the temporary file containing new tags. */
2655 logf("inserting new tags...");
2656 for (i = 0; i < h->entry_count; i++)
2658 struct temp_file_entry entry;
2660 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2661 sizeof(struct temp_file_entry))
2663 logf("read fail #3");
2664 error = true;
2665 goto error_exit;
2668 /* Read data. */
2669 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2671 logf("too long entry!");
2672 error = true;
2673 goto error_exit;
2676 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2677 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2678 entry.tag_length[index_type])
2680 logf("read fail #4");
2681 error = true;
2682 goto error_exit;
2685 if (TAGCACHE_IS_UNIQUE(index_type))
2686 error = !tempbuf_insert(buf, i, -1, true);
2687 else
2688 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2690 if (error)
2692 logf("insert error");
2693 goto error_exit;
2696 /* Skip to next. */
2697 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2698 entry.tag_length[index_type], SEEK_CUR);
2699 do_timed_yield();
2701 logf("done");
2703 /* Sort the buffer data and write it to the index file. */
2704 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2706 * We need to truncate the index file now. There can be junk left
2707 * at the end of file (however, we _should_ always follow the
2708 * entry_count and don't crash with that).
2710 ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2712 i = tempbuf_sort(fd);
2713 if (i < 0)
2714 goto error_exit;
2715 logf("sorted %d tags", i);
2718 * Now update all indexes in the master lookup file.
2720 logf("updating indices...");
2721 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2722 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2724 int j;
2725 int loc = lseek(masterfd, 0, SEEK_CUR);
2727 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2729 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2730 != (int)sizeof(struct index_entry)*idxbuf_pos)
2732 logf("read fail #5");
2733 error = true;
2734 goto error_exit ;
2736 lseek(masterfd, loc, SEEK_SET);
2738 for (j = 0; j < idxbuf_pos; j++)
2740 if (idxbuf[j].flag & FLAG_DELETED)
2742 /* We can just ignore deleted entries. */
2743 // idxbuf[j].tag_seek[index_type] = 0;
2744 continue;
2747 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2748 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2749 + commit_entry_count);
2751 if (idxbuf[j].tag_seek[index_type] < 0)
2753 logf("update error: %ld/%d/%ld",
2754 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2755 error = true;
2756 goto error_exit;
2759 do_timed_yield();
2762 /* Write back the updated index. */
2763 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2764 index_entry_ec, tc_stat.econ) !=
2765 (int)sizeof(struct index_entry)*idxbuf_pos)
2767 logf("write fail");
2768 error = true;
2769 goto error_exit;
2772 logf("done");
2776 * Walk through the temporary file containing the new tags.
2778 // build_normal_index(h, tmpfd, masterfd, idx);
2779 logf("updating new indices...");
2780 lseek(masterfd, masterfd_pos, SEEK_SET);
2781 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2782 lseek(fd, 0, SEEK_END);
2783 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2785 int j;
2787 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2788 if (init)
2790 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2792 else
2794 int loc = lseek(masterfd, 0, SEEK_CUR);
2796 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2797 != (int)sizeof(struct index_entry)*idxbuf_pos)
2799 logf("read fail #6");
2800 error = true;
2801 break ;
2803 lseek(masterfd, loc, SEEK_SET);
2806 /* Read entry headers. */
2807 for (j = 0; j < idxbuf_pos; j++)
2809 if (!TAGCACHE_IS_SORTED(index_type))
2811 struct temp_file_entry entry;
2812 struct tagfile_entry fe;
2814 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2815 sizeof(struct temp_file_entry))
2817 logf("read fail #7");
2818 error = true;
2819 break ;
2822 /* Read data. */
2823 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2825 logf("too long entry!");
2826 logf("length=%d", entry.tag_length[index_type]);
2827 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2828 error = true;
2829 break ;
2832 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2833 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2834 entry.tag_length[index_type])
2836 logf("read fail #8");
2837 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2838 logf("length=0x%02x", entry.tag_length[index_type]);
2839 error = true;
2840 break ;
2843 /* Write to index file. */
2844 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2845 fe.tag_length = entry.tag_length[index_type];
2846 fe.idx_id = tcmh.tch.entry_count + i + j;
2847 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2848 write(fd, buf, fe.tag_length);
2849 tempbufidx++;
2851 /* Skip to next. */
2852 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2853 entry.tag_length[index_type], SEEK_CUR);
2855 else
2857 /* Locate the correct entry from the sorted array. */
2858 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2859 if (idxbuf[j].tag_seek[index_type] < 0)
2861 logf("entry not found (%d)", j);
2862 error = true;
2863 break ;
2868 /* Write index. */
2869 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2870 index_entry_ec, tc_stat.econ) !=
2871 (int)sizeof(struct index_entry)*idxbuf_pos)
2873 logf("tagcache: write fail #4");
2874 error = true;
2875 break ;
2878 do_timed_yield();
2880 logf("done");
2882 /* Finally write the header. */
2883 tch.magic = TAGCACHE_MAGIC;
2884 tch.entry_count = tempbufidx;
2885 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2886 lseek(fd, 0, SEEK_SET);
2887 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2889 if (index_type != tag_filename)
2890 h->datasize += tch.datasize;
2891 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2892 error_exit:
2894 close(fd);
2895 close(masterfd);
2897 if (error)
2898 return -2;
2900 return 1;
2903 static bool commit(void)
2905 struct tagcache_header tch;
2906 struct master_header tcmh;
2907 int i, len, rc;
2908 int tmpfd;
2909 int masterfd;
2910 #ifdef HAVE_DIRCACHE
2911 bool dircache_buffer_stolen = false;
2912 #endif
2913 bool local_allocation = false;
2915 logf("committing tagcache");
2917 while (write_lock)
2918 sleep(1);
2920 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2921 if (tmpfd < 0)
2923 logf("nothing to commit");
2924 return true;
2928 /* Load the header. */
2929 len = sizeof(struct tagcache_header);
2930 rc = read(tmpfd, &tch, len);
2932 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2934 logf("incorrect tmpheader");
2935 close(tmpfd);
2936 remove(TAGCACHE_FILE_TEMP);
2937 return false;
2940 if (tch.entry_count == 0)
2942 logf("nothing to commit");
2943 close(tmpfd);
2944 remove(TAGCACHE_FILE_TEMP);
2945 return true;
2948 /* Fully initialize existing headers (if any) before going further. */
2949 tc_stat.ready = check_all_headers();
2951 #ifdef HAVE_EEPROM_SETTINGS
2952 remove(TAGCACHE_STATEFILE);
2953 #endif
2955 /* At first be sure to unload the ramcache! */
2956 #ifdef HAVE_TC_RAMCACHE
2957 tc_stat.ramcache = false;
2958 #endif
2960 read_lock++;
2962 /* Try to steal every buffer we can :) */
2963 if (tempbuf_size == 0)
2964 local_allocation = true;
2966 #ifdef HAVE_DIRCACHE
2967 if (tempbuf_size == 0)
2969 /* Try to steal the dircache buffer. */
2970 tempbuf = dircache_steal_buffer(&tempbuf_size);
2971 tempbuf_size &= ~0x03;
2973 if (tempbuf_size > 0)
2975 dircache_buffer_stolen = true;
2978 #endif
2980 #ifdef HAVE_TC_RAMCACHE
2981 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2983 tempbuf = (char *)(hdr + 1);
2984 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2985 tempbuf_size &= ~0x03;
2987 #endif
2989 /* And finally fail if there are no buffers available. */
2990 if (tempbuf_size == 0)
2992 logf("delaying commit until next boot");
2993 tc_stat.commit_delayed = true;
2994 close(tmpfd);
2995 read_lock--;
2996 return false;
2999 logf("commit %ld entries...", tch.entry_count);
3001 /* Mark DB dirty so it will stay disabled if commit fails. */
3002 current_tcmh.dirty = true;
3003 update_master_header();
3005 /* Now create the index files. */
3006 tc_stat.commit_step = 0;
3007 tch.datasize = 0;
3008 tc_stat.commit_delayed = false;
3010 for (i = 0; i < TAG_COUNT; i++)
3012 int ret;
3014 if (TAGCACHE_IS_NUMERIC(i))
3015 continue;
3017 tc_stat.commit_step++;
3018 ret = build_index(i, &tch, tmpfd);
3019 if (ret <= 0)
3021 close(tmpfd);
3022 logf("tagcache failed init");
3023 if (ret == 0)
3024 tc_stat.commit_delayed = true;
3026 tc_stat.commit_step = 0;
3027 read_lock--;
3028 return false;
3032 if (!build_numeric_indices(&tch, tmpfd))
3034 logf("Failure to commit numeric indices");
3035 close(tmpfd);
3036 tc_stat.commit_step = 0;
3037 read_lock--;
3038 return false;
3041 close(tmpfd);
3042 remove(TAGCACHE_FILE_TEMP);
3044 tc_stat.commit_step = 0;
3046 /* Update the master index headers. */
3047 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3049 read_lock--;
3050 return false;
3053 tcmh.tch.entry_count += tch.entry_count;
3054 tcmh.tch.datasize = sizeof(struct master_header)
3055 + sizeof(struct index_entry) * tcmh.tch.entry_count
3056 + tch.datasize;
3057 tcmh.dirty = false;
3058 tcmh.commitid++;
3060 lseek(masterfd, 0, SEEK_SET);
3061 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3062 close(masterfd);
3064 logf("tagcache committed");
3065 tc_stat.ready = check_all_headers();
3066 tc_stat.readyvalid = true;
3068 if (local_allocation)
3070 tempbuf = NULL;
3071 tempbuf_size = 0;
3074 #ifdef HAVE_DIRCACHE
3075 /* Rebuild the dircache, if we stole the buffer. */
3076 if (dircache_buffer_stolen)
3077 dircache_build(0);
3078 #endif
3080 #ifdef HAVE_TC_RAMCACHE
3081 /* Reload tagcache. */
3082 if (tc_stat.ramcache_allocated > 0)
3083 tagcache_start_scan();
3084 #endif
3086 read_lock--;
3088 return true;
3091 static void allocate_tempbuf(void)
3093 /* Yeah, malloc would be really nice now :) */
3094 #ifdef __PCTOOL__
3095 tempbuf_size = 32*1024*1024;
3096 tempbuf = malloc(tempbuf_size);
3097 #else
3098 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3099 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3100 audiobuf += tempbuf_size;
3101 #endif
3104 static void free_tempbuf(void)
3106 if (tempbuf_size == 0)
3107 return ;
3109 #ifdef __PCTOOL__
3110 free(tempbuf);
3111 #else
3112 audiobuf -= tempbuf_size;
3113 #endif
3114 tempbuf = NULL;
3115 tempbuf_size = 0;
3118 #ifndef __PCTOOL__
3120 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3122 struct index_entry idx;
3124 if (!tc_stat.ready)
3125 return false;
3127 if (!TAGCACHE_IS_NUMERIC(tag))
3128 return false;
3130 if (!get_index(masterfd, idx_id, &idx, false))
3131 return false;
3133 idx.tag_seek[tag] = data;
3134 idx.flag |= FLAG_DIRTYNUM;
3136 return write_index(masterfd, idx_id, &idx);
3139 #if 0
3140 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3141 int tag, long data)
3143 struct master_header myhdr;
3145 if (tcs->masterfd < 0)
3147 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3148 return false;
3151 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3153 #endif
3155 static bool command_queue_is_full(void)
3157 int next;
3159 next = command_queue_widx + 1;
3160 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3161 next = 0;
3163 return (next == command_queue_ridx);
3166 static void command_queue_sync_callback(void *data)
3168 (void)data;
3169 struct master_header myhdr;
3170 int masterfd;
3172 mutex_lock(&command_queue_mutex);
3174 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3175 return;
3177 while (command_queue_ridx != command_queue_widx)
3179 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3181 switch (ce->command)
3183 case CMD_UPDATE_MASTER_HEADER:
3185 close(masterfd);
3186 update_master_header();
3188 /* Re-open the masterfd. */
3189 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3190 return;
3192 break;
3194 case CMD_UPDATE_NUMERIC:
3196 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3197 break;
3201 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3202 command_queue_ridx = 0;
3205 close(masterfd);
3207 tc_stat.queue_length = 0;
3208 mutex_unlock(&command_queue_mutex);
3211 static void run_command_queue(bool force)
3213 if (COMMAND_QUEUE_IS_EMPTY)
3214 return;
3216 if (force || command_queue_is_full())
3217 command_queue_sync_callback(NULL);
3218 else
3219 register_storage_idle_func(command_queue_sync_callback);
3222 static void queue_command(int cmd, long idx_id, int tag, long data)
3224 while (1)
3226 int next;
3228 mutex_lock(&command_queue_mutex);
3229 next = command_queue_widx + 1;
3230 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3231 next = 0;
3233 /* Make sure queue is not full. */
3234 if (next != command_queue_ridx)
3236 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3238 ce->command = cmd;
3239 ce->idx_id = idx_id;
3240 ce->tag = tag;
3241 ce->data = data;
3243 command_queue_widx = next;
3245 tc_stat.queue_length++;
3247 mutex_unlock(&command_queue_mutex);
3248 break;
3251 /* Queue is full, try again later... */
3252 mutex_unlock(&command_queue_mutex);
3253 sleep(1);
3257 long tagcache_increase_serial(void)
3259 long old;
3261 if (!tc_stat.ready)
3262 return -2;
3264 while (read_lock)
3265 sleep(1);
3267 old = current_tcmh.serial++;
3268 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3270 return old;
3273 void tagcache_update_numeric(int idx_id, int tag, long data)
3275 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3277 #endif /* !__PCTOOL__ */
3279 long tagcache_get_serial(void)
3281 return current_tcmh.serial;
3284 long tagcache_get_commitid(void)
3286 return current_tcmh.commitid;
3289 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3291 char buf[512];
3292 int i;
3294 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3295 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3297 if (*datastr == '\0')
3298 break;
3300 if (*datastr == '"' || *datastr == '\\')
3301 buf[i++] = '\\';
3303 buf[i] = *(datastr++);
3306 strcpy(&buf[i], "\" ");
3308 write(fd, buf, i + 2);
3310 return true;
3313 #ifndef __PCTOOL__
3315 static bool read_tag(char *dest, long size,
3316 const char *src, const char *tagstr)
3318 int pos;
3319 char current_tag[32];
3321 while (*src != '\0')
3323 /* Skip all whitespace */
3324 while (*src == ' ')
3325 src++;
3327 if (*src == '\0')
3328 break;
3330 pos = 0;
3331 /* Read in tag name */
3332 while (*src != '=' && *src != ' ')
3334 current_tag[pos] = *src;
3335 src++;
3336 pos++;
3338 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3339 return false;
3341 current_tag[pos] = '\0';
3343 /* Read in tag data */
3345 /* Find the start. */
3346 while (*src != '"' && *src != '\0')
3347 src++;
3349 if (*src == '\0' || *(++src) == '\0')
3350 return false;
3352 /* Read the data. */
3353 for (pos = 0; pos < size; pos++)
3355 if (*src == '\0')
3356 break;
3358 if (*src == '\\')
3360 dest[pos] = *(src+1);
3361 src += 2;
3362 continue;
3365 dest[pos] = *src;
3367 if (*src == '"')
3369 src++;
3370 break;
3373 if (*src == '\0')
3374 break;
3376 src++;
3378 dest[pos] = '\0';
3380 if (!strcasecmp(tagstr, current_tag))
3381 return true;
3384 return false;
3387 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3389 struct index_entry idx;
3390 char tag_data[TAG_MAXLEN+32];
3391 int idx_id;
3392 long masterfd = (long)parameters;
3393 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3394 tag_commitid };
3395 int i;
3396 (void)line_n;
3398 if (*buf == '#')
3399 return 0;
3401 logf("%d/%s", line_n, buf);
3402 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3404 logf("filename missing");
3405 logf("-> %s", buf);
3406 return 0;
3409 idx_id = find_index(tag_data);
3410 if (idx_id < 0)
3412 logf("entry not found");
3413 return 0;
3416 if (!get_index(masterfd, idx_id, &idx, false))
3418 logf("failed to retrieve index entry");
3419 return 0;
3422 /* Stop if tag has already been modified. */
3423 if (idx.flag & FLAG_DIRTYNUM)
3424 return 0;
3426 logf("import: %s", tag_data);
3428 idx.flag |= FLAG_DIRTYNUM;
3429 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3431 int data;
3433 if (!read_tag(tag_data, sizeof tag_data, buf,
3434 tagcache_tag_to_str(import_tags[i])))
3436 continue;
3439 data = atoi(tag_data);
3440 if (data < 0)
3441 continue;
3443 idx.tag_seek[import_tags[i]] = data;
3445 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3446 current_tcmh.serial = data + 1;
3447 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3448 current_tcmh.commitid = data + 1;
3451 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3454 bool tagcache_import_changelog(void)
3456 struct master_header myhdr;
3457 struct tagcache_header tch;
3458 int clfd;
3459 long masterfd;
3460 char buf[2048];
3462 if (!tc_stat.ready)
3463 return false;
3465 while (read_lock)
3466 sleep(1);
3468 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3469 if (clfd < 0)
3471 logf("failure to open changelog");
3472 return false;
3475 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3477 close(clfd);
3478 return false;
3481 write_lock++;
3483 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3485 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3486 parse_changelog_line);
3488 close(clfd);
3489 close(masterfd);
3491 if (filenametag_fd >= 0)
3493 close(filenametag_fd);
3494 filenametag_fd = -1;
3497 write_lock--;
3499 update_master_header();
3501 return true;
3504 #endif /* !__PCTOOL__ */
3506 bool tagcache_create_changelog(struct tagcache_search *tcs)
3508 struct master_header myhdr;
3509 struct index_entry idx;
3510 char buf[TAG_MAXLEN+32];
3511 char temp[32];
3512 int clfd;
3513 int i, j;
3515 if (!tc_stat.ready)
3516 return false;
3518 if (!tagcache_search(tcs, tag_filename))
3519 return false;
3521 /* Initialize the changelog */
3522 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3523 if (clfd < 0)
3525 logf("failure to open changelog");
3526 return false;
3529 if (tcs->masterfd < 0)
3531 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3532 return false;
3534 else
3536 lseek(tcs->masterfd, 0, SEEK_SET);
3537 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3540 write(clfd, "## Changelog version 1\n", 23);
3542 for (i = 0; i < myhdr.tch.entry_count; i++)
3544 if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3546 logf("read error #9");
3547 tagcache_search_finish(tcs);
3548 close(clfd);
3549 return false;
3552 /* Skip until the entry found has been modified. */
3553 if (! (idx.flag & FLAG_DIRTYNUM) )
3554 continue;
3556 /* Skip deleted entries too. */
3557 if (idx.flag & FLAG_DELETED)
3558 continue;
3560 /* Now retrieve all tags. */
3561 for (j = 0; j < TAG_COUNT; j++)
3563 if (TAGCACHE_IS_NUMERIC(j))
3565 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3566 write_tag(clfd, tagcache_tag_to_str(j), temp);
3567 continue;
3570 tcs->type = j;
3571 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3572 write_tag(clfd, tagcache_tag_to_str(j), buf);
3575 write(clfd, "\n", 1);
3576 do_timed_yield();
3579 close(clfd);
3581 tagcache_search_finish(tcs);
3583 return true;
3586 static bool delete_entry(long idx_id)
3588 int fd = -1;
3589 int masterfd = -1;
3590 int tag, i;
3591 struct index_entry idx, myidx;
3592 struct master_header myhdr;
3593 char buf[TAG_MAXLEN+32];
3594 int in_use[TAG_COUNT];
3596 logf("delete_entry(): %ld", idx_id);
3598 #ifdef HAVE_TC_RAMCACHE
3599 /* At first mark the entry removed from ram cache. */
3600 if (tc_stat.ramcache)
3601 hdr->indices[idx_id].flag |= FLAG_DELETED;
3602 #endif
3604 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3605 return false;
3607 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3608 if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3610 logf("delete_entry(): read error");
3611 goto cleanup;
3614 if (myidx.flag & FLAG_DELETED)
3616 logf("delete_entry(): already deleted!");
3617 goto cleanup;
3620 myidx.flag |= FLAG_DELETED;
3621 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3622 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3624 logf("delete_entry(): write_error #1");
3625 goto cleanup;
3628 /* Now check which tags are no longer in use (if any) */
3629 for (tag = 0; tag < TAG_COUNT; tag++)
3630 in_use[tag] = 0;
3632 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3633 for (i = 0; i < myhdr.tch.entry_count; i++)
3635 struct index_entry *idxp;
3637 #ifdef HAVE_TC_RAMCACHE
3638 /* Use RAM DB if available for greater speed */
3639 if (tc_stat.ramcache)
3640 idxp = &hdr->indices[i];
3641 else
3642 #endif
3644 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3646 logf("delete_entry(): read error #2");
3647 goto cleanup;
3649 idxp = &idx;
3652 if (idxp->flag & FLAG_DELETED)
3653 continue;
3655 for (tag = 0; tag < TAG_COUNT; tag++)
3657 if (TAGCACHE_IS_NUMERIC(tag))
3658 continue;
3660 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3661 in_use[tag]++;
3665 /* Now delete all tags no longer in use. */
3666 for (tag = 0; tag < TAG_COUNT; tag++)
3668 struct tagcache_header tch;
3669 int oldseek = myidx.tag_seek[tag];
3671 if (TAGCACHE_IS_NUMERIC(tag))
3672 continue;
3674 /**
3675 * Replace tag seek with a hash value of the field string data.
3676 * That way runtime statistics of moved or altered files can be
3677 * resurrected.
3679 #ifdef HAVE_TC_RAMCACHE
3680 if (tc_stat.ramcache && tag != tag_filename)
3682 struct tagfile_entry *tfe;
3683 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3685 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3686 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3687 myidx.tag_seek[tag] = *seek;
3689 else
3690 #endif
3692 struct tagfile_entry tfe;
3694 /* Open the index file, which contains the tag names. */
3695 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3696 goto cleanup;
3698 /* Skip the header block */
3699 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3700 if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3702 logf("delete_entry(): read error #3");
3703 goto cleanup;
3706 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3708 logf("delete_entry(): read error #4");
3709 goto cleanup;
3712 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3715 if (in_use[tag])
3717 logf("in use: %d/%d", tag, in_use[tag]);
3718 if (fd >= 0)
3720 close(fd);
3721 fd = -1;
3723 continue;
3726 #ifdef HAVE_TC_RAMCACHE
3727 /* Delete from ram. */
3728 if (tc_stat.ramcache && tag != tag_filename)
3730 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3731 tagentry->tag_data[0] = '\0';
3733 #endif
3735 /* Open the index file, which contains the tag names. */
3736 if (fd < 0)
3738 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3739 goto cleanup;
3742 /* Skip the header block */
3743 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3745 /* Debug, print 10 first characters of the tag
3746 read(fd, buf, 10);
3747 buf[10]='\0';
3748 logf("TAG:%s", buf);
3749 lseek(fd, -10, SEEK_CUR);
3752 /* Write first data byte in tag as \0 */
3753 write(fd, "", 1);
3755 /* Now tag data has been removed */
3756 close(fd);
3757 fd = -1;
3760 /* Write index entry back into master index. */
3761 lseek(masterfd, sizeof(struct master_header) +
3762 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3763 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3765 logf("delete_entry(): write_error #2");
3766 goto cleanup;
3769 close(masterfd);
3771 return true;
3773 cleanup:
3774 if (fd >= 0)
3775 close(fd);
3776 if (masterfd >= 0)
3777 close(masterfd);
3779 return false;
3782 #ifndef __PCTOOL__
3784 * Returns true if there is an event waiting in the queue
3785 * that requires the current operation to be aborted.
3787 static bool check_event_queue(void)
3789 struct queue_event ev;
3791 if(!queue_peek(&tagcache_queue, &ev))
3792 return false;
3794 switch (ev.id)
3796 case Q_STOP_SCAN:
3797 case SYS_POWEROFF:
3798 case SYS_USB_CONNECTED:
3799 return true;
3802 return false;
3804 #endif
3806 #ifdef HAVE_TC_RAMCACHE
3807 static bool allocate_tagcache(void)
3809 struct master_header tcmh;
3810 int fd;
3812 /* Load the header. */
3813 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3815 hdr = NULL;
3816 return false;
3819 close(fd);
3821 /**
3822 * Now calculate the required cache size plus
3823 * some extra space for alignment fixes.
3825 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3826 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3827 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3828 memset(hdr, 0, sizeof(struct ramcache_header));
3829 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3830 hdr->indices = (struct index_entry *)(hdr + 1);
3831 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3833 return true;
3836 # ifdef HAVE_EEPROM_SETTINGS
3837 static bool tagcache_dumpload(void)
3839 struct statefile_header shdr;
3840 int fd, rc;
3841 long offpos;
3842 int i;
3844 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3845 if (fd < 0)
3847 logf("no tagcache statedump");
3848 return false;
3851 /* Check the statefile memory placement */
3852 hdr = buffer_alloc(0);
3853 rc = read(fd, &shdr, sizeof(struct statefile_header));
3854 if (rc != sizeof(struct statefile_header)
3855 /* || (long)hdr != (long)shdr.hdr */)
3857 logf("incorrect statefile");
3858 hdr = NULL;
3859 close(fd);
3860 return false;
3863 offpos = (long)hdr - (long)shdr.hdr;
3865 /* Lets allocate real memory and load it */
3866 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3867 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3868 close(fd);
3870 if (rc != shdr.tc_stat.ramcache_allocated)
3872 logf("read failure!");
3873 hdr = NULL;
3874 return false;
3877 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3879 /* Now fix the pointers */
3880 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3881 for (i = 0; i < TAG_COUNT; i++)
3882 hdr->tags[i] += offpos;
3884 return true;
3887 static bool tagcache_dumpsave(void)
3889 struct statefile_header shdr;
3890 int fd;
3892 if (!tc_stat.ramcache)
3893 return false;
3895 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3896 if (fd < 0)
3898 logf("failed to create a statedump");
3899 return false;
3902 /* Create the header */
3903 shdr.hdr = hdr;
3904 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3905 write(fd, &shdr, sizeof(struct statefile_header));
3907 /* And dump the data too */
3908 write(fd, hdr, tc_stat.ramcache_allocated);
3909 close(fd);
3911 return true;
3913 # endif
3915 static bool load_tagcache(void)
3917 struct tagcache_header *tch;
3918 long bytesleft = tc_stat.ramcache_allocated;
3919 struct index_entry *idx;
3920 int rc, fd;
3921 char *p;
3922 int i, tag;
3924 # ifdef HAVE_DIRCACHE
3925 while (dircache_is_initializing())
3926 sleep(1);
3928 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3929 # endif
3931 logf("loading tagcache to ram...");
3933 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3934 if (fd < 0)
3936 logf("tagcache open failed");
3937 return false;
3940 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3941 != sizeof(struct master_header)
3942 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3944 logf("incorrect header");
3945 return false;
3948 idx = hdr->indices;
3950 /* Load the master index table. */
3951 for (i = 0; i < hdr->h.tch.entry_count; i++)
3953 rc = ecread_index_entry(fd, idx);
3954 if (rc != sizeof(struct index_entry))
3956 logf("read error #10");
3957 close(fd);
3958 return false;
3961 bytesleft -= sizeof(struct index_entry);
3962 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3964 logf("too big tagcache.");
3965 close(fd);
3966 return false;
3969 idx++;
3972 close(fd);
3974 /* Load the tags. */
3975 p = (char *)idx;
3976 for (tag = 0; tag < TAG_COUNT; tag++)
3978 struct tagfile_entry *fe;
3979 char buf[TAG_MAXLEN+32];
3981 if (TAGCACHE_IS_NUMERIC(tag))
3982 continue ;
3984 //p = ((void *)p+1);
3985 p = (char *)((long)p & ~0x03) + 0x04;
3986 hdr->tags[tag] = p;
3988 /* Check the header. */
3989 tch = (struct tagcache_header *)p;
3990 p += sizeof(struct tagcache_header);
3992 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3993 return false;
3995 for (hdr->entry_count[tag] = 0;
3996 hdr->entry_count[tag] < tch->entry_count;
3997 hdr->entry_count[tag]++)
3999 long pos;
4001 if (do_timed_yield())
4003 /* Abort if we got a critical event in queue */
4004 if (check_event_queue())
4005 return false;
4008 fe = (struct tagfile_entry *)p;
4009 pos = lseek(fd, 0, SEEK_CUR);
4010 rc = ecread_tagfile_entry(fd, fe);
4011 if (rc != sizeof(struct tagfile_entry))
4013 /* End of lookup table. */
4014 logf("read error #11");
4015 close(fd);
4016 return false;
4019 /* We have a special handling for the filename tags. */
4020 if (tag == tag_filename)
4022 # ifdef HAVE_DIRCACHE
4023 const struct dircache_entry *dc;
4024 # endif
4026 // FIXME: This is wrong!
4027 // idx = &hdr->indices[hdr->entry_count[i]];
4028 idx = &hdr->indices[fe->idx_id];
4030 if (fe->tag_length >= (long)sizeof(buf)-1)
4032 read(fd, buf, 10);
4033 buf[10] = '\0';
4034 logf("TAG:%s", buf);
4035 logf("too long filename");
4036 close(fd);
4037 return false;
4040 rc = read(fd, buf, fe->tag_length);
4041 if (rc != fe->tag_length)
4043 logf("read error #12");
4044 close(fd);
4045 return false;
4048 /* Check if the entry has already been removed */
4049 if (idx->flag & FLAG_DELETED)
4050 continue;
4052 /* This flag must not be used yet. */
4053 if (idx->flag & FLAG_DIRCACHE)
4055 logf("internal error!");
4056 close(fd);
4057 return false;
4060 if (idx->tag_seek[tag] != pos)
4062 logf("corrupt data structures!");
4063 close(fd);
4064 return false;
4067 # ifdef HAVE_DIRCACHE
4068 if (dircache_is_enabled())
4070 dc = dircache_get_entry_ptr(buf);
4071 if (dc == NULL)
4073 logf("Entry no longer valid.");
4074 logf("-> %s", buf);
4075 if (global_settings.tagcache_autoupdate)
4076 delete_entry(fe->idx_id);
4077 continue ;
4080 idx->flag |= FLAG_DIRCACHE;
4081 idx->tag_seek[tag_filename] = (long)dc;
4083 else
4084 # endif
4086 /* This will be very slow unless dircache is enabled
4087 or target is flash based, but do it anyway for
4088 consistency. */
4089 /* Check if entry has been removed. */
4090 if (global_settings.tagcache_autoupdate)
4092 if (!file_exists(buf))
4094 logf("Entry no longer valid.");
4095 logf("-> %s", buf);
4096 delete_entry(fe->idx_id);
4097 continue;
4102 continue ;
4105 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4106 if (bytesleft < 0)
4108 logf("too big tagcache #2");
4109 logf("tl: %ld", fe->tag_length);
4110 logf("bl: %ld", bytesleft);
4111 close(fd);
4112 return false;
4115 p = fe->tag_data;
4116 rc = read(fd, fe->tag_data, fe->tag_length);
4117 p += rc;
4119 if (rc != fe->tag_length)
4121 logf("read error #13");
4122 logf("rc=0x%04x", rc); // 0x431
4123 logf("len=0x%04lx", fe->tag_length); // 0x4000
4124 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4125 logf("tag=0x%02x", tag); // 0x00
4126 close(fd);
4127 return false;
4130 close(fd);
4133 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4134 logf("tagcache loaded into ram!");
4136 return true;
4138 #endif /* HAVE_TC_RAMCACHE */
4140 static bool check_deleted_files(void)
4142 int fd;
4143 char buf[TAG_MAXLEN+32];
4144 struct tagfile_entry tfe;
4146 logf("reverse scan...");
4147 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4148 fd = open(buf, O_RDONLY);
4150 if (fd < 0)
4152 logf("%s open fail", buf);
4153 return false;
4156 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4157 while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4158 #ifndef __PCTOOL__
4159 && !check_event_queue()
4160 #endif
4163 if (tfe.tag_length >= (long)sizeof(buf)-1)
4165 logf("too long tag");
4166 close(fd);
4167 return false;
4170 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4172 logf("read error #14");
4173 close(fd);
4174 return false;
4177 /* Check if the file has already deleted from the db. */
4178 if (*buf == '\0')
4179 continue;
4181 /* Now check if the file exists. */
4182 if (!file_exists(buf))
4184 logf("Entry no longer valid.");
4185 logf("-> %s / %ld", buf, tfe.tag_length);
4186 delete_entry(tfe.idx_id);
4190 close(fd);
4192 logf("done");
4194 return true;
4198 /* Note that this function must not be inlined, otherwise the whole point
4199 * of having the code in a separate function is lost.
4201 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4202 int *ignore, int *unignore)
4204 char newpath[MAX_PATH];
4206 /* check for a database.ignore file */
4207 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4208 *ignore = file_exists(newpath);
4209 /* check for a database.unignore file */
4210 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4211 *unignore = file_exists(newpath);
4214 static struct search_roots_ll {
4215 const char *path;
4216 struct search_roots_ll * next;
4217 } roots_ll;
4219 #ifdef APPLICATION
4221 * This adds a path to the search roots, possibly during traveling through
4222 * the filesystem. It only adds if the path is not inside an already existing
4223 * search root.
4225 * Returns true if it added the path to the search roots
4227 * Windows 2000 and greater supports symlinks, but they don't provide
4228 * realpath() or readlink(), and symlinks are rarely used on them so
4229 * ignore this for windows for now
4231 static bool add_search_root(const char *name)
4233 (void)name;
4234 #ifndef WIN32
4235 struct search_roots_ll *this, *prev = NULL;
4236 char target[MAX_PATH];
4237 /* Okay, realpath() is almost completely broken on android
4239 * It doesn't accept NULL for resolved_name to dynamically allocate
4240 * the resulting path; and it assumes resolved_name to be PATH_MAX
4241 * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4242 * and blindly writes to the end if it
4244 * therefore use sufficiently large static storage here
4245 * Note that PATH_MAX != MAX_PATH
4247 static char abs_target[PATH_MAX];
4248 ssize_t len;
4250 len = readlink(name, target, sizeof(target));
4251 if (len < 0)
4252 return false;
4254 target[len] = '\0';
4255 if (realpath(target, abs_target) == NULL)
4256 return false;
4258 for(this = &roots_ll; this; prev = this, this = this->next)
4260 size_t root_len = strlen(this->path);
4261 /* check if the link target is inside of an existing search root
4262 * don't add if target is inside, we'll scan it later */
4263 if (!strncmp(this->path, abs_target, root_len))
4264 return false;
4267 if (prev)
4269 size_t len = strlen(abs_target) + 1; /* count \0 */
4270 this = malloc(sizeof(struct search_roots_ll) + len );
4271 if (!this || len > MAX_PATH)
4273 logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4274 free(this);
4275 prev->next = NULL;
4276 return false;
4278 this->path = ((char*)this) + sizeof(struct search_roots_ll);
4279 strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4280 this->next = NULL;
4281 prev->next = this;
4282 logf("Added %s to the search roots\n", abs_target);
4283 return true;
4285 #endif
4286 return false;
4289 static int free_search_roots(struct search_roots_ll * start)
4291 int ret = 0;
4292 if (start->next)
4294 ret += free_search_roots(start->next);
4295 ret += sizeof(struct search_roots_ll);
4296 free(start->next);
4298 return ret;
4300 #else /* native, simulator */
4301 #define add_search_root(a) do {} while(0)
4302 #define free_search_roots(a) do {} while(0)
4303 #endif
4305 static bool check_dir(const char *dirname, int add_files)
4307 DIR *dir;
4308 int len;
4309 int success = false;
4310 int ignore, unignore;
4312 dir = opendir(dirname);
4313 if (!dir)
4315 logf("tagcache: opendir(%s) failed", dirname);
4316 return false;
4318 /* check for a database.ignore and database.unignore */
4319 check_ignore(dirname, &ignore, &unignore);
4321 /* don't do anything if both ignore and unignore are there */
4322 if (ignore != unignore)
4323 add_files = unignore;
4325 /* Recursively scan the dir. */
4326 #ifdef __PCTOOL__
4327 while (1)
4328 #else
4329 while (!check_event_queue())
4330 #endif
4332 struct dirent *entry = readdir(dir);
4333 if (entry == NULL)
4335 success = true;
4336 break;
4339 if (!strcmp((char *)entry->d_name, ".") ||
4340 !strcmp((char *)entry->d_name, ".."))
4341 continue;
4343 struct dirinfo info = dir_get_info(dir, entry);
4345 yield();
4347 len = strlen(curpath);
4348 /* don't add an extra / for curpath == / */
4349 if (len <= 1) len = 0;
4350 snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4352 processed_dir_count++;
4353 if (info.attribute & ATTR_DIRECTORY)
4354 { /* don't follow symlinks to dirs, but try to add it as a search root
4355 * this makes able to avoid looping in recursive symlinks */
4356 if (info.attribute & ATTR_LINK)
4357 add_search_root(curpath);
4358 else
4359 check_dir(curpath, add_files);
4361 else if (add_files)
4363 tc_stat.curentry = curpath;
4365 /* Add a new entry to the temporary db file. */
4366 add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4367 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4368 , dir->internal_entry
4369 #endif
4372 /* Wait until current path for debug screen is read and unset. */
4373 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4374 yield();
4376 tc_stat.curentry = NULL;
4379 curpath[len] = '\0';
4382 closedir(dir);
4384 return success;
4387 void tagcache_screensync_event(void)
4389 tc_stat.curentry = NULL;
4392 void tagcache_screensync_enable(bool state)
4394 tc_stat.syncscreen = state;
4397 void tagcache_build(const char *path)
4399 struct tagcache_header header;
4400 bool ret;
4402 curpath[0] = '\0';
4403 data_size = 0;
4404 total_entry_count = 0;
4405 processed_dir_count = 0;
4407 #ifdef HAVE_DIRCACHE
4408 while (dircache_is_initializing())
4409 sleep(1);
4410 #endif
4412 logf("updating tagcache");
4414 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4415 if (cachefd >= 0)
4417 logf("skipping, cache already waiting for commit");
4418 close(cachefd);
4419 return ;
4422 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4423 if (cachefd < 0)
4425 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4426 return ;
4429 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4431 cpu_boost(true);
4433 logf("Scanning files...");
4434 /* Scan for new files. */
4435 memset(&header, 0, sizeof(struct tagcache_header));
4436 write(cachefd, &header, sizeof(struct tagcache_header));
4438 ret = true;
4439 roots_ll.path = path;
4440 roots_ll.next = NULL;
4441 struct search_roots_ll * this;
4442 /* check_dir might add new roots */
4443 for(this = &roots_ll; this; this = this->next)
4445 strcpy(curpath, this->path);
4446 ret = ret && check_dir(this->path, true);
4448 if (roots_ll.next)
4449 free_search_roots(roots_ll.next);
4451 /* Write the header. */
4452 header.magic = TAGCACHE_MAGIC;
4453 header.datasize = data_size;
4454 header.entry_count = total_entry_count;
4455 lseek(cachefd, 0, SEEK_SET);
4456 write(cachefd, &header, sizeof(struct tagcache_header));
4457 close(cachefd);
4459 if (filenametag_fd >= 0)
4461 close(filenametag_fd);
4462 filenametag_fd = -1;
4465 if (!ret)
4467 logf("Aborted.");
4468 cpu_boost(false);
4469 return ;
4472 /* Commit changes to the database. */
4473 #ifdef __PCTOOL__
4474 allocate_tempbuf();
4475 #endif
4476 if (commit())
4478 remove(TAGCACHE_FILE_TEMP);
4479 logf("tagcache built!");
4481 #ifdef __PCTOOL__
4482 free_tempbuf();
4483 #endif
4485 #ifdef HAVE_TC_RAMCACHE
4486 if (hdr)
4488 /* Import runtime statistics if we just initialized the db. */
4489 if (hdr->h.serial == 0)
4490 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4492 #endif
4494 cpu_boost(false);
4497 #ifdef HAVE_TC_RAMCACHE
4498 static void load_ramcache(void)
4500 if (!hdr)
4501 return ;
4503 cpu_boost(true);
4505 /* At first we should load the cache (if exists). */
4506 tc_stat.ramcache = load_tagcache();
4508 if (!tc_stat.ramcache)
4510 /* If loading failed, it must indicate some problem with the db
4511 * so disable it entirely to prevent further issues. */
4512 tc_stat.ready = false;
4513 hdr = NULL;
4516 cpu_boost(false);
4519 void tagcache_unload_ramcache(void)
4521 tc_stat.ramcache = false;
4522 /* Just to make sure there is no statefile present. */
4523 // remove(TAGCACHE_STATEFILE);
4525 #endif
4527 #ifndef __PCTOOL__
4528 static void tagcache_thread(void)
4530 struct queue_event ev;
4531 bool check_done = false;
4533 /* If the previous cache build/update was interrupted, commit
4534 * the changes first in foreground. */
4535 cpu_boost(true);
4536 allocate_tempbuf();
4537 commit();
4538 free_tempbuf();
4540 #ifdef HAVE_TC_RAMCACHE
4541 # ifdef HAVE_EEPROM_SETTINGS
4542 if (firmware_settings.initialized && firmware_settings.disk_clean
4543 && global_settings.tagcache_ram)
4545 check_done = tagcache_dumpload();
4548 remove(TAGCACHE_STATEFILE);
4549 # endif
4551 /* Allocate space for the tagcache if found on disk. */
4552 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4553 allocate_tagcache();
4554 #endif
4556 cpu_boost(false);
4557 tc_stat.initialized = true;
4559 /* Don't delay bootup with the header check but do it on background. */
4560 if (!tc_stat.ready)
4562 sleep(HZ);
4563 tc_stat.ready = check_all_headers();
4564 tc_stat.readyvalid = true;
4567 while (1)
4569 run_command_queue(false);
4571 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4573 switch (ev.id)
4575 case Q_IMPORT_CHANGELOG:
4576 tagcache_import_changelog();
4577 break;
4579 case Q_REBUILD:
4580 remove_files();
4581 remove(TAGCACHE_FILE_TEMP);
4582 tagcache_build("/");
4583 break;
4585 case Q_UPDATE:
4586 tagcache_build("/");
4587 #ifdef HAVE_TC_RAMCACHE
4588 load_ramcache();
4589 #endif
4590 check_deleted_files();
4591 break ;
4593 case Q_START_SCAN:
4594 check_done = false;
4595 case SYS_TIMEOUT:
4596 if (check_done || !tc_stat.ready)
4597 break ;
4599 #ifdef HAVE_TC_RAMCACHE
4600 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4602 load_ramcache();
4603 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4604 tagcache_build("/");
4606 else
4607 #endif
4608 if (global_settings.tagcache_autoupdate)
4610 tagcache_build("/");
4612 /* This will be very slow unless dircache is enabled
4613 or target is flash based, but do it anyway for
4614 consistency. */
4615 check_deleted_files();
4618 logf("tagcache check done");
4620 check_done = true;
4621 break ;
4623 case Q_STOP_SCAN:
4624 break ;
4626 case SYS_POWEROFF:
4627 break ;
4629 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4630 case SYS_USB_CONNECTED:
4631 logf("USB: TagCache");
4632 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4633 usb_wait_for_disconnect(&tagcache_queue);
4634 break ;
4635 #endif
4640 bool tagcache_prepare_shutdown(void)
4642 if (tagcache_get_commit_step() > 0)
4643 return false;
4645 tagcache_stop_scan();
4646 while (read_lock || write_lock)
4647 sleep(1);
4649 return true;
4652 void tagcache_shutdown(void)
4654 /* Flush the command queue. */
4655 run_command_queue(true);
4657 #ifdef HAVE_EEPROM_SETTINGS
4658 if (tc_stat.ramcache)
4659 tagcache_dumpsave();
4660 #endif
4663 static int get_progress(void)
4665 int total_count = -1;
4667 #ifdef HAVE_DIRCACHE
4668 if (dircache_is_enabled())
4670 total_count = dircache_get_entry_count();
4672 else
4673 #endif
4674 #ifdef HAVE_TC_RAMCACHE
4676 if (hdr && tc_stat.ramcache)
4677 total_count = hdr->h.tch.entry_count;
4679 #endif
4681 if (total_count < 0)
4682 return -1;
4684 return processed_dir_count * 100 / total_count;
4687 struct tagcache_stat* tagcache_get_stat(void)
4689 tc_stat.progress = get_progress();
4690 tc_stat.processed_entries = processed_dir_count;
4692 return &tc_stat;
4695 void tagcache_start_scan(void)
4697 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4700 bool tagcache_update(void)
4702 if (!tc_stat.ready)
4703 return false;
4705 queue_post(&tagcache_queue, Q_UPDATE, 0);
4706 return false;
4709 bool tagcache_rebuild()
4711 queue_post(&tagcache_queue, Q_REBUILD, 0);
4712 return false;
4715 void tagcache_stop_scan(void)
4717 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4720 #ifdef HAVE_TC_RAMCACHE
4721 bool tagcache_is_ramcache(void)
4723 return tc_stat.ramcache;
4725 #endif
4727 #endif /* !__PCTOOL__ */
4730 void tagcache_init(void)
4732 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4733 memset(&current_tcmh, 0, sizeof(struct master_header));
4734 filenametag_fd = -1;
4735 write_lock = read_lock = 0;
4737 #ifndef __PCTOOL__
4738 mutex_init(&command_queue_mutex);
4739 queue_init(&tagcache_queue, true);
4740 create_thread(tagcache_thread, tagcache_stack,
4741 sizeof(tagcache_stack), 0, tagcache_thread_name
4742 IF_PRIO(, PRIORITY_BACKGROUND)
4743 IF_COP(, CPU));
4744 #else
4745 tc_stat.initialized = true;
4746 allocate_tempbuf();
4747 commit();
4748 free_tempbuf();
4749 tc_stat.ready = check_all_headers();
4750 #endif
4753 #ifdef __PCTOOL__
4754 void tagcache_reverse_scan(void)
4756 logf("Checking for deleted files");
4757 check_deleted_files();
4759 #endif
4761 bool tagcache_is_initialized(void)
4763 return tc_stat.initialized;
4765 bool tagcache_is_fully_initialized(void)
4767 return tc_stat.readyvalid;
4769 bool tagcache_is_usable(void)
4771 return tc_stat.initialized && tc_stat.ready;
4773 int tagcache_get_commit_step(void)
4775 return tc_stat.commit_step;
4777 int tagcache_get_max_commit_step(void)
4779 return (int)(SORTED_TAGS_COUNT)+1;