1 /***************************************************************************
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 * Copyright (C) 2005 by Miika Pekkarinen
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
20 ****************************************************************************/
25 * ----------x---------x------------------x-----
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
58 /*#define LOGF_ENABLE*/
64 #include <unistd.h> /* readlink() */
65 #include <limits.h> /* PATH_MAX */
68 #include "ata_idle_notify.h"
73 #include "string-extra.h"
77 #include "core_alloc.h"
82 #include "filefuncs.h"
88 #include "eeprom_settings.h"
92 #define yield() do { } while(0)
93 #define sim_sleep(timeout) do { } while(0)
94 #define do_timed_yield() do { } while(0)
98 /* Tag Cache thread. */
99 static struct event_queue tagcache_queue SHAREDBSS_ATTR
;
100 static long tagcache_stack
[(DEFAULT_STACK_SIZE
+ 0x4000)/sizeof(long)];
101 static const char tagcache_thread_name
[] = "tagcache";
104 /* Previous path when scanning directory tree recursively. */
105 static char curpath
[TAG_MAXLEN
+32];
107 /* Used when removing duplicates. */
108 static char *tempbuf
; /* Allocated when needed. */
109 static long tempbufidx
; /* Current location in buffer. */
110 static size_t tempbuf_size
; /* Buffer size (TEMPBUF_SIZE). */
111 static long tempbuf_left
; /* Buffer space left. */
112 static long tempbuf_pos
;
114 #define SORTED_TAGS_COUNT 8
115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
118 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
119 /* Tags we want to get sorted (loaded to the tempbuf). */
120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
121 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
122 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
126 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
127 (1LU << tag_albumartist) | (1LU << tag_grouping))
129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
130 static const char *tags_str
[] = { "artist", "album", "genre", "title",
131 "filename", "composer", "comment", "albumartist", "grouping", "year",
132 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
133 "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
135 /* Status information of the tagcache. */
136 static struct tagcache_stat tc_stat
;
138 /* Queue commands. */
139 enum tagcache_queue
{
146 /* Internal tagcache command queue. */
147 CMD_UPDATE_MASTER_HEADER
,
151 struct tagcache_command_entry
{
159 static struct tagcache_command_entry command_queue
[TAGCACHE_COMMAND_QUEUE_LENGTH
];
160 static volatile int command_queue_widx
= 0;
161 static volatile int command_queue_ridx
= 0;
162 static struct mutex command_queue_mutex SHAREDBSS_ATTR
;
165 /* Tag database structures. */
167 /* Variable-length tag entry in tag files. */
168 struct tagfile_entry
{
169 int32_t tag_length
; /* Length of the data in bytes including '\0' */
170 int32_t idx_id
; /* Corresponding entry location in index file of not unique tags */
171 char tag_data
[0]; /* Begin of the tag data */
174 /* Fixed-size tag entry in master db index. */
176 int32_t tag_seek
[TAG_COUNT
]; /* Location of tag data or numeric tag data */
177 int32_t flag
; /* Status flags */
180 /* Header is the same in every file. */
181 struct tagcache_header
{
182 int32_t magic
; /* Header version number */
183 int32_t datasize
; /* Data size in bytes */
184 int32_t entry_count
; /* Number of entries in this file */
187 struct master_header
{
188 struct tagcache_header tch
;
189 int32_t serial
; /* Increasing counting number */
190 int32_t commitid
; /* Number of commits so far */
194 /* For the endianess correction */
195 static const char * const tagfile_entry_ec
= "ll";
197 Note: This should be (1 + TAG_COUNT) amount of l's.
199 static const char * const index_entry_ec
= "llllllllllllllllllllll";
201 static const char * const tagcache_header_ec
= "lll";
202 static const char * const master_header_ec
= "llllll";
204 static struct master_header current_tcmh
;
206 #ifdef HAVE_TC_RAMCACHE
207 /* Header is created when loading database to ram. */
208 struct ramcache_header
{
209 char *tags
[TAG_COUNT
]; /* Tag file content (not including filename tag) */
210 int entry_count
[TAG_COUNT
]; /* Number of entries in the indices. */
211 struct index_entry indices
[0]; /* Master index file content */
214 # ifdef HAVE_EEPROM_SETTINGS
215 struct statefile_header
{
216 int32_t magic
; /* Statefile version number */
217 struct master_header mh
; /* Header from the master index */
218 struct ramcache_header
*hdr
; /* Old load address of hdr for relocation */
219 struct tagcache_stat tc_stat
;
223 /* Pointer to allocated ramcache_header */
224 static struct ramcache_header
*ramcache_hdr
;
225 /* lock entity to temporarily prevent ramcache_hdr from moving */
226 static int move_lock
;
230 * Full tag entries stored in a temporary file waiting
231 * for commit to the cache. */
232 struct temp_file_entry
{
233 long tag_offset
[TAG_COUNT
];
234 short tag_length
[TAG_COUNT
];
240 struct tempbuf_id_list
{
242 struct tempbuf_id_list
*next
;
245 struct tempbuf_searchidx
{
249 struct tempbuf_id_list idlist
;
252 /* Lookup buffer for fixing messed up index while after sorting. */
253 static long commit_entry_count
;
254 static long lookup_buffer_depth
;
255 static struct tempbuf_searchidx
**lookup
;
257 /* Used when building the temporary file. */
258 static int cachefd
= -1, filenametag_fd
;
259 static int total_entry_count
= 0;
260 static int data_size
= 0;
261 static int processed_dir_count
;
263 /* Thread safe locking */
264 static volatile int write_lock
;
265 static volatile int read_lock
;
267 static bool delete_entry(long idx_id
);
269 const char* tagcache_tag_to_str(int tag
)
271 return tags_str
[tag
];
274 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
275 static ssize_t
ecread_tagfile_entry(int fd
, struct tagfile_entry
*buf
)
277 return ecread(fd
, buf
, 1, tagfile_entry_ec
, tc_stat
.econ
);
280 static ssize_t
ecread_index_entry(int fd
, struct index_entry
*buf
)
282 return ecread(fd
, buf
, 1, index_entry_ec
, tc_stat
.econ
);
285 static ssize_t
ecwrite_index_entry(int fd
, struct index_entry
*buf
)
287 return ecwrite(fd
, buf
, 1, index_entry_ec
, tc_stat
.econ
);
292 * Returns true if specified flag is still present, i.e., dircache
293 * has not been reloaded.
295 static bool is_dircache_intact(void)
297 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE
);
301 static int open_tag_fd(struct tagcache_header
*hdr
, int tag
, bool write
)
307 if (TAGCACHE_IS_NUMERIC(tag
) || tag
< 0 || tag
>= TAG_COUNT
)
310 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag
);
312 fd
= open(buf
, write
? O_RDWR
: O_RDONLY
);
315 logf("tag file open failed: tag=%d write=%d file=%s", tag
, write
, buf
);
316 tc_stat
.ready
= false;
320 /* Check the header. */
321 rc
= ecread(fd
, hdr
, 1, tagcache_header_ec
, tc_stat
.econ
);
322 if (hdr
->magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct tagcache_header
))
324 logf("header error");
325 tc_stat
.ready
= false;
333 static int open_master_fd(struct master_header
*hdr
, bool write
)
338 fd
= open(TAGCACHE_FILE_MASTER
, write
? O_RDWR
: O_RDONLY
);
341 logf("master file open failed for R/W");
342 tc_stat
.ready
= false;
346 tc_stat
.econ
= false;
348 /* Check the header. */
349 rc
= read(fd
, hdr
, sizeof(struct master_header
));
350 if (hdr
->tch
.magic
== TAGCACHE_MAGIC
&& rc
== sizeof(struct master_header
))
356 /* Trying to read again, this time with endianess correction enabled. */
357 lseek(fd
, 0, SEEK_SET
);
359 rc
= ecread(fd
, hdr
, 1, master_header_ec
, true);
360 if (hdr
->tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct master_header
))
362 logf("header error");
363 tc_stat
.ready
= false;
374 static bool do_timed_yield(void)
376 /* Sorting can lock up for quite a while, so yield occasionally */
377 static long wakeup_tick
= 0;
378 if (TIME_AFTER(current_tick
, wakeup_tick
))
380 wakeup_tick
= current_tick
+ (HZ
/4);
388 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
389 /* find the ramcache entry corresponding to the file indicated by
390 * filename and dc (it's corresponding dircache id). */
391 static long find_entry_ram(const char *filename
, int dc
)
393 static long last_pos
= 0;
396 /* Check if tagcache is loaded into ram. */
397 if (!tc_stat
.ramcache
|| !is_dircache_intact())
401 dc
= dircache_get_entry_id(filename
);
405 logf("tagcache: file not found.");
416 for (; i
< current_tcmh
.tch
.entry_count
; i
++)
418 if (ramcache_hdr
->indices
[i
].tag_seek
[tag_filename
] == dc
)
420 last_pos
= MAX(0, i
- 3);
437 static long find_entry_disk(const char *filename_raw
, bool localfd
)
439 struct tagcache_header tch
;
440 static long last_pos
= -1;
441 long pos_history
[POS_HISTORY_COUNT
];
442 long pos_history_idx
= 0;
444 struct tagfile_entry tfe
;
446 char buf
[TAG_MAXLEN
+32];
450 const char *filename
= filename_raw
;
452 char pathbuf
[PATH_MAX
]; /* Note: Don't use MAX_PATH here, it's too small */
453 if (realpath(filename
, pathbuf
) == pathbuf
)
461 if (fd
< 0 || localfd
)
464 if ( (fd
= open_tag_fd(&tch
, tag_filename
, false)) < 0)
471 lseek(fd
, last_pos
, SEEK_SET
);
473 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
477 pos
= lseek(fd
, 0, SEEK_CUR
);
478 for (i
= pos_history_idx
-1; i
>= 0; i
--)
479 pos_history
[i
+1] = pos_history
[i
];
480 pos_history
[0] = pos
;
482 if (ecread_tagfile_entry(fd
, &tfe
)
483 != sizeof(struct tagfile_entry
))
488 if (tfe
.tag_length
>= (long)sizeof(buf
))
490 logf("too long tag #1");
498 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
500 logf("read error #2");
508 if (!strcmp(filename
, buf
))
510 last_pos
= pos_history
[pos_history_idx
];
515 if (pos_history_idx
< POS_HISTORY_COUNT
- 1)
529 if (fd
!= filenametag_fd
|| localfd
)
534 if (fd
!= filenametag_fd
|| localfd
)
540 static int find_index(const char *filename
)
544 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
545 idx_id
= find_entry_ram(filename
, -1);
549 idx_id
= find_entry_disk(filename
, true);
554 bool tagcache_find_index(struct tagcache_search
*tcs
, const char *filename
)
561 idx_id
= find_index(filename
);
565 if (!tagcache_search(tcs
, tag_filename
))
568 tcs
->entry_count
= 0;
569 tcs
->idx_id
= idx_id
;
574 static bool get_index(int masterfd
, int idxid
,
575 struct index_entry
*idx
, bool use_ram
)
577 bool localfd
= false;
581 logf("Incorrect idxid: %d", idxid
);
585 #ifdef HAVE_TC_RAMCACHE
586 if (tc_stat
.ramcache
&& use_ram
)
588 if (ramcache_hdr
->indices
[idxid
].flag
& FLAG_DELETED
)
591 # ifdef HAVE_DIRCACHE
592 if (!(ramcache_hdr
->indices
[idxid
].flag
& FLAG_DIRCACHE
)
593 || is_dircache_intact())
596 memcpy(idx
, &ramcache_hdr
->indices
[idxid
], sizeof(struct index_entry
));
606 struct master_header tcmh
;
609 masterfd
= open_master_fd(&tcmh
, false);
614 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
615 + sizeof(struct master_header
), SEEK_SET
);
616 if (ecread_index_entry(masterfd
, idx
)
617 != sizeof(struct index_entry
))
619 logf("read error #3");
629 if (idx
->flag
& FLAG_DELETED
)
637 static bool write_index(int masterfd
, int idxid
, struct index_entry
*idx
)
639 /* We need to exclude all memory only flags & tags when writing to disk. */
640 if (idx
->flag
& FLAG_DIRCACHE
)
642 logf("memory only flags!");
646 #ifdef HAVE_TC_RAMCACHE
647 /* Only update numeric data. Writing the whole index to RAM by memcpy
648 * destroys dircache pointers!
650 if (tc_stat
.ramcache
)
653 struct index_entry
*idx_ram
= &ramcache_hdr
->indices
[idxid
];
655 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
657 if (TAGCACHE_IS_NUMERIC(tag
))
659 idx_ram
->tag_seek
[tag
] = idx
->tag_seek
[tag
];
663 /* Don't touch the dircache flag or attributes. */
664 idx_ram
->flag
= (idx
->flag
& 0x0000ffff)
665 | (idx_ram
->flag
& (0xffff0000 | FLAG_DIRCACHE
));
669 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
670 + sizeof(struct master_header
), SEEK_SET
);
671 if (ecwrite_index_entry(masterfd
, idx
) != sizeof(struct index_entry
))
673 logf("write error #3");
674 logf("idxid: %d", idxid
);
681 #endif /* !__PCTOOL__ */
683 static bool open_files(struct tagcache_search
*tcs
, int tag
)
685 if (tcs
->idxfd
[tag
] < 0)
689 snprintf(fn
, sizeof fn
, TAGCACHE_FILE_INDEX
, tag
);
690 tcs
->idxfd
[tag
] = open(fn
, O_RDONLY
);
693 if (tcs
->idxfd
[tag
] < 0)
695 logf("File not open!");
702 static bool retrieve(struct tagcache_search
*tcs
, struct index_entry
*idx
,
703 int tag
, char *buf
, long size
)
705 struct tagfile_entry tfe
;
710 if (TAGCACHE_IS_NUMERIC(tag
))
713 seek
= idx
->tag_seek
[tag
];
716 logf("Retrieve failed");
720 #ifdef HAVE_TC_RAMCACHE
723 struct tagfile_entry
*ep
;
725 # ifdef HAVE_DIRCACHE
726 if (tag
== tag_filename
&& (idx
->flag
& FLAG_DIRCACHE
))
728 /* for tag_filename, seek is a dircache index */
729 if (is_dircache_intact())
731 dircache_copy_path(seek
, buf
, size
);
736 /* The seek is useless now, there's nothing we can return. */
737 logf("retrieve: dircache gone, cannot read file name");
738 tagcache_unload_ramcache();
739 // XXX do this when there's a way to not trigger an
740 // update before reloading:
741 // tagcache_start_scan();
747 if (tag
!= tag_filename
)
749 ep
= (struct tagfile_entry
*)&ramcache_hdr
->tags
[tag
][seek
];
750 strlcpy(buf
, ep
->tag_data
, size
);
757 if (!open_files(tcs
, tag
))
760 lseek(tcs
->idxfd
[tag
], seek
, SEEK_SET
);
761 if (ecread_tagfile_entry(tcs
->idxfd
[tag
], &tfe
)
762 != sizeof(struct tagfile_entry
))
764 logf("read error #5");
768 if (tfe
.tag_length
>= size
)
770 logf("too small buffer");
774 if (read(tcs
->idxfd
[tag
], buf
, tfe
.tag_length
) !=
777 logf("read error #6");
781 buf
[tfe
.tag_length
] = '\0';
786 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
788 static long find_tag(int tag
, int idx_id
, const struct index_entry
*idx
)
791 if (! COMMAND_QUEUE_IS_EMPTY
&& TAGCACHE_IS_NUMERIC(tag
))
793 /* Attempt to find tag data through store-to-load forwarding in
797 mutex_lock(&command_queue_mutex
);
799 int ridx
= command_queue_widx
;
801 while (ridx
!= command_queue_ridx
)
804 ridx
= TAGCACHE_COMMAND_QUEUE_LENGTH
- 1;
806 if (command_queue
[ridx
].command
== CMD_UPDATE_NUMERIC
807 && command_queue
[ridx
].idx_id
== idx_id
808 && command_queue
[ridx
].tag
== tag
)
810 result
= command_queue
[ridx
].data
;
815 mutex_unlock(&command_queue_mutex
);
820 "Recovered tag %d value %lX from write queue",
827 return idx
->tag_seek
[tag
];
831 static long check_virtual_tags(int tag
, int idx_id
,
832 const struct index_entry
*idx
)
838 case tag_virt_length_sec
:
839 data
= (find_tag(tag_length
, idx_id
, idx
)/1000) % 60;
842 case tag_virt_length_min
:
843 data
= (find_tag(tag_length
, idx_id
, idx
)/1000) / 60;
846 case tag_virt_playtime_sec
:
847 data
= (find_tag(tag_playtime
, idx_id
, idx
)/1000) % 60;
850 case tag_virt_playtime_min
:
851 data
= (find_tag(tag_playtime
, idx_id
, idx
)/1000) / 60;
854 case tag_virt_autoscore
:
855 if (find_tag(tag_length
, idx_id
, idx
) == 0
856 || find_tag(tag_playcount
, idx_id
, idx
) == 0)
862 /* A straight calculus gives:
863 autoscore = 100 * playtime / length / playcout (1)
864 Now, consider the euclidian division of playtime by length:
865 playtime = alpha * length + beta
869 autoscore = 100 * (alpha / playcout + beta / length / playcount)
870 Both terms should be small enough to avoid any overflow
872 data
= 100 * (find_tag(tag_playtime
, idx_id
, idx
)
873 / find_tag(tag_length
, idx_id
, idx
))
874 + (100 * (find_tag(tag_playtime
, idx_id
, idx
)
875 % find_tag(tag_length
, idx_id
, idx
)))
876 / find_tag(tag_length
, idx_id
, idx
);
877 data
/= find_tag(tag_playcount
, idx_id
, idx
);
881 /* How many commits before the file has been added to the DB. */
882 case tag_virt_entryage
:
883 data
= current_tcmh
.commitid
884 - find_tag(tag_commitid
, idx_id
, idx
) - 1;
887 case tag_virt_basename
:
888 tag
= tag_filename
; /* return filename; caller handles basename */
892 data
= find_tag(tag
, idx_id
, idx
);
898 long tagcache_get_numeric(const struct tagcache_search
*tcs
, int tag
)
900 struct index_entry idx
;
905 if (!TAGCACHE_IS_NUMERIC(tag
))
908 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
911 return check_virtual_tags(tag
, tcs
->idx_id
, &idx
);
914 inline static bool str_ends_with(const char *str1
, const char *str2
)
916 int str_len
= strlen(str1
);
917 int clause_len
= strlen(str2
);
919 if (clause_len
> str_len
)
922 return !strcasecmp(&str1
[str_len
- clause_len
], str2
);
925 inline static bool str_oneof(const char *str
, const char *list
)
928 int l
, len
= strlen(str
);
932 sep
= strchr(list
, '|');
933 l
= sep
? (long)sep
- (long)list
: (int)strlen(list
);
934 if ((l
==len
) && !strncasecmp(str
, list
, len
))
936 list
+= sep
? l
+ 1 : l
;
942 static bool check_against_clause(long numeric
, const char *str
,
943 const struct tagcache_search_clause
*clause
)
947 switch (clause
->type
)
950 return numeric
== clause
->numeric_data
;
952 return numeric
!= clause
->numeric_data
;
954 return numeric
> clause
->numeric_data
;
956 return numeric
>= clause
->numeric_data
;
958 return numeric
< clause
->numeric_data
;
960 return numeric
<= clause
->numeric_data
;
962 logf("Incorrect numeric tag: %d", clause
->type
);
967 switch (clause
->type
)
970 return !strcasecmp(clause
->str
, str
);
972 return strcasecmp(clause
->str
, str
);
974 return 0>strcasecmp(clause
->str
, str
);
976 return 0>=strcasecmp(clause
->str
, str
);
978 return 0<strcasecmp(clause
->str
, str
);
980 return 0<=strcasecmp(clause
->str
, str
);
981 case clause_contains
:
982 return (strcasestr(str
, clause
->str
) != NULL
);
983 case clause_not_contains
:
984 return (strcasestr(str
, clause
->str
) == NULL
);
985 case clause_begins_with
:
986 return (strcasestr(str
, clause
->str
) == str
);
987 case clause_not_begins_with
:
988 return (strcasestr(str
, clause
->str
) != str
);
989 case clause_ends_with
:
990 return str_ends_with(str
, clause
->str
);
991 case clause_not_ends_with
:
992 return !str_ends_with(str
, clause
->str
);
994 return str_oneof(str
, clause
->str
);
997 logf("Incorrect tag: %d", clause
->type
);
1004 static bool check_clauses(struct tagcache_search
*tcs
,
1005 struct index_entry
*idx
,
1006 struct tagcache_search_clause
**clauses
, int count
)
1010 /* Go through all conditional clauses. */
1011 for (i
= 0; i
< count
; i
++)
1016 struct tagcache_search_clause
*clause
= clauses
[i
];
1018 if (clause
->type
== clause_logical_or
)
1019 break; /* all conditions before logical-or satisfied --
1020 stop processing clauses */
1022 seek
= check_virtual_tags(clause
->tag
, tcs
->idx_id
, idx
);
1024 #ifdef HAVE_TC_RAMCACHE
1027 struct tagfile_entry
*tfe
;
1029 if (!TAGCACHE_IS_NUMERIC(clause
->tag
))
1031 if (clause
->tag
== tag_filename
1032 || clause
->tag
== tag_virt_basename
)
1034 retrieve(tcs
, idx
, tag_filename
, buf
, sizeof buf
);
1038 tfe
= (struct tagfile_entry
*)
1039 &ramcache_hdr
->tags
[clause
->tag
][seek
];
1040 /* str points to movable data, but no locking required here,
1041 * as no yield() is following */
1042 str
= tfe
->tag_data
;
1049 struct tagfile_entry tfe
;
1051 if (!TAGCACHE_IS_NUMERIC(clause
->tag
))
1053 int tag
= clause
->tag
;
1054 if (tag
== tag_virt_basename
)
1057 int fd
= tcs
->idxfd
[tag
];
1058 lseek(fd
, seek
, SEEK_SET
);
1059 ecread_tagfile_entry(fd
, &tfe
);
1060 if (tfe
.tag_length
>= (int)sizeof(buf
))
1062 logf("Too long tag read!");
1066 read(fd
, str
, tfe
.tag_length
);
1067 str
[tfe
.tag_length
] = '\0';
1069 /* Check if entry has been deleted. */
1075 if (clause
->tag
== tag_virt_basename
)
1077 char *basename
= strrchr(str
, '/');
1082 if (!check_against_clause(seek
, str
, clause
))
1084 /* Clause failed -- try finding a logical-or clause */
1087 if (clauses
[i
]->type
== clause_logical_or
)
1091 if (i
< count
) /* Found logical-or? */
1092 continue; /* Check clauses after logical-or */
1101 bool tagcache_check_clauses(struct tagcache_search
*tcs
,
1102 struct tagcache_search_clause
**clause
, int count
)
1104 struct index_entry idx
;
1109 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
1112 return check_clauses(tcs
, &idx
, clause
, count
);
1115 static bool add_uniqbuf(struct tagcache_search
*tcs
, unsigned long id
)
1119 /* If uniq buffer is not defined we must return true for search to work. */
1120 if (tcs
->unique_list
== NULL
|| (!TAGCACHE_IS_UNIQUE(tcs
->type
)
1121 && !TAGCACHE_IS_NUMERIC(tcs
->type
)))
1126 for (i
= 0; i
< tcs
->unique_list_count
; i
++)
1128 /* Return false if entry is found. */
1129 if (tcs
->unique_list
[i
] == id
)
1133 if (tcs
->unique_list_count
< tcs
->unique_list_capacity
)
1135 tcs
->unique_list
[i
] = id
;
1136 tcs
->unique_list_count
++;
1142 static bool build_lookup_list(struct tagcache_search
*tcs
)
1144 struct index_entry entry
;
1147 tcs
->seek_list_count
= 0;
1149 #ifdef HAVE_TC_RAMCACHE
1151 # ifdef HAVE_DIRCACHE
1152 && (tcs
->type
!= tag_filename
|| is_dircache_intact())
1156 move_lock
++; /* lock because below makes a pointer to movable data */
1157 for (i
= tcs
->seek_pos
; i
< current_tcmh
.tch
.entry_count
; i
++)
1159 struct tagcache_seeklist_entry
*seeklist
;
1160 /* idx points to movable data, don't yield or reload */
1161 struct index_entry
*idx
= &ramcache_hdr
->indices
[i
];
1162 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
1165 /* Skip deleted files. */
1166 if (idx
->flag
& FLAG_DELETED
)
1169 /* Go through all filters.. */
1170 for (j
= 0; j
< tcs
->filter_count
; j
++)
1172 if (idx
->tag_seek
[tcs
->filter_tag
[j
]] != tcs
->filter_seek
[j
])
1178 if (j
< tcs
->filter_count
)
1181 /* Check for conditions. */
1182 if (!check_clauses(tcs
, idx
, tcs
->clause
, tcs
->clause_count
))
1184 /* Add to the seek list if not already in uniq buffer (doesn't yield)*/
1185 if (!add_uniqbuf(tcs
, idx
->tag_seek
[tcs
->type
]))
1189 seeklist
= &tcs
->seeklist
[tcs
->seek_list_count
];
1190 seeklist
->seek
= idx
->tag_seek
[tcs
->type
];
1191 seeklist
->flag
= idx
->flag
;
1192 seeklist
->idx_id
= i
;
1193 tcs
->seek_list_count
++;
1199 return tcs
->seek_list_count
> 0;
1203 if (tcs
->masterfd
< 0)
1205 struct master_header tcmh
;
1206 tcs
->masterfd
= open_master_fd(&tcmh
, false);
1209 lseek(tcs
->masterfd
, tcs
->seek_pos
* sizeof(struct index_entry
) +
1210 sizeof(struct master_header
), SEEK_SET
);
1212 while (ecread_index_entry(tcs
->masterfd
, &entry
)
1213 == sizeof(struct index_entry
))
1215 struct tagcache_seeklist_entry
*seeklist
;
1217 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
1223 /* Check if entry has been deleted. */
1224 if (entry
.flag
& FLAG_DELETED
)
1227 /* Go through all filters.. */
1228 for (j
= 0; j
< tcs
->filter_count
; j
++)
1230 if (entry
.tag_seek
[tcs
->filter_tag
[j
]] != tcs
->filter_seek
[j
])
1234 if (j
< tcs
->filter_count
)
1237 /* Check for conditions. */
1238 if (!check_clauses(tcs
, &entry
, tcs
->clause
, tcs
->clause_count
))
1241 /* Add to the seek list if not already in uniq buffer. */
1242 if (!add_uniqbuf(tcs
, entry
.tag_seek
[tcs
->type
]))
1246 seeklist
= &tcs
->seeklist
[tcs
->seek_list_count
];
1247 seeklist
->seek
= entry
.tag_seek
[tcs
->type
];
1248 seeklist
->flag
= entry
.flag
;
1249 seeklist
->idx_id
= i
;
1250 tcs
->seek_list_count
++;
1255 return tcs
->seek_list_count
> 0;
1259 static void remove_files(void)
1264 tc_stat
.ready
= false;
1265 tc_stat
.ramcache
= false;
1266 tc_stat
.econ
= false;
1267 remove(TAGCACHE_FILE_MASTER
);
1268 for (i
= 0; i
< TAG_COUNT
; i
++)
1270 if (TAGCACHE_IS_NUMERIC(i
))
1273 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, i
);
1279 static bool check_all_headers(void)
1281 struct master_header myhdr
;
1282 struct tagcache_header tch
;
1286 if ( (fd
= open_master_fd(&myhdr
, false)) < 0)
1292 logf("tagcache is dirty!");
1296 memcpy(¤t_tcmh
, &myhdr
, sizeof(struct master_header
));
1298 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
1300 if (TAGCACHE_IS_NUMERIC(tag
))
1303 if ( (fd
= open_tag_fd(&tch
, tag
, false)) < 0)
1312 bool tagcache_search(struct tagcache_search
*tcs
, int tag
)
1314 struct tagcache_header tag_hdr
;
1315 struct master_header master_hdr
;
1321 memset(tcs
, 0, sizeof(struct tagcache_search
));
1322 if (tc_stat
.commit_step
> 0 || !tc_stat
.ready
)
1325 tcs
->position
= sizeof(struct tagcache_header
);
1328 tcs
->list_position
= 0;
1329 tcs
->seek_list_count
= 0;
1330 tcs
->filter_count
= 0;
1333 for (i
= 0; i
< TAG_COUNT
; i
++)
1336 #ifndef HAVE_TC_RAMCACHE
1337 tcs
->ramsearch
= false;
1339 tcs
->ramsearch
= tc_stat
.ramcache
;
1342 tcs
->entry_count
= ramcache_hdr
->entry_count
[tcs
->type
];
1347 /* Always open as R/W so we can pass tcs to functions that modify data also
1348 * without failing. */
1349 tcs
->masterfd
= open_master_fd(&master_hdr
, true);
1350 if (tcs
->masterfd
< 0)
1353 if (!TAGCACHE_IS_NUMERIC(tcs
->type
))
1355 tcs
->idxfd
[tcs
->type
] = open_tag_fd(&tag_hdr
, tcs
->type
, false);
1356 if (tcs
->idxfd
[tcs
->type
] < 0)
1359 tcs
->entry_count
= tag_hdr
.entry_count
;
1363 tcs
->entry_count
= master_hdr
.tch
.entry_count
;
1368 tcs
->initialized
= true;
1374 void tagcache_search_set_uniqbuf(struct tagcache_search
*tcs
,
1375 void *buffer
, long length
)
1377 tcs
->unique_list
= (unsigned long *)buffer
;
1378 tcs
->unique_list_capacity
= length
/ sizeof(*tcs
->unique_list
);
1379 tcs
->unique_list_count
= 0;
1382 bool tagcache_search_add_filter(struct tagcache_search
*tcs
,
1385 if (tcs
->filter_count
== TAGCACHE_MAX_FILTERS
)
1388 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag
))
1391 tcs
->filter_tag
[tcs
->filter_count
] = tag
;
1392 tcs
->filter_seek
[tcs
->filter_count
] = seek
;
1393 tcs
->filter_count
++;
1398 bool tagcache_search_add_clause(struct tagcache_search
*tcs
,
1399 struct tagcache_search_clause
*clause
)
1403 if (tcs
->clause_count
>= TAGCACHE_MAX_CLAUSES
)
1405 logf("Too many clauses");
1409 if (clause
->type
!= clause_logical_or
)
1411 /* Check if there is already a similar filter in present (filters are
1412 * much faster than clauses).
1414 for (i
= 0; i
< tcs
->filter_count
; i
++)
1416 if (tcs
->filter_tag
[i
] == clause
->tag
)
1420 if (!TAGCACHE_IS_NUMERIC(clause
->tag
) && tcs
->idxfd
[clause
->tag
] < 0)
1424 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, clause
->tag
);
1425 tcs
->idxfd
[clause
->tag
] = open(buf
, O_RDONLY
);
1429 tcs
->clause
[tcs
->clause_count
] = clause
;
1430 tcs
->clause_count
++;
1435 static bool get_next(struct tagcache_search
*tcs
)
1437 static char buf
[TAG_MAXLEN
+32];
1438 struct tagfile_entry entry
;
1439 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1443 if (!tcs
->valid
|| !tc_stat
.ready
)
1446 if (tcs
->idxfd
[tcs
->type
] < 0 && !TAGCACHE_IS_NUMERIC(tcs
->type
)
1447 #ifdef HAVE_TC_RAMCACHE
1453 /* Relative fetch. */
1454 if (tcs
->filter_count
> 0 || tcs
->clause_count
> 0
1455 || TAGCACHE_IS_NUMERIC(tcs
->type
)
1456 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1457 /* We need to retrieve flag status for dircache. */
1458 || (tcs
->ramsearch
&& tcs
->type
== tag_filename
)
1462 struct tagcache_seeklist_entry
*seeklist
;
1464 /* Check for end of list. */
1465 if (tcs
->list_position
== tcs
->seek_list_count
)
1467 tcs
->list_position
= 0;
1469 /* Try to fetch more. */
1470 if (!build_lookup_list(tcs
))
1477 seeklist
= &tcs
->seeklist
[tcs
->list_position
];
1478 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1479 flag
= seeklist
->flag
;
1481 tcs
->position
= seeklist
->seek
;
1482 tcs
->idx_id
= seeklist
->idx_id
;
1483 tcs
->list_position
++;
1487 if (tcs
->entry_count
== 0)
1496 tcs
->result_seek
= tcs
->position
;
1498 if (TAGCACHE_IS_NUMERIC(tcs
->type
))
1500 snprintf(buf
, sizeof(buf
), "%ld", tcs
->position
);
1502 tcs
->result_len
= strlen(buf
) + 1;
1507 #ifdef HAVE_TC_RAMCACHE
1511 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1512 if (tcs
->type
== tag_filename
&& (flag
& FLAG_DIRCACHE
))
1514 if (is_dircache_intact())
1516 size_t len
= dircache_copy_path(tcs
->position
, buf
, sizeof buf
);
1517 tcs
->result_len
= len
+ 1;
1519 tcs
->ramresult
= false;
1525 /* The seek is useless now, there's nothing we can return. */
1526 logf("get_next: dircache gone, cannot read file name");
1527 tagcache_unload_ramcache();
1528 // XXX do this when there's a way to not trigger an
1529 // update before reloading:
1530 // tagcache_start_scan();
1537 if (tcs
->type
!= tag_filename
)
1539 struct tagfile_entry
*ep
;
1541 ep
= (struct tagfile_entry
*)&ramcache_hdr
->tags
[tcs
->type
][tcs
->position
];
1542 /* don't return ep->tag_data directly as it may move */
1543 tcs
->result_len
= strlcpy(buf
, ep
->tag_data
, sizeof(buf
)) + 1;
1545 tcs
->idx_id
= ep
->idx_id
;
1546 tcs
->ramresult
= false; /* was true before we copied to buf too */
1548 /* Increase position for the next run. This may get overwritten. */
1549 tcs
->position
+= sizeof(struct tagfile_entry
) + ep
->tag_length
;
1556 if (!open_files(tcs
, tcs
->type
))
1562 /* Seek stream to the correct position and continue to direct fetch. */
1563 lseek(tcs
->idxfd
[tcs
->type
], tcs
->position
, SEEK_SET
);
1565 if (ecread_tagfile_entry(tcs
->idxfd
[tcs
->type
], &entry
) != sizeof(struct tagfile_entry
))
1567 logf("read error #5");
1572 if (entry
.tag_length
> (long)sizeof(buf
))
1575 logf("too long tag #2");
1576 logf("P:%lX/%lX", tcs
->position
, entry
.tag_length
);
1580 if (read(tcs
->idxfd
[tcs
->type
], buf
, entry
.tag_length
) != entry
.tag_length
)
1583 logf("read error #4");
1588 Update the position for the next read (this may be overridden
1589 if filters or clauses are being used).
1591 tcs
->position
+= sizeof(struct tagfile_entry
) + entry
.tag_length
;
1593 tcs
->result_len
= strlen(tcs
->result
) + 1;
1594 tcs
->idx_id
= entry
.idx_id
;
1595 tcs
->ramresult
= false;
1600 bool tagcache_get_next(struct tagcache_search
*tcs
)
1602 while (get_next(tcs
))
1604 if (tcs
->result_len
> 1)
1611 bool tagcache_retrieve(struct tagcache_search
*tcs
, int idxid
,
1612 int tag
, char *buf
, long size
)
1614 struct index_entry idx
;
1617 if (!get_index(tcs
->masterfd
, idxid
, &idx
, true))
1620 return retrieve(tcs
, &idx
, tag
, buf
, size
);
1623 static bool update_master_header(void)
1625 struct master_header myhdr
;
1631 if ( (fd
= open_master_fd(&myhdr
, true)) < 0)
1634 myhdr
.serial
= current_tcmh
.serial
;
1635 myhdr
.commitid
= current_tcmh
.commitid
;
1636 myhdr
.dirty
= current_tcmh
.dirty
;
1639 lseek(fd
, 0, SEEK_SET
);
1640 ecwrite(fd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
1646 void tagcache_search_finish(struct tagcache_search
*tcs
)
1650 if (!tcs
->initialized
)
1653 if (tcs
->masterfd
>= 0)
1655 close(tcs
->masterfd
);
1659 for (i
= 0; i
< TAG_COUNT
; i
++)
1661 if (tcs
->idxfd
[i
] >= 0)
1663 close(tcs
->idxfd
[i
]);
1668 tcs
->ramsearch
= false;
1670 tcs
->initialized
= 0;
1675 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1676 static struct tagfile_entry
*get_tag(const struct index_entry
*entry
, int tag
)
1678 return (struct tagfile_entry
*)&ramcache_hdr
->tags
[tag
][entry
->tag_seek
[tag
]];
1681 static long get_tag_numeric(const struct index_entry
*entry
, int tag
, int idx_id
)
1683 return check_virtual_tags(tag
, idx_id
, entry
);
1686 static char* get_tag_string(const struct index_entry
*entry
, int tag
)
1688 char* s
= get_tag(entry
, tag
)->tag_data
;
1689 return strcmp(s
, UNTAGGED
) ? s
: NULL
;
1692 bool tagcache_fill_tags(struct mp3entry
*id3
, const char *filename
)
1694 struct index_entry
*entry
;
1697 if (!tc_stat
.ready
|| !tc_stat
.ramcache
)
1700 /* Find the corresponding entry in tagcache. */
1701 idx_id
= find_entry_ram(filename
, -1);
1705 entry
= &ramcache_hdr
->indices
[idx_id
];
1707 memset(id3
, 0, sizeof(struct mp3entry
));
1708 char* buf
= id3
->id3v2buf
;
1709 ssize_t remaining
= sizeof(id3
->id3v2buf
);
1711 /* this macro sets id3 strings by copying to the id3v2buf */
1712 #define SET(x, y) do \
1714 if (remaining > 0) \
1716 x = NULL; /* initialize with null if tag doesn't exist */ \
1717 char* src = get_tag_string(entry, y); \
1721 size_t len = strlcpy(buf, src, remaining) +1; \
1722 buf += len; remaining -= len; \
1728 SET(id3
->title
, tag_title
);
1729 SET(id3
->artist
, tag_artist
);
1730 SET(id3
->album
, tag_album
);
1731 SET(id3
->genre_string
, tag_genre
);
1732 SET(id3
->composer
, tag_composer
);
1733 SET(id3
->comment
, tag_comment
);
1734 SET(id3
->albumartist
, tag_albumartist
);
1735 SET(id3
->grouping
, tag_grouping
);
1737 id3
->length
= get_tag_numeric(entry
, tag_length
, idx_id
);
1738 id3
->playcount
= get_tag_numeric(entry
, tag_playcount
, idx_id
);
1739 id3
->rating
= get_tag_numeric(entry
, tag_rating
, idx_id
);
1740 id3
->lastplayed
= get_tag_numeric(entry
, tag_lastplayed
, idx_id
);
1741 id3
->score
= get_tag_numeric(entry
, tag_virt_autoscore
, idx_id
) / 10;
1742 id3
->year
= get_tag_numeric(entry
, tag_year
, idx_id
);
1744 id3
->discnum
= get_tag_numeric(entry
, tag_discnumber
, idx_id
);
1745 id3
->tracknum
= get_tag_numeric(entry
, tag_tracknumber
, idx_id
);
1746 id3
->bitrate
= get_tag_numeric(entry
, tag_bitrate
, idx_id
);
1747 if (id3
->bitrate
== 0)
1750 #if CONFIG_CODEC == SWCODEC
1751 if (global_settings
.autoresume_enable
)
1753 id3
->offset
= get_tag_numeric(entry
, tag_lastoffset
, idx_id
);
1754 logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1755 id3
->title
, id3
->offset
);
1763 static inline void write_item(const char *item
)
1765 int len
= strlen(item
) + 1;
1768 write(cachefd
, item
, len
);
1771 static int check_if_empty(char **tag
)
1775 if (*tag
== NULL
|| **tag
== '\0')
1778 return sizeof(UNTAGGED
); /* Tag length */
1781 length
= strlen(*tag
);
1782 if (length
> TAG_MAXLEN
)
1784 logf("over length tag: %s", *tag
);
1785 length
= TAG_MAXLEN
;
1786 (*tag
)[length
] = '\0';
1792 #define ADD_TAG(entry,tag,data) \
1794 entry.tag_offset[tag] = offset; \
1795 entry.tag_length[tag] = check_if_empty(data); \
1796 offset += entry.tag_length[tag]
1797 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1798 * idea, as it uses lots of stack and is called from a recursive function
1801 static void __attribute__ ((noinline
)) add_tagcache(char *path
,
1803 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1808 struct mp3entry id3
;
1809 struct temp_file_entry entry
;
1813 char tracknumfix
[3];
1815 int path_length
= strlen(path
);
1816 bool has_albumartist
;
1820 /* Crude logging for the sim - to aid in debugging */
1821 int logfd
= open(ROCKBOX_DIR
"/database.log",
1822 O_WRONLY
| O_APPEND
| O_CREAT
, 0666);
1824 write(logfd
, path
, strlen(path
));
1825 write(logfd
, "\n", 1);
1833 /* Check for overlength file path. */
1834 if (path_length
> TAG_MAXLEN
)
1836 /* Path can't be shortened. */
1837 logf("Too long path: %s", path
);
1841 /* Check if the file is supported. */
1842 if (probe_file_format(path
) == AFMT_UNKNOWN
)
1845 /* Check if the file is already cached. */
1846 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1847 idx_id
= find_entry_ram(path
, dc
);
1850 /* Be sure the entry doesn't exist. */
1851 if (filenametag_fd
>= 0 && idx_id
< 0)
1852 idx_id
= find_entry_disk(path
, false);
1854 /* Check if file has been modified. */
1857 struct index_entry idx
;
1859 /* TODO: Mark that the index exists (for fast reverse scan) */
1860 //found_idx[idx_id/8] |= idx_id%8;
1862 if (!get_index(-1, idx_id
, &idx
, true))
1864 logf("failed to retrieve index entry");
1868 if ((unsigned long)idx
.tag_seek
[tag_mtime
] == mtime
)
1870 /* No changes to file. */
1874 /* Metadata might have been changed. Delete the entry. */
1875 logf("Re-adding: %s", path
);
1876 if (!delete_entry(idx_id
))
1878 logf("delete_entry failed: %d", idx_id
);
1883 fd
= open(path
, O_RDONLY
);
1886 logf("open fail: %s", path
);
1890 memset(&id3
, 0, sizeof(struct mp3entry
));
1891 memset(&entry
, 0, sizeof(struct temp_file_entry
));
1892 memset(&tracknumfix
, 0, sizeof(tracknumfix
));
1893 ret
= get_metadata(&id3
, fd
, path
);
1899 logf("-> %s", path
);
1901 if (id3
.tracknum
<= 0) /* Track number missing? */
1907 entry
.tag_offset
[tag_year
] = id3
.year
;
1908 entry
.tag_offset
[tag_discnumber
] = id3
.discnum
;
1909 entry
.tag_offset
[tag_tracknumber
] = id3
.tracknum
;
1910 entry
.tag_offset
[tag_length
] = id3
.length
;
1911 entry
.tag_offset
[tag_bitrate
] = id3
.bitrate
;
1912 entry
.tag_offset
[tag_mtime
] = mtime
;
1915 has_albumartist
= id3
.albumartist
!= NULL
1916 && strlen(id3
.albumartist
) > 0;
1917 has_grouping
= id3
.grouping
!= NULL
1918 && strlen(id3
.grouping
) > 0;
1920 ADD_TAG(entry
, tag_filename
, &path
);
1921 ADD_TAG(entry
, tag_title
, &id3
.title
);
1922 ADD_TAG(entry
, tag_artist
, &id3
.artist
);
1923 ADD_TAG(entry
, tag_album
, &id3
.album
);
1924 ADD_TAG(entry
, tag_genre
, &id3
.genre_string
);
1925 ADD_TAG(entry
, tag_composer
, &id3
.composer
);
1926 ADD_TAG(entry
, tag_comment
, &id3
.comment
);
1927 if (has_albumartist
)
1929 ADD_TAG(entry
, tag_albumartist
, &id3
.albumartist
);
1933 ADD_TAG(entry
, tag_albumartist
, &id3
.artist
);
1937 ADD_TAG(entry
, tag_grouping
, &id3
.grouping
);
1941 ADD_TAG(entry
, tag_grouping
, &id3
.title
);
1943 entry
.data_length
= offset
;
1945 /* Write the header */
1946 write(cachefd
, &entry
, sizeof(struct temp_file_entry
));
1948 /* And tags also... Correct order is critical */
1950 write_item(id3
.title
);
1951 write_item(id3
.artist
);
1952 write_item(id3
.album
);
1953 write_item(id3
.genre_string
);
1954 write_item(id3
.composer
);
1955 write_item(id3
.comment
);
1956 if (has_albumartist
)
1958 write_item(id3
.albumartist
);
1962 write_item(id3
.artist
);
1966 write_item(id3
.grouping
);
1970 write_item(id3
.title
);
1972 total_entry_count
++;
1975 static bool tempbuf_insert(char *str
, int id
, int idx_id
, bool unique
)
1977 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1978 int len
= strlen(str
)+1;
1981 unsigned *crcbuf
= (unsigned *)&tempbuf
[tempbuf_size
-4];
1982 char buf
[TAG_MAXLEN
+32];
1984 for (i
= 0; str
[i
] != '\0' && i
< (int)sizeof(buf
)-1; i
++)
1985 buf
[i
] = tolower(str
[i
]);
1988 crc32
= crc_32(buf
, i
, 0xffffffff);
1992 /* Check if the crc does not exist -> entry does not exist for sure. */
1993 for (i
= 0; i
< tempbufidx
; i
++)
1995 if (crcbuf
[-i
] != crc32
)
1998 if (!strcasecmp(str
, index
[i
].str
))
2000 if (id
< 0 || id
>= lookup_buffer_depth
)
2002 logf("lookup buf overf.: %d", id
);
2006 lookup
[id
] = &index
[i
];
2012 /* Insert to CRC buffer. */
2013 crcbuf
[-tempbufidx
] = crc32
;
2016 /* Insert it to the buffer. */
2017 tempbuf_left
-= len
;
2018 if (tempbuf_left
- 4 < 0 || tempbufidx
>= commit_entry_count
-1)
2021 if (id
>= lookup_buffer_depth
)
2023 logf("lookup buf overf. #2: %d", id
);
2029 lookup
[id
] = &index
[tempbufidx
];
2030 index
[tempbufidx
].idlist
.id
= id
;
2033 index
[tempbufidx
].idlist
.id
= -1;
2035 index
[tempbufidx
].idlist
.next
= NULL
;
2036 index
[tempbufidx
].idx_id
= idx_id
;
2037 index
[tempbufidx
].seek
= -1;
2038 index
[tempbufidx
].str
= &tempbuf
[tempbuf_pos
];
2039 memcpy(index
[tempbufidx
].str
, str
, len
);
2046 static int compare(const void *p1
, const void *p2
)
2050 struct tempbuf_searchidx
*e1
= (struct tempbuf_searchidx
*)p1
;
2051 struct tempbuf_searchidx
*e2
= (struct tempbuf_searchidx
*)p2
;
2053 if (strcmp(e1
->str
, UNTAGGED
) == 0)
2055 if (strcmp(e2
->str
, UNTAGGED
) == 0)
2059 else if (strcmp(e2
->str
, UNTAGGED
) == 0)
2062 return strncasecmp(e1
->str
, e2
->str
, TAG_MAXLEN
);
2065 static int tempbuf_sort(int fd
)
2067 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
2068 struct tagfile_entry fe
;
2072 /* Generate reverse lookup entries. */
2073 for (i
= 0; i
< lookup_buffer_depth
; i
++)
2075 struct tempbuf_id_list
*idlist
;
2080 if (lookup
[i
]->idlist
.id
== i
)
2083 idlist
= &lookup
[i
]->idlist
;
2084 while (idlist
->next
!= NULL
)
2085 idlist
= idlist
->next
;
2087 tempbuf_left
-= sizeof(struct tempbuf_id_list
);
2088 if (tempbuf_left
- 4 < 0)
2091 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
2092 if (tempbuf_pos
& 0x03)
2094 tempbuf_pos
= (tempbuf_pos
& ~0x03) + 0x04;
2096 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
2098 tempbuf_pos
+= sizeof(struct tempbuf_id_list
);
2100 idlist
= idlist
->next
;
2102 idlist
->next
= NULL
;
2107 qsort(index
, tempbufidx
, sizeof(struct tempbuf_searchidx
), compare
);
2108 memset(lookup
, 0, lookup_buffer_depth
* sizeof(struct tempbuf_searchidx
**));
2110 for (i
= 0; i
< tempbufidx
; i
++)
2112 struct tempbuf_id_list
*idlist
= &index
[i
].idlist
;
2114 /* Fix the lookup list. */
2115 while (idlist
!= NULL
)
2117 if (idlist
->id
>= 0)
2118 lookup
[idlist
->id
] = &index
[i
];
2119 idlist
= idlist
->next
;
2122 index
[i
].seek
= lseek(fd
, 0, SEEK_CUR
);
2123 length
= strlen(index
[i
].str
) + 1;
2124 fe
.tag_length
= length
;
2125 fe
.idx_id
= index
[i
].idx_id
;
2127 /* Check the chunk alignment. */
2128 if ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
2129 % TAGFILE_ENTRY_CHUNK_LENGTH
)
2131 fe
.tag_length
+= TAGFILE_ENTRY_CHUNK_LENGTH
-
2132 ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
2133 % TAGFILE_ENTRY_CHUNK_LENGTH
);
2136 #ifdef TAGCACHE_STRICT_ALIGN
2137 /* Make sure the entry is long aligned. */
2138 if (index
[i
].seek
& 0x03)
2140 logf("tempbuf_sort: alignment error!");
2145 if (ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
) !=
2146 sizeof(struct tagfile_entry
))
2148 logf("tempbuf_sort: write error #1");
2152 if (write(fd
, index
[i
].str
, length
) != length
)
2154 logf("tempbuf_sort: write error #2");
2158 /* Write some padding. */
2159 if (fe
.tag_length
- length
> 0)
2160 write(fd
, "XXXXXXXX", fe
.tag_length
- length
);
2166 inline static struct tempbuf_searchidx
* tempbuf_locate(int id
)
2168 if (id
< 0 || id
>= lookup_buffer_depth
)
2175 inline static int tempbuf_find_location(int id
)
2177 struct tempbuf_searchidx
*entry
;
2179 entry
= tempbuf_locate(id
);
2186 static bool build_numeric_indices(struct tagcache_header
*h
, int tmpfd
)
2188 struct master_header tcmh
;
2189 struct index_entry idx
;
2192 struct temp_file_entry
*entrybuf
= (struct temp_file_entry
*)tempbuf
;
2194 int entries_processed
= 0;
2196 char buf
[TAG_MAXLEN
];
2198 max_entries
= tempbuf_size
/ sizeof(struct temp_file_entry
) - 1;
2200 logf("Building numeric indices...");
2201 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2203 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
2206 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2208 if (masterfd_pos
== filesize(masterfd
))
2210 logf("we can't append!");
2215 while (entries_processed
< h
->entry_count
)
2217 int count
= MIN(h
->entry_count
- entries_processed
, max_entries
);
2219 /* Read in as many entries as possible. */
2220 for (i
= 0; i
< count
; i
++)
2222 struct temp_file_entry
*tfe
= &entrybuf
[i
];
2225 /* Read in numeric data. */
2226 if (read(tmpfd
, tfe
, sizeof(struct temp_file_entry
)) !=
2227 sizeof(struct temp_file_entry
))
2229 logf("read fail #1");
2234 datastart
= lseek(tmpfd
, 0, SEEK_CUR
);
2237 * Read string data from the following tags:
2243 * A crc32 hash is calculated from the read data
2244 * and stored back to the data offset field kept in memory.
2246 #define tmpdb_read_string_tag(tag) \
2247 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2248 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2250 logf("read fail: buffer overflow"); \
2255 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2256 tfe->tag_length[tag]) \
2258 logf("read fail #2"); \
2263 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2264 lseek(tmpfd, datastart, SEEK_SET)
2266 tmpdb_read_string_tag(tag_filename
);
2267 tmpdb_read_string_tag(tag_artist
);
2268 tmpdb_read_string_tag(tag_album
);
2269 tmpdb_read_string_tag(tag_title
);
2271 /* Seek to the end of the string data. */
2272 lseek(tmpfd
, tfe
->data_length
, SEEK_CUR
);
2275 /* Backup the master index position. */
2276 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2277 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2279 /* Check if we can resurrect some deleted runtime statistics data. */
2280 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
++)
2282 /* Read the index entry. */
2283 if (ecread_index_entry(masterfd
, &idx
)
2284 != sizeof(struct index_entry
))
2286 logf("read fail #3");
2292 * Skip unless the entry is marked as being deleted
2293 * or the data has already been resurrected.
2295 if (!(idx
.flag
& FLAG_DELETED
) || idx
.flag
& FLAG_RESURRECTED
)
2298 /* Now try to match the entry. */
2300 * To succesfully match a song, the following conditions
2303 * For numeric fields: tag_length
2304 * - Full identical match is required
2306 * If tag_filename matches, no further checking necessary.
2308 * For string hashes: tag_artist, tag_album, tag_title
2309 * - All three of these must match
2311 for (j
= 0; j
< count
; j
++)
2313 struct temp_file_entry
*tfe
= &entrybuf
[j
];
2315 /* Try to match numeric fields first. */
2316 if (tfe
->tag_offset
[tag_length
] != idx
.tag_seek
[tag_length
])
2319 /* Now it's time to do the hash matching. */
2320 if (tfe
->tag_offset
[tag_filename
] != idx
.tag_seek
[tag_filename
])
2322 int match_count
= 0;
2324 /* No filename match, check if we can match two other tags. */
2325 #define tmpdb_match(tag) \
2326 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2329 tmpdb_match(tag_artist
);
2330 tmpdb_match(tag_album
);
2331 tmpdb_match(tag_title
);
2333 if (match_count
< 3)
2335 /* Still no match found, give up. */
2340 /* A match found, now copy & resurrect the statistical data. */
2341 #define tmpdb_copy_tag(tag) \
2342 tfe->tag_offset[tag] = idx.tag_seek[tag]
2344 tmpdb_copy_tag(tag_playcount
);
2345 tmpdb_copy_tag(tag_rating
);
2346 tmpdb_copy_tag(tag_playtime
);
2347 tmpdb_copy_tag(tag_lastplayed
);
2348 tmpdb_copy_tag(tag_commitid
);
2349 tmpdb_copy_tag(tag_lastoffset
);
2351 /* Avoid processing this entry again. */
2352 idx
.flag
|= FLAG_RESURRECTED
;
2354 lseek(masterfd
, -sizeof(struct index_entry
), SEEK_CUR
);
2355 if (ecwrite_index_entry(masterfd
, &idx
) != sizeof(struct index_entry
))
2357 logf("masterfd writeback fail #1");
2362 logf("Entry resurrected");
2367 /* Restore the master index position. */
2368 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2370 /* Commit the data to the index. */
2371 for (i
= 0; i
< count
; i
++)
2373 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2375 if (ecread_index_entry(masterfd
, &idx
) != sizeof(struct index_entry
))
2377 logf("read fail #3");
2382 for (j
= 0; j
< TAG_COUNT
; j
++)
2384 if (!TAGCACHE_IS_NUMERIC(j
))
2387 idx
.tag_seek
[j
] = entrybuf
[i
].tag_offset
[j
];
2389 idx
.flag
= entrybuf
[i
].flag
;
2391 if (idx
.tag_seek
[tag_commitid
])
2393 /* Data has been resurrected. */
2394 idx
.flag
|= FLAG_DIRTYNUM
;
2396 else if (tc_stat
.ready
&& current_tcmh
.commitid
> 0)
2398 idx
.tag_seek
[tag_commitid
] = current_tcmh
.commitid
;
2399 idx
.flag
|= FLAG_DIRTYNUM
;
2402 /* Write back the updated index. */
2403 lseek(masterfd
, loc
, SEEK_SET
);
2404 if (ecwrite_index_entry(masterfd
, &idx
) != sizeof(struct index_entry
))
2412 entries_processed
+= count
;
2413 logf("%d/%ld entries processed", entries_processed
, h
->entry_count
);
2424 * == 0 temporary failure
2427 static int build_index(int index_type
, struct tagcache_header
*h
, int tmpfd
)
2430 struct tagcache_header tch
;
2431 struct master_header tcmh
;
2432 struct index_entry idxbuf
[IDX_BUF_DEPTH
];
2434 char buf
[TAG_MAXLEN
+32];
2435 int fd
= -1, masterfd
;
2440 logf("Building index: %d", index_type
);
2442 /* Check the number of entries we need to allocate ram for. */
2443 commit_entry_count
= h
->entry_count
+ 1;
2445 masterfd
= open_master_fd(&tcmh
, false);
2448 commit_entry_count
+= tcmh
.tch
.entry_count
;
2452 remove_files(); /* Just to be sure we are clean. */
2454 /* Open the index file, which contains the tag names. */
2455 fd
= open_tag_fd(&tch
, index_type
, true);
2458 logf("tch.datasize=%ld", tch
.datasize
);
2459 lookup_buffer_depth
= 1 +
2460 /* First part */ commit_entry_count
+
2461 /* Second part */ (tch
.datasize
/ TAGFILE_ENTRY_CHUNK_LENGTH
);
2465 lookup_buffer_depth
= 1 +
2466 /* First part */ commit_entry_count
+
2467 /* Second part */ 0;
2470 logf("lookup_buffer_depth=%ld", lookup_buffer_depth
);
2471 logf("commit_entry_count=%ld", commit_entry_count
);
2473 /* Allocate buffer for all index entries from both old and new
2476 tempbuf_pos
= commit_entry_count
* sizeof(struct tempbuf_searchidx
);
2478 /* Allocate lookup buffer. The first portion of commit_entry_count
2479 * contains the new tags in the temporary file and the second
2480 * part for locating entries already in the db.
2483 * +---------+---------------------------+
2484 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2485 * +---------+---------------------------+
2487 * Old tags are inserted to a temporary buffer with position:
2488 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2489 * And new tags with index:
2490 * tempbuf_insert(idx, ...);
2492 * The buffer is sorted and written into tag file:
2493 * tempbuf_sort(...);
2494 * leaving master index locations messed up.
2496 * That is fixed using the lookup buffer for old tags:
2497 * new_seek = tempbuf_find_location(old_seek, ...);
2499 * new_seek = tempbuf_find_location(idx);
2501 lookup
= (struct tempbuf_searchidx
**)&tempbuf
[tempbuf_pos
];
2502 tempbuf_pos
+= lookup_buffer_depth
* sizeof(void **);
2503 memset(lookup
, 0, lookup_buffer_depth
* sizeof(void **));
2505 /* And calculate the remaining data space used mainly for storing
2506 * tag data (strings). */
2507 tempbuf_left
= tempbuf_size
- tempbuf_pos
- 8;
2508 if (tempbuf_left
- TAGFILE_ENTRY_AVG_LENGTH
* commit_entry_count
< 0)
2510 logf("Buffer way too small!");
2517 * If tag file contains unique tags (sorted index), we will load
2518 * it entirely into memory so we can resort it later for use with
2521 if (TAGCACHE_IS_SORTED(index_type
))
2523 logf("loading tags...");
2524 for (i
= 0; i
< tch
.entry_count
; i
++)
2526 struct tagfile_entry entry
;
2527 int loc
= lseek(fd
, 0, SEEK_CUR
);
2530 if (ecread_tagfile_entry(fd
, &entry
) != sizeof(struct tagfile_entry
))
2532 logf("read error #7");
2537 if (entry
.tag_length
>= (int)sizeof(buf
))
2539 logf("too long tag #3");
2544 if (read(fd
, buf
, entry
.tag_length
) != entry
.tag_length
)
2546 logf("read error #8");
2551 /* Skip deleted entries. */
2556 * Save the tag and tag id in the memory buffer. Tag id
2557 * is saved so we can later reindex the master lookup
2558 * table when the index gets resorted.
2560 ret
= tempbuf_insert(buf
, loc
/TAGFILE_ENTRY_CHUNK_LENGTH
2561 + commit_entry_count
, entry
.idx_id
,
2562 TAGCACHE_IS_UNIQUE(index_type
));
2573 tempbufidx
= tch
.entry_count
;
2578 * Creating new index file to store the tags. No need to preload
2579 * anything whether the index type is sorted or not.
2581 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, index_type
);
2582 fd
= open(buf
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
2585 logf("%s open fail", buf
);
2589 tch
.magic
= TAGCACHE_MAGIC
;
2590 tch
.entry_count
= 0;
2593 if (ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
)
2594 != sizeof(struct tagcache_header
))
2596 logf("header write failed");
2602 /* Loading the tag lookup file as "master file". */
2603 logf("Loading index file");
2604 masterfd
= open(TAGCACHE_FILE_MASTER
, O_RDWR
);
2608 logf("Creating new DB");
2609 masterfd
= open(TAGCACHE_FILE_MASTER
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
2613 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER
);
2618 /* Write the header (write real values later). */
2619 memset(&tcmh
, 0, sizeof(struct master_header
));
2621 tcmh
.tch
.entry_count
= 0;
2622 tcmh
.tch
.datasize
= 0;
2624 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2626 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2631 * Master file already exists so we need to process the current
2636 if (ecread(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
) !=
2637 sizeof(struct master_header
) || tcmh
.tch
.magic
!= TAGCACHE_MAGIC
)
2639 logf("header error");
2646 * If we reach end of the master file, we need to expand it to
2647 * hold new tags. If the current index is not sorted, we can
2648 * simply append new data to end of the file.
2649 * However, if the index is sorted, we need to update all tag
2650 * pointers in the master file for the current index.
2652 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2654 if (masterfd_pos
== filesize(masterfd
))
2656 logf("appending...");
2662 * Load new unique tags in memory to be sorted later and added
2663 * to the master lookup file.
2665 if (TAGCACHE_IS_SORTED(index_type
))
2667 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2668 /* h is the header of the temporary file containing new tags. */
2669 logf("inserting new tags...");
2670 for (i
= 0; i
< h
->entry_count
; i
++)
2672 struct temp_file_entry entry
;
2674 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2675 sizeof(struct temp_file_entry
))
2677 logf("read fail #3");
2683 if (entry
.tag_length
[index_type
] >= (long)sizeof(buf
))
2685 logf("too long entry!");
2690 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2691 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2692 entry
.tag_length
[index_type
])
2694 logf("read fail #4");
2699 if (TAGCACHE_IS_UNIQUE(index_type
))
2700 error
= !tempbuf_insert(buf
, i
, -1, true);
2702 error
= !tempbuf_insert(buf
, i
, tcmh
.tch
.entry_count
+ i
, false);
2706 logf("insert error");
2711 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2712 entry
.tag_length
[index_type
], SEEK_CUR
);
2717 /* Sort the buffer data and write it to the index file. */
2718 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
2720 * We need to truncate the index file now. There can be junk left
2721 * at the end of file (however, we _should_ always follow the
2722 * entry_count and don't crash with that).
2724 ftruncate(fd
, lseek(fd
, 0, SEEK_CUR
));
2726 i
= tempbuf_sort(fd
);
2729 logf("sorted %d tags", i
);
2732 * Now update all indexes in the master lookup file.
2734 logf("updating indices...");
2735 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2736 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
+= idxbuf_pos
)
2739 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2741 idxbuf_pos
= MIN(tcmh
.tch
.entry_count
- i
, IDX_BUF_DEPTH
);
2743 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2744 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2746 logf("read fail #5");
2750 lseek(masterfd
, loc
, SEEK_SET
);
2752 for (j
= 0; j
< idxbuf_pos
; j
++)
2754 if (idxbuf
[j
].flag
& FLAG_DELETED
)
2756 /* We can just ignore deleted entries. */
2757 // idxbuf[j].tag_seek[index_type] = 0;
2761 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(
2762 idxbuf
[j
].tag_seek
[index_type
]/TAGFILE_ENTRY_CHUNK_LENGTH
2763 + commit_entry_count
);
2765 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2767 logf("update error: %ld/%d/%ld",
2768 idxbuf
[j
].flag
, i
+j
, tcmh
.tch
.entry_count
);
2776 /* Write back the updated index. */
2777 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2778 index_entry_ec
, tc_stat
.econ
) !=
2779 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2790 * Walk through the temporary file containing the new tags.
2792 // build_normal_index(h, tmpfd, masterfd, idx);
2793 logf("updating new indices...");
2794 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2795 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2796 lseek(fd
, 0, SEEK_END
);
2797 for (i
= 0; i
< h
->entry_count
; i
+= idxbuf_pos
)
2801 idxbuf_pos
= MIN(h
->entry_count
- i
, IDX_BUF_DEPTH
);
2804 memset(idxbuf
, 0, sizeof(struct index_entry
)*IDX_BUF_DEPTH
);
2808 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2810 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2811 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2813 logf("read fail #6");
2817 lseek(masterfd
, loc
, SEEK_SET
);
2820 /* Read entry headers. */
2821 for (j
= 0; j
< idxbuf_pos
; j
++)
2823 if (!TAGCACHE_IS_SORTED(index_type
))
2825 struct temp_file_entry entry
;
2826 struct tagfile_entry fe
;
2828 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2829 sizeof(struct temp_file_entry
))
2831 logf("read fail #7");
2837 if (entry
.tag_length
[index_type
] >= (int)sizeof(buf
))
2839 logf("too long entry!");
2840 logf("length=%d", entry
.tag_length
[index_type
]);
2841 logf("pos=0x%02lx", lseek(tmpfd
, 0, SEEK_CUR
));
2846 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2847 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2848 entry
.tag_length
[index_type
])
2850 logf("read fail #8");
2851 logf("offset=0x%02lx", entry
.tag_offset
[index_type
]);
2852 logf("length=0x%02x", entry
.tag_length
[index_type
]);
2857 /* Write to index file. */
2858 idxbuf
[j
].tag_seek
[index_type
] = lseek(fd
, 0, SEEK_CUR
);
2859 fe
.tag_length
= entry
.tag_length
[index_type
];
2860 fe
.idx_id
= tcmh
.tch
.entry_count
+ i
+ j
;
2861 ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
2862 write(fd
, buf
, fe
.tag_length
);
2866 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2867 entry
.tag_length
[index_type
], SEEK_CUR
);
2871 /* Locate the correct entry from the sorted array. */
2872 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(i
+ j
);
2873 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2875 logf("entry not found (%d)", j
);
2883 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2884 index_entry_ec
, tc_stat
.econ
) !=
2885 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2887 logf("tagcache: write fail #4");
2896 /* Finally write the header. */
2897 tch
.magic
= TAGCACHE_MAGIC
;
2898 tch
.entry_count
= tempbufidx
;
2899 tch
.datasize
= lseek(fd
, 0, SEEK_END
) - sizeof(struct tagcache_header
);
2900 lseek(fd
, 0, SEEK_SET
);
2901 ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
);
2903 if (index_type
!= tag_filename
)
2904 h
->datasize
+= tch
.datasize
;
2905 logf("s:%d/%ld/%ld", index_type
, tch
.datasize
, h
->datasize
);
2917 static bool commit(void)
2919 struct tagcache_header tch
;
2920 struct master_header tcmh
;
2924 #ifdef HAVE_DIRCACHE
2925 bool dircache_buffer_stolen
= false;
2927 #ifdef HAVE_TC_RAMCACHE
2928 bool ramcache_buffer_stolen
= false;
2930 bool local_allocation
= false;
2932 logf("committing tagcache");
2937 tmpfd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
2940 logf("nothing to commit");
2945 /* Load the header. */
2946 len
= sizeof(struct tagcache_header
);
2947 rc
= read(tmpfd
, &tch
, len
);
2949 if (tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= len
)
2951 logf("incorrect tmpheader");
2953 remove(TAGCACHE_FILE_TEMP
);
2957 if (tch
.entry_count
== 0)
2959 logf("nothing to commit");
2961 remove(TAGCACHE_FILE_TEMP
);
2965 /* Fully initialize existing headers (if any) before going further. */
2966 tc_stat
.ready
= check_all_headers();
2968 #ifdef HAVE_EEPROM_SETTINGS
2969 remove(TAGCACHE_STATEFILE
);
2972 /* At first be sure to unload the ramcache! */
2973 #ifdef HAVE_TC_RAMCACHE
2974 tc_stat
.ramcache
= false;
2979 /* Try to steal every buffer we can :) */
2980 if (tempbuf_size
== 0)
2981 local_allocation
= true;
2983 #ifdef HAVE_DIRCACHE
2984 if (tempbuf_size
== 0)
2986 /* Try to steal the dircache buffer. */
2987 tempbuf
= dircache_steal_buffer(&tempbuf_size
);
2988 tempbuf_size
&= ~0x03;
2990 if (tempbuf_size
> 0)
2992 dircache_buffer_stolen
= true;
2997 #ifdef HAVE_TC_RAMCACHE
2998 if (tempbuf_size
== 0 && tc_stat
.ramcache_allocated
> 0)
3000 tempbuf
= (char *)(ramcache_hdr
+ 1);
3001 tempbuf_size
= tc_stat
.ramcache_allocated
- sizeof(struct ramcache_header
) - 128;
3002 tempbuf_size
&= ~0x03;
3004 ramcache_buffer_stolen
= true;
3008 /* And finally fail if there are no buffers available. */
3009 if (tempbuf_size
== 0)
3011 logf("delaying commit until next boot");
3012 tc_stat
.commit_delayed
= true;
3018 logf("commit %ld entries...", tch
.entry_count
);
3020 /* Mark DB dirty so it will stay disabled if commit fails. */
3021 current_tcmh
.dirty
= true;
3022 update_master_header();
3024 /* Now create the index files. */
3025 tc_stat
.commit_step
= 0;
3027 tc_stat
.commit_delayed
= false;
3029 for (i
= 0; i
< TAG_COUNT
; i
++)
3033 if (TAGCACHE_IS_NUMERIC(i
))
3036 tc_stat
.commit_step
++;
3037 ret
= build_index(i
, &tch
, tmpfd
);
3041 logf("tagcache failed init");
3043 tc_stat
.commit_delayed
= true;
3045 tc_stat
.commit_step
= 0;
3051 if (!build_numeric_indices(&tch
, tmpfd
))
3053 logf("Failure to commit numeric indices");
3055 tc_stat
.commit_step
= 0;
3062 tc_stat
.commit_step
= 0;
3064 /* Update the master index headers. */
3065 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
3071 remove(TAGCACHE_FILE_TEMP
);
3073 tcmh
.tch
.entry_count
+= tch
.entry_count
;
3074 tcmh
.tch
.datasize
= sizeof(struct master_header
)
3075 + sizeof(struct index_entry
) * tcmh
.tch
.entry_count
3080 lseek(masterfd
, 0, SEEK_SET
);
3081 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
3084 logf("tagcache committed");
3085 tc_stat
.ready
= check_all_headers();
3086 tc_stat
.readyvalid
= true;
3088 if (local_allocation
)
3094 #ifdef HAVE_DIRCACHE
3095 /* Rebuild the dircache, if we stole the buffer. */
3096 if (dircache_buffer_stolen
)
3100 #ifdef HAVE_TC_RAMCACHE
3101 if (ramcache_buffer_stolen
)
3103 /* Reload tagcache. */
3104 if (tc_stat
.ramcache_allocated
> 0)
3105 tagcache_start_scan();
3113 static int tempbuf_handle
;
3114 static void allocate_tempbuf(void)
3116 /* Yeah, malloc would be really nice now :) */
3118 tempbuf_size
= 32*1024*1024;
3119 tempbuf
= malloc(tempbuf_size
);
3121 tempbuf_handle
= core_alloc_maximum("tc tempbuf", &tempbuf_size
, NULL
);
3122 tempbuf
= core_get_data(tempbuf_handle
);
3126 static void free_tempbuf(void)
3128 if (tempbuf_size
== 0)
3134 tempbuf_handle
= core_free(tempbuf_handle
);
3142 static bool modify_numeric_entry(int masterfd
, int idx_id
, int tag
, long data
)
3144 struct index_entry idx
;
3149 if (!TAGCACHE_IS_NUMERIC(tag
))
3152 if (!get_index(masterfd
, idx_id
, &idx
, false))
3155 idx
.tag_seek
[tag
] = data
;
3156 idx
.flag
|= FLAG_DIRTYNUM
;
3158 return write_index(masterfd
, idx_id
, &idx
);
3162 bool tagcache_modify_numeric_entry(struct tagcache_search
*tcs
,
3165 struct master_header myhdr
;
3167 if (tcs
->masterfd
< 0)
3169 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, true)) < 0)
3173 return modify_numeric_entry(tcs
->masterfd
, tcs
->idx_id
, tag
, data
);
3177 static bool command_queue_is_full(void)
3181 next
= command_queue_widx
+ 1;
3182 if (next
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3185 return (next
== command_queue_ridx
);
3188 static void command_queue_sync_callback(void *data
)
3191 struct master_header myhdr
;
3194 mutex_lock(&command_queue_mutex
);
3196 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3199 while (command_queue_ridx
!= command_queue_widx
)
3201 struct tagcache_command_entry
*ce
= &command_queue
[command_queue_ridx
];
3203 switch (ce
->command
)
3205 case CMD_UPDATE_MASTER_HEADER
:
3208 update_master_header();
3210 /* Re-open the masterfd. */
3211 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3216 case CMD_UPDATE_NUMERIC
:
3218 modify_numeric_entry(masterfd
, ce
->idx_id
, ce
->tag
, ce
->data
);
3223 if (++command_queue_ridx
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3224 command_queue_ridx
= 0;
3229 tc_stat
.queue_length
= 0;
3230 mutex_unlock(&command_queue_mutex
);
3233 static void run_command_queue(bool force
)
3235 if (COMMAND_QUEUE_IS_EMPTY
)
3238 if (force
|| command_queue_is_full())
3239 command_queue_sync_callback(NULL
);
3241 register_storage_idle_func(command_queue_sync_callback
);
3244 static void queue_command(int cmd
, long idx_id
, int tag
, long data
)
3250 mutex_lock(&command_queue_mutex
);
3251 next
= command_queue_widx
+ 1;
3252 if (next
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3255 /* Make sure queue is not full. */
3256 if (next
!= command_queue_ridx
)
3258 struct tagcache_command_entry
*ce
= &command_queue
[command_queue_widx
];
3261 ce
->idx_id
= idx_id
;
3265 command_queue_widx
= next
;
3267 tc_stat
.queue_length
++;
3269 mutex_unlock(&command_queue_mutex
);
3273 /* Queue is full, try again later... */
3274 mutex_unlock(&command_queue_mutex
);
3279 long tagcache_increase_serial(void)
3289 old
= current_tcmh
.serial
++;
3290 queue_command(CMD_UPDATE_MASTER_HEADER
, 0, 0, 0);
3295 void tagcache_update_numeric(int idx_id
, int tag
, long data
)
3297 queue_command(CMD_UPDATE_NUMERIC
, idx_id
, tag
, data
);
3299 #endif /* !__PCTOOL__ */
3301 static bool write_tag(int fd
, const char *tagstr
, const char *datastr
)
3306 snprintf(buf
, sizeof buf
, "%s=\"", tagstr
);
3307 for (i
= strlen(buf
); i
< (long)sizeof(buf
)-4; i
++)
3309 if (*datastr
== '\0')
3312 if (*datastr
== '"' || *datastr
== '\\')
3315 else if (*datastr
== '\n')
3322 buf
[i
] = *(datastr
++);
3325 strcpy(&buf
[i
], "\" ");
3327 write(fd
, buf
, i
+ 2);
3334 static bool read_tag(char *dest
, long size
,
3335 const char *src
, const char *tagstr
)
3338 char current_tag
[32];
3340 while (*src
!= '\0')
3342 /* Skip all whitespace */
3350 /* Read in tag name */
3351 while (*src
!= '=' && *src
!= ' ')
3353 current_tag
[pos
] = *src
;
3357 if (*src
== '\0' || pos
>= (long)sizeof(current_tag
))
3360 current_tag
[pos
] = '\0';
3362 /* Read in tag data */
3364 /* Find the start. */
3365 while (*src
!= '"' && *src
!= '\0')
3368 if (*src
== '\0' || *(++src
) == '\0')
3371 /* Read the data. */
3372 for (pos
= 0; pos
< size
; pos
++)
3398 dest
[pos
] = *(src
++);
3403 if (!strcasecmp(tagstr
, current_tag
))
3410 static int parse_changelog_line(int line_n
, char *buf
, void *parameters
)
3412 struct index_entry idx
;
3413 char tag_data
[TAG_MAXLEN
+32];
3415 long masterfd
= (long)parameters
;
3416 const int import_tags
[] = { tag_playcount
, tag_rating
, tag_playtime
,
3417 tag_lastplayed
, tag_commitid
, tag_lastoffset
};
3424 /* logf("%d/%s", line_n, buf); */
3425 if (!read_tag(tag_data
, sizeof tag_data
, buf
, "filename"))
3427 logf("%d/filename missing", line_n
);
3432 idx_id
= find_index(tag_data
);
3435 logf("%d/entry not found", line_n
);
3439 if (!get_index(masterfd
, idx_id
, &idx
, false))
3441 logf("%d/failed to retrieve index entry", line_n
);
3445 /* Stop if tag has already been modified. */
3446 if (idx
.flag
& FLAG_DIRTYNUM
)
3449 logf("%d/import: %s", line_n
, tag_data
);
3451 idx
.flag
|= FLAG_DIRTYNUM
;
3452 for (i
= 0; i
< (long)(sizeof(import_tags
)/sizeof(import_tags
[0])); i
++)
3456 if (!read_tag(tag_data
, sizeof tag_data
, buf
,
3457 tagcache_tag_to_str(import_tags
[i
])))
3462 data
= atoi(tag_data
);
3466 idx
.tag_seek
[import_tags
[i
]] = data
;
3468 if (import_tags
[i
] == tag_lastplayed
&& data
>= current_tcmh
.serial
)
3469 current_tcmh
.serial
= data
+ 1;
3470 else if (import_tags
[i
] == tag_commitid
&& data
>= current_tcmh
.commitid
)
3471 current_tcmh
.commitid
= data
+ 1;
3474 return write_index(masterfd
, idx_id
, &idx
) ? 0 : -5;
3477 bool tagcache_import_changelog(void)
3479 struct master_header myhdr
;
3480 struct tagcache_header tch
;
3491 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_RDONLY
);
3494 logf("failure to open changelog");
3498 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3506 filenametag_fd
= open_tag_fd(&tch
, tag_filename
, false);
3508 fast_readline(clfd
, buf
, sizeof buf
, (long *)masterfd
,
3509 parse_changelog_line
);
3514 if (filenametag_fd
>= 0)
3516 close(filenametag_fd
);
3517 filenametag_fd
= -1;
3522 update_master_header();
3527 #endif /* !__PCTOOL__ */
3529 bool tagcache_create_changelog(struct tagcache_search
*tcs
)
3531 struct master_header myhdr
;
3532 struct index_entry idx
;
3533 char buf
[TAG_MAXLEN
+32];
3541 if (!tagcache_search(tcs
, tag_filename
))
3544 /* Initialize the changelog */
3545 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
3548 logf("failure to open changelog");
3552 if (tcs
->masterfd
< 0)
3554 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, false)) < 0)
3562 lseek(tcs
->masterfd
, 0, SEEK_SET
);
3563 ecread(tcs
->masterfd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
3566 write(clfd
, "## Changelog version 1\n", 23);
3568 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3570 if (ecread_index_entry(tcs
->masterfd
, &idx
) != sizeof(struct index_entry
))
3572 logf("read error #9");
3573 tagcache_search_finish(tcs
);
3578 /* Skip until the entry found has been modified. */
3579 if (! (idx
.flag
& FLAG_DIRTYNUM
) )
3582 /* Skip deleted entries too. */
3583 if (idx
.flag
& FLAG_DELETED
)
3586 /* Now retrieve all tags. */
3587 for (j
= 0; j
< TAG_COUNT
; j
++)
3589 if (TAGCACHE_IS_NUMERIC(j
))
3591 snprintf(temp
, sizeof temp
, "%d", (int)idx
.tag_seek
[j
]);
3592 write_tag(clfd
, tagcache_tag_to_str(j
), temp
);
3597 tagcache_retrieve(tcs
, i
, tcs
->type
, buf
, sizeof buf
);
3598 write_tag(clfd
, tagcache_tag_to_str(j
), buf
);
3601 write(clfd
, "\n", 1);
3607 tagcache_search_finish(tcs
);
3612 static bool delete_entry(long idx_id
)
3617 struct index_entry idx
, myidx
;
3618 struct master_header myhdr
;
3619 char buf
[TAG_MAXLEN
+32];
3620 int in_use
[TAG_COUNT
];
3622 logf("delete_entry(): %ld", idx_id
);
3624 #ifdef HAVE_TC_RAMCACHE
3625 /* At first mark the entry removed from ram cache. */
3626 if (tc_stat
.ramcache
)
3627 ramcache_hdr
->indices
[idx_id
].flag
|= FLAG_DELETED
;
3630 if ( (masterfd
= open_master_fd(&myhdr
, true) ) < 0)
3633 lseek(masterfd
, idx_id
* sizeof(struct index_entry
), SEEK_CUR
);
3634 if (ecread_index_entry(masterfd
, &myidx
) != sizeof(struct index_entry
))
3636 logf("delete_entry(): read error");
3640 if (myidx
.flag
& FLAG_DELETED
)
3642 logf("delete_entry(): already deleted!");
3646 myidx
.flag
|= FLAG_DELETED
;
3647 lseek(masterfd
, -sizeof(struct index_entry
), SEEK_CUR
);
3648 if (ecwrite_index_entry(masterfd
, &myidx
) != sizeof(struct index_entry
))
3650 logf("delete_entry(): write_error #1");
3654 /* Now check which tags are no longer in use (if any) */
3655 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3658 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
3659 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3661 struct index_entry
*idxp
;
3663 #ifdef HAVE_TC_RAMCACHE
3664 /* Use RAM DB if available for greater speed */
3665 if (tc_stat
.ramcache
)
3666 idxp
= &ramcache_hdr
->indices
[i
];
3670 if (ecread_index_entry(masterfd
, &idx
) != sizeof(struct index_entry
))
3672 logf("delete_entry(): read error #2");
3678 if (idxp
->flag
& FLAG_DELETED
)
3681 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3683 if (TAGCACHE_IS_NUMERIC(tag
))
3686 if (idxp
->tag_seek
[tag
] == myidx
.tag_seek
[tag
])
3691 /* Now delete all tags no longer in use. */
3692 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3694 struct tagcache_header tch
;
3695 int oldseek
= myidx
.tag_seek
[tag
];
3697 if (TAGCACHE_IS_NUMERIC(tag
))
3701 * Replace tag seek with a hash value of the field string data.
3702 * That way runtime statistics of moved or altered files can be
3705 #ifdef HAVE_TC_RAMCACHE
3706 if (tc_stat
.ramcache
&& tag
!= tag_filename
)
3708 struct tagfile_entry
*tfe
;
3709 int32_t *seek
= &ramcache_hdr
->indices
[idx_id
].tag_seek
[tag
];
3711 tfe
= (struct tagfile_entry
*)&ramcache_hdr
->tags
[tag
][*seek
];
3712 move_lock
++; /* protect tfe and seek if crc_32() yield()s */
3713 *seek
= crc_32(tfe
->tag_data
, strlen(tfe
->tag_data
), 0xffffffff);
3715 myidx
.tag_seek
[tag
] = *seek
;
3720 struct tagfile_entry tfe
;
3722 /* Open the index file, which contains the tag names. */
3723 if ((fd
= open_tag_fd(&tch
, tag
, true)) < 0)
3726 /* Skip the header block */
3727 lseek(fd
, myidx
.tag_seek
[tag
], SEEK_SET
);
3728 if (ecread_tagfile_entry(fd
, &tfe
) != sizeof(struct tagfile_entry
))
3730 logf("delete_entry(): read error #3");
3734 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
3736 logf("delete_entry(): read error #4");
3740 myidx
.tag_seek
[tag
] = crc_32(buf
, strlen(buf
), 0xffffffff);
3745 logf("in use: %d/%d", tag
, in_use
[tag
]);
3754 #ifdef HAVE_TC_RAMCACHE
3755 /* Delete from ram. */
3756 if (tc_stat
.ramcache
&& tag
!= tag_filename
)
3758 struct tagfile_entry
*tagentry
=
3759 (struct tagfile_entry
*)&ramcache_hdr
->tags
[tag
][oldseek
];
3760 tagentry
->tag_data
[0] = '\0';
3764 /* Open the index file, which contains the tag names. */
3767 if ((fd
= open_tag_fd(&tch
, tag
, true)) < 0)
3771 /* Skip the header block */
3772 lseek(fd
, oldseek
+ sizeof(struct tagfile_entry
), SEEK_SET
);
3774 /* Debug, print 10 first characters of the tag
3777 logf("TAG:%s", buf);
3778 lseek(fd, -10, SEEK_CUR);
3781 /* Write first data byte in tag as \0 */
3784 /* Now tag data has been removed */
3789 /* Write index entry back into master index. */
3790 lseek(masterfd
, sizeof(struct master_header
) +
3791 (idx_id
* sizeof(struct index_entry
)), SEEK_SET
);
3792 if (ecwrite_index_entry(masterfd
, &myidx
) != sizeof(struct index_entry
))
3794 logf("delete_entry(): write_error #2");
3813 * Returns true if there is an event waiting in the queue
3814 * that requires the current operation to be aborted.
3816 static bool check_event_queue(void)
3818 struct queue_event ev
;
3820 if(!queue_peek(&tagcache_queue
, &ev
))
3827 case SYS_USB_CONNECTED
:
3835 #ifdef HAVE_TC_RAMCACHE
3837 static void fix_ramcache(void* old_addr
, void* new_addr
)
3839 ptrdiff_t offpos
= new_addr
- old_addr
;
3840 for (int i
= 0; i
< TAG_COUNT
; i
++)
3841 ramcache_hdr
->tags
[i
] += offpos
;
3844 static int move_cb(int handle
, void* current
, void* new)
3848 return BUFLIB_CB_CANNOT_MOVE
;
3850 fix_ramcache(current
, new);
3852 return BUFLIB_CB_OK
;
3855 static struct buflib_callbacks ops
= {
3856 .move_callback
= move_cb
,
3857 .shrink_callback
= NULL
,
3860 static bool allocate_tagcache(void)
3862 struct master_header tcmh
;
3865 /* Load the header. */
3866 if ( (fd
= open_master_fd(&tcmh
, false)) < 0)
3868 ramcache_hdr
= NULL
;
3875 * Now calculate the required cache size plus
3876 * some extra space for alignment fixes.
3878 tc_stat
.ramcache_allocated
= tcmh
.tch
.datasize
+ 256 + TAGCACHE_RESERVE
+
3879 sizeof(struct ramcache_header
) + TAG_COUNT
*sizeof(void *);
3880 int handle
= core_alloc_ex("tc ramcache", tc_stat
.ramcache_allocated
, &ops
);
3881 ramcache_hdr
= core_get_data(handle
);
3882 memset(ramcache_hdr
, 0, sizeof(struct ramcache_header
));
3883 memcpy(¤t_tcmh
, &tcmh
, sizeof current_tcmh
);
3884 logf("tagcache: %d bytes allocated.", tc_stat
.ramcache_allocated
);
3889 # ifdef HAVE_EEPROM_SETTINGS
3890 static bool tagcache_dumpload(void)
3892 struct statefile_header shdr
;
3895 fd
= open(TAGCACHE_STATEFILE
, O_RDONLY
);
3898 logf("no tagcache statedump");
3902 /* Check the statefile memory placement */
3903 rc
= read(fd
, &shdr
, sizeof(struct statefile_header
));
3904 if (rc
!= sizeof(struct statefile_header
)
3905 || shdr
.magic
!= TAGCACHE_STATEFILE_MAGIC
3906 || shdr
.mh
.tch
.magic
!= TAGCACHE_MAGIC
)
3908 logf("incorrect statefile");
3909 ramcache_hdr
= NULL
;
3915 /* Lets allocate real memory and load it */
3916 handle
= core_alloc_ex("tc ramcache", shdr
.tc_stat
.ramcache_allocated
, &ops
);
3917 ramcache_hdr
= core_get_data(handle
);
3919 rc
= read(fd
, ramcache_hdr
, shdr
.tc_stat
.ramcache_allocated
);
3923 if (rc
!= shdr
.tc_stat
.ramcache_allocated
)
3925 logf("read failure!");
3926 ramcache_hdr
= NULL
;
3930 memcpy(&tc_stat
, &shdr
.tc_stat
, sizeof(struct tagcache_stat
));
3932 /* Now fix the pointers */
3933 fix_ramcache(shdr
.hdr
, ramcache_hdr
);
3935 /* Load the tagcache master header (should match the actual DB file header). */
3936 memcpy(¤t_tcmh
, &shdr
.mh
, sizeof current_tcmh
);
3941 static bool tagcache_dumpsave(void)
3943 struct statefile_header shdr
;
3946 if (!tc_stat
.ramcache
)
3949 fd
= open(TAGCACHE_STATEFILE
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
3952 logf("failed to create a statedump");
3956 /* Create the header */
3957 shdr
.magic
= TAGCACHE_STATEFILE_MAGIC
;
3958 shdr
.hdr
= ramcache_hdr
;
3959 memcpy(&shdr
.mh
, ¤t_tcmh
, sizeof current_tcmh
);
3960 memcpy(&shdr
.tc_stat
, &tc_stat
, sizeof tc_stat
);
3961 write(fd
, &shdr
, sizeof shdr
);
3963 /* And dump the data too */
3965 write(fd
, ramcache_hdr
, tc_stat
.ramcache_allocated
);
3973 static bool load_tagcache(void)
3975 struct tagcache_header
*tch
;
3976 struct master_header tcmh
;
3977 long bytesleft
= tc_stat
.ramcache_allocated
- sizeof(struct ramcache_header
);
3978 struct index_entry
*idx
;
3983 # ifdef HAVE_DIRCACHE
3984 while (dircache_is_initializing())
3987 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE
);
3990 logf("loading tagcache to ram...");
3992 fd
= open(TAGCACHE_FILE_MASTER
, O_RDONLY
);
3995 logf("tagcache open failed");
3999 if (ecread(fd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
)
4000 != sizeof(struct master_header
)
4001 || tcmh
.tch
.magic
!= TAGCACHE_MAGIC
)
4003 logf("incorrect header");
4007 /* Master header copy should already match, this can be redundant to do. */
4008 memcpy(¤t_tcmh
, &tcmh
, sizeof current_tcmh
);
4010 move_lock
++; /* lock for the reset of the scan, simpler to handle */
4011 idx
= ramcache_hdr
->indices
;
4013 /* Load the master index table. */
4014 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
++)
4016 bytesleft
-= sizeof(struct index_entry
);
4019 logf("too big tagcache.");
4023 /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture
4024 * may become corrupt. */
4025 rc
= ecread_index_entry(fd
, idx
);
4026 if (rc
!= sizeof(struct index_entry
))
4028 logf("read error #10");
4037 /* Load the tags. */
4039 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
4041 struct tagfile_entry
*fe
;
4042 char buf
[TAG_MAXLEN
+32];
4044 if (TAGCACHE_IS_NUMERIC(tag
))
4047 //p = ((void *)p+1);
4048 p
= (char *)((long)p
& ~0x03) + 0x04;
4049 ramcache_hdr
->tags
[tag
] = p
;
4051 /* Check the header. */
4052 tch
= (struct tagcache_header
*)p
;
4053 p
+= sizeof(struct tagcache_header
);
4055 if ( (fd
= open_tag_fd(tch
, tag
, false)) < 0)
4058 for (ramcache_hdr
->entry_count
[tag
] = 0;
4059 ramcache_hdr
->entry_count
[tag
] < tch
->entry_count
;
4060 ramcache_hdr
->entry_count
[tag
]++)
4064 if (do_timed_yield())
4066 /* Abort if we got a critical event in queue */
4067 if (check_event_queue())
4071 fe
= (struct tagfile_entry
*)p
;
4072 pos
= lseek(fd
, 0, SEEK_CUR
);
4073 rc
= ecread_tagfile_entry(fd
, fe
);
4074 if (rc
!= sizeof(struct tagfile_entry
))
4076 /* End of lookup table. */
4077 logf("read error #11");
4081 /* We have a special handling for the filename tags. */
4082 if (tag
== tag_filename
)
4084 # ifdef HAVE_DIRCACHE
4088 idx
= &ramcache_hdr
->indices
[fe
->idx_id
];
4090 if (fe
->tag_length
>= (long)sizeof(buf
)-1)
4094 logf("TAG:%s", buf
);
4095 logf("too long filename");
4099 rc
= read(fd
, buf
, fe
->tag_length
);
4100 if (rc
!= fe
->tag_length
)
4102 logf("read error #12");
4106 /* Check if the entry has already been removed */
4107 if (idx
->flag
& FLAG_DELETED
)
4110 /* This flag must not be used yet. */
4111 if (idx
->flag
& FLAG_DIRCACHE
)
4113 logf("internal error!");
4117 if (idx
->tag_seek
[tag
] != pos
)
4119 logf("corrupt data structures!");
4123 # ifdef HAVE_DIRCACHE
4124 if (dircache_is_enabled())
4126 dc
= dircache_get_entry_id(buf
);
4129 logf("Entry no longer valid.");
4131 if (global_settings
.tagcache_autoupdate
)
4132 delete_entry(fe
->idx_id
);
4136 idx
->flag
|= FLAG_DIRCACHE
;
4137 idx
->tag_seek
[tag_filename
] = dc
;
4142 /* This will be very slow unless dircache is enabled
4143 or target is flash based, but do it anyway for
4145 /* Check if entry has been removed. */
4146 if (global_settings
.tagcache_autoupdate
)
4148 if (!file_exists(buf
))
4150 logf("Entry no longer valid.");
4152 delete_entry(fe
->idx_id
);
4161 bytesleft
-= sizeof(struct tagfile_entry
) + fe
->tag_length
;
4164 logf("too big tagcache #2");
4165 logf("tl: %ld", fe
->tag_length
);
4166 logf("bl: %ld", bytesleft
);
4171 rc
= read(fd
, fe
->tag_data
, fe
->tag_length
);
4174 if (rc
!= fe
->tag_length
)
4176 logf("read error #13");
4177 logf("rc=0x%04x", rc
); // 0x431
4178 logf("len=0x%04lx", fe
->tag_length
); // 0x4000
4179 logf("pos=0x%04lx", lseek(fd
, 0, SEEK_CUR
)); // 0x433
4180 logf("tag=0x%02x", tag
); // 0x00
4187 tc_stat
.ramcache_used
= tc_stat
.ramcache_allocated
- bytesleft
;
4188 logf("tagcache loaded into ram!");
4199 #endif /* HAVE_TC_RAMCACHE */
4201 static bool check_deleted_files(void)
4204 char buf
[TAG_MAXLEN
+32];
4205 struct tagfile_entry tfe
;
4207 logf("reverse scan...");
4208 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag_filename
);
4209 fd
= open(buf
, O_RDONLY
);
4213 logf("%s open fail", buf
);
4217 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
4218 while (ecread_tagfile_entry(fd
, &tfe
) == sizeof(struct tagfile_entry
)
4220 && !check_event_queue()
4224 if (tfe
.tag_length
>= (long)sizeof(buf
)-1)
4226 logf("too long tag");
4231 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
4233 logf("read error #14");
4238 /* Check if the file has already deleted from the db. */
4242 /* Now check if the file exists. */
4243 if (!file_exists(buf
))
4245 logf("Entry no longer valid.");
4246 logf("-> %s / %ld", buf
, tfe
.tag_length
);
4247 delete_entry(tfe
.idx_id
);
4259 /* Note that this function must not be inlined, otherwise the whole point
4260 * of having the code in a separate function is lost.
4262 static void __attribute__ ((noinline
)) check_ignore(const char *dirname
,
4263 int *ignore
, int *unignore
)
4265 char newpath
[MAX_PATH
];
4267 /* check for a database.ignore file */
4268 snprintf(newpath
, MAX_PATH
, "%s/database.ignore", dirname
);
4269 *ignore
= file_exists(newpath
);
4270 /* check for a database.unignore file */
4271 snprintf(newpath
, MAX_PATH
, "%s/database.unignore", dirname
);
4272 *unignore
= file_exists(newpath
);
4275 static struct search_roots_ll
{
4277 struct search_roots_ll
* next
;
4282 * This adds a path to the search roots, possibly during traveling through
4283 * the filesystem. It only adds if the path is not inside an already existing
4286 * Returns true if it added the path to the search roots
4288 * Windows 2000 and greater supports symlinks, but they don't provide
4289 * realpath() or readlink(), and symlinks are rarely used on them so
4290 * ignore this for windows for now
4292 static bool add_search_root(const char *name
)
4296 struct search_roots_ll
*this, *prev
= NULL
;
4297 char target
[MAX_PATH
];
4298 /* Okay, realpath() is almost completely broken on android
4300 * It doesn't accept NULL for resolved_name to dynamically allocate
4301 * the resulting path; and it assumes resolved_name to be PATH_MAX
4302 * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4303 * and blindly writes to the end if it
4305 * therefore use sufficiently large static storage here
4306 * Note that PATH_MAX != MAX_PATH
4308 static char abs_target
[PATH_MAX
];
4311 len
= readlink(name
, target
, sizeof(target
));
4316 if (realpath(target
, abs_target
) == NULL
)
4319 for(this = &roots_ll
; this; prev
= this, this = this->next
)
4321 size_t root_len
= strlen(this->path
);
4322 /* check if the link target is inside of an existing search root
4323 * don't add if target is inside, we'll scan it later */
4324 if (!strncmp(this->path
, abs_target
, root_len
))
4330 size_t len
= strlen(abs_target
) + 1; /* count \0 */
4331 this = malloc(sizeof(struct search_roots_ll
) + len
);
4332 if (!this || len
> MAX_PATH
)
4334 logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4339 this->path
= ((char*)this) + sizeof(struct search_roots_ll
);
4340 strcpy((char*)this->path
, abs_target
); /* ok to cast const away here */
4343 logf("Added %s to the search roots\n", abs_target
);
4350 static int free_search_roots(struct search_roots_ll
* start
)
4355 ret
+= free_search_roots(start
->next
);
4356 ret
+= sizeof(struct search_roots_ll
);
4361 #else /* native, simulator */
4362 #define add_search_root(a) do {} while(0)
4363 #define free_search_roots(a) do {} while(0)
4366 static bool check_dir(const char *dirname
, int add_files
)
4370 int success
= false;
4371 int ignore
, unignore
;
4373 dir
= opendir(dirname
);
4376 logf("tagcache: opendir(%s) failed", dirname
);
4379 /* check for a database.ignore and database.unignore */
4380 check_ignore(dirname
, &ignore
, &unignore
);
4382 /* don't do anything if both ignore and unignore are there */
4383 if (ignore
!= unignore
)
4384 add_files
= unignore
;
4386 /* Recursively scan the dir. */
4390 while (!check_event_queue())
4393 struct dirent
*entry
= readdir(dir
);
4400 if (!strcmp((char *)entry
->d_name
, ".") ||
4401 !strcmp((char *)entry
->d_name
, ".."))
4404 struct dirinfo info
= dir_get_info(dir
, entry
);
4408 len
= strlen(curpath
);
4409 /* don't add an extra / for curpath == / */
4410 if (len
<= 1) len
= 0;
4411 snprintf(&curpath
[len
], sizeof(curpath
) - len
, "/%s", entry
->d_name
);
4413 processed_dir_count
++;
4414 if (info
.attribute
& ATTR_DIRECTORY
)
4416 { /* don't follow symlinks to dirs, but try to add it as a search root
4417 * this makes able to avoid looping in recursive symlinks */
4418 if (info
.attribute
& ATTR_LINK
)
4419 add_search_root(curpath
);
4421 check_dir(curpath
, add_files
);
4424 check_dir(curpath
, add_files
);
4428 tc_stat
.curentry
= curpath
;
4430 /* Add a new entry to the temporary db file. */
4431 add_tagcache(curpath
, (info
.wrtdate
<< 16) | info
.wrttime
4432 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4433 , dir
->internal_entry
4437 /* Wait until current path for debug screen is read and unset. */
4438 while (tc_stat
.syncscreen
&& tc_stat
.curentry
!= NULL
)
4441 tc_stat
.curentry
= NULL
;
4444 curpath
[len
] = '\0';
4452 void tagcache_screensync_event(void)
4454 tc_stat
.curentry
= NULL
;
4457 void tagcache_screensync_enable(bool state
)
4459 tc_stat
.syncscreen
= state
;
4462 void tagcache_build(const char *path
)
4464 struct tagcache_header header
;
4469 total_entry_count
= 0;
4470 processed_dir_count
= 0;
4472 #ifdef HAVE_DIRCACHE
4473 while (dircache_is_initializing())
4477 logf("updating tagcache");
4479 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
4482 logf("skipping, cache already waiting for commit");
4487 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDWR
| O_CREAT
| O_TRUNC
, 0666);
4490 logf("master file open failed: %s", TAGCACHE_FILE_TEMP
);
4494 filenametag_fd
= open_tag_fd(&header
, tag_filename
, false);
4498 logf("Scanning files...");
4499 /* Scan for new files. */
4500 memset(&header
, 0, sizeof(struct tagcache_header
));
4501 write(cachefd
, &header
, sizeof(struct tagcache_header
));
4504 roots_ll
.path
= path
;
4505 roots_ll
.next
= NULL
;
4506 struct search_roots_ll
* this;
4507 /* check_dir might add new roots */
4508 for(this = &roots_ll
; this; this = this->next
)
4510 strcpy(curpath
, this->path
);
4511 ret
= ret
&& check_dir(this->path
, true);
4514 free_search_roots(roots_ll
.next
);
4516 /* Write the header. */
4517 header
.magic
= TAGCACHE_MAGIC
;
4518 header
.datasize
= data_size
;
4519 header
.entry_count
= total_entry_count
;
4520 lseek(cachefd
, 0, SEEK_SET
);
4521 write(cachefd
, &header
, sizeof(struct tagcache_header
));
4524 if (filenametag_fd
>= 0)
4526 close(filenametag_fd
);
4527 filenametag_fd
= -1;
4537 /* Commit changes to the database. */
4543 logf("tagcache built!");
4549 #ifdef HAVE_TC_RAMCACHE
4552 /* Import runtime statistics if we just initialized the db. */
4553 if (current_tcmh
.serial
== 0)
4554 queue_post(&tagcache_queue
, Q_IMPORT_CHANGELOG
, 0);
4561 #ifdef HAVE_TC_RAMCACHE
4562 static void load_ramcache(void)
4569 /* At first we should load the cache (if exists). */
4570 tc_stat
.ramcache
= load_tagcache();
4572 if (!tc_stat
.ramcache
)
4574 /* If loading failed, it must indicate some problem with the db
4575 * so disable it entirely to prevent further issues. */
4576 tc_stat
.ready
= false;
4577 ramcache_hdr
= NULL
;
4583 void tagcache_unload_ramcache(void)
4585 tc_stat
.ramcache
= false;
4586 /* Just to make sure there is no statefile present. */
4587 // remove(TAGCACHE_STATEFILE);
4592 static void tagcache_thread(void)
4594 struct queue_event ev
;
4595 bool check_done
= false;
4597 /* If the previous cache build/update was interrupted, commit
4598 * the changes first in foreground. */
4604 #ifdef HAVE_TC_RAMCACHE
4605 # ifdef HAVE_EEPROM_SETTINGS
4606 if (firmware_settings
.initialized
&& firmware_settings
.disk_clean
4607 && global_settings
.tagcache_ram
)
4609 check_done
= tagcache_dumpload();
4612 remove(TAGCACHE_STATEFILE
);
4615 /* Allocate space for the tagcache if found on disk. */
4616 if (global_settings
.tagcache_ram
&& !tc_stat
.ramcache
)
4617 allocate_tagcache();
4621 tc_stat
.initialized
= true;
4623 /* Don't delay bootup with the header check but do it on background. */
4627 tc_stat
.ready
= check_all_headers();
4628 tc_stat
.readyvalid
= true;
4633 run_command_queue(false);
4635 queue_wait_w_tmo(&tagcache_queue
, &ev
, HZ
);
4639 case Q_IMPORT_CHANGELOG
:
4640 tagcache_import_changelog();
4645 remove(TAGCACHE_FILE_TEMP
);
4646 tagcache_build("/");
4650 tagcache_build("/");
4651 #ifdef HAVE_TC_RAMCACHE
4654 check_deleted_files();
4660 if (check_done
|| !tc_stat
.ready
)
4663 #ifdef HAVE_TC_RAMCACHE
4664 if (!tc_stat
.ramcache
&& global_settings
.tagcache_ram
)
4667 if (tc_stat
.ramcache
&& global_settings
.tagcache_autoupdate
)
4668 tagcache_build("/");
4672 if (global_settings
.tagcache_autoupdate
)
4674 tagcache_build("/");
4676 /* This will be very slow unless dircache is enabled
4677 or target is flash based, but do it anyway for
4679 check_deleted_files();
4682 logf("tagcache check done");
4693 case SYS_USB_CONNECTED
:
4694 logf("USB: TagCache");
4695 usb_acknowledge(SYS_USB_CONNECTED_ACK
);
4696 usb_wait_for_disconnect(&tagcache_queue
);
4702 bool tagcache_prepare_shutdown(void)
4704 if (tagcache_get_commit_step() > 0)
4707 tagcache_stop_scan();
4708 while (read_lock
|| write_lock
)
4714 void tagcache_shutdown(void)
4716 /* Flush the command queue. */
4717 run_command_queue(true);
4719 #ifdef HAVE_EEPROM_SETTINGS
4720 if (tc_stat
.ramcache
)
4721 tagcache_dumpsave();
4725 static int get_progress(void)
4727 int total_count
= -1;
4729 #ifdef HAVE_DIRCACHE
4730 if (dircache_is_enabled())
4732 total_count
= dircache_get_entry_count();
4736 #ifdef HAVE_TC_RAMCACHE
4738 if (ramcache_hdr
&& tc_stat
.ramcache
)
4739 total_count
= current_tcmh
.tch
.entry_count
;
4743 if (total_count
< 0)
4746 return processed_dir_count
* 100 / total_count
;
4749 struct tagcache_stat
* tagcache_get_stat(void)
4751 tc_stat
.progress
= get_progress();
4752 tc_stat
.processed_entries
= processed_dir_count
;
4757 void tagcache_start_scan(void)
4759 queue_post(&tagcache_queue
, Q_START_SCAN
, 0);
4762 bool tagcache_update(void)
4767 queue_post(&tagcache_queue
, Q_UPDATE
, 0);
4771 bool tagcache_rebuild()
4773 queue_post(&tagcache_queue
, Q_REBUILD
, 0);
4777 void tagcache_stop_scan(void)
4779 queue_post(&tagcache_queue
, Q_STOP_SCAN
, 0);
4782 #endif /* !__PCTOOL__ */
4785 void tagcache_init(void)
4787 memset(&tc_stat
, 0, sizeof(struct tagcache_stat
));
4788 memset(¤t_tcmh
, 0, sizeof(struct master_header
));
4789 filenametag_fd
= -1;
4790 write_lock
= read_lock
= 0;
4793 mutex_init(&command_queue_mutex
);
4794 queue_init(&tagcache_queue
, true);
4795 create_thread(tagcache_thread
, tagcache_stack
,
4796 sizeof(tagcache_stack
), 0, tagcache_thread_name
4797 IF_PRIO(, PRIORITY_BACKGROUND
)
4800 tc_stat
.initialized
= true;
4804 tc_stat
.ready
= check_all_headers();
4809 void tagcache_reverse_scan(void)
4811 logf("Checking for deleted files");
4812 check_deleted_files();
4816 bool tagcache_is_initialized(void)
4818 return tc_stat
.initialized
;
4820 bool tagcache_is_fully_initialized(void)
4822 return tc_stat
.readyvalid
;
4824 bool tagcache_is_usable(void)
4826 return tc_stat
.initialized
&& tc_stat
.ready
;
4828 int tagcache_get_commit_step(void)
4830 return tc_stat
.commit_step
;
4832 int tagcache_get_max_commit_step(void)
4834 return (int)(SORTED_TAGS_COUNT
)+1;