1 /***************************************************************************
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 * Copyright (C) 2005 by Miika Pekkarinen
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
18 ****************************************************************************/
23 * ----------x---------x------------------x-----
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
60 #include "ata_idle_notify.h"
80 #include "eeprom_settings.h"
84 #define yield() do { } while(0)
85 #define sim_sleep(timeout) do { } while(0)
86 #define do_timed_yield() do { } while(0)
90 /* Tag Cache thread. */
91 static struct event_queue tagcache_queue
;
92 static long tagcache_stack
[(DEFAULT_STACK_SIZE
+ 0x4000)/sizeof(long)];
93 static const char tagcache_thread_name
[] = "tagcache";
96 #define UNTAGGED "<Untagged>"
98 /* Previous path when scanning directory tree recursively. */
99 static char curpath
[TAG_MAXLEN
+32];
100 static long curpath_size
= sizeof(curpath
);
102 /* Used when removing duplicates. */
103 static char *tempbuf
; /* Allocated when needed. */
104 static long tempbufidx
; /* Current location in buffer. */
105 static long tempbuf_size
; /* Buffer size (TEMPBUF_SIZE). */
106 static long tempbuf_left
; /* Buffer space left. */
107 static long tempbuf_pos
;
109 /* Tags we want to get sorted (loaded to the tempbuf). */
110 static const int sorted_tags
[] = { tag_artist
, tag_album
, tag_genre
,
111 tag_composer
, tag_comment
, tag_albumartist
, tag_grouping
, tag_title
};
113 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
114 static const int unique_tags
[] = { tag_artist
, tag_album
, tag_genre
,
115 tag_composer
, tag_comment
, tag_albumartist
, tag_grouping
};
117 /* Numeric tags (we can use these tags with conditional clauses). */
118 static const int numeric_tags
[] = { tag_year
, tag_discnumber
,
119 tag_tracknumber
, tag_length
, tag_bitrate
, tag_playcount
, tag_rating
,
120 tag_playtime
, tag_lastplayed
, tag_commitid
, tag_mtime
,
121 tag_virt_length_min
, tag_virt_length_sec
,
122 tag_virt_playtime_min
, tag_virt_playtime_sec
,
123 tag_virt_entryage
, tag_virt_autoscore
};
125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
126 static const char *tags_str
[] = { "artist", "album", "genre", "title",
127 "filename", "composer", "comment", "albumartist", "grouping", "year",
128 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
129 "playtime", "lastplayed", "commitid", "mtime" };
131 /* Status information of the tagcache. */
132 static struct tagcache_stat tc_stat
;
134 /* Queue commands. */
135 enum tagcache_queue
{
142 /* Internal tagcache command queue. */
143 CMD_UPDATE_MASTER_HEADER
,
147 struct tagcache_command_entry
{
154 static struct tagcache_command_entry command_queue
[TAGCACHE_COMMAND_QUEUE_LENGTH
];
155 static volatile int command_queue_widx
= 0;
156 static volatile int command_queue_ridx
= 0;
157 static struct mutex command_queue_mutex
;
159 /* Tag database structures. */
161 /* Variable-length tag entry in tag files. */
162 struct tagfile_entry
{
163 short tag_length
; /* Length of the data in bytes including '\0' */
164 short idx_id
; /* Corresponding entry location in index file of not unique tags */
165 char tag_data
[0]; /* Begin of the tag data */
168 /* Fixed-size tag entry in master db index. */
170 int32_t tag_seek
[TAG_COUNT
]; /* Location of tag data or numeric tag data */
171 int32_t flag
; /* Status flags */
174 /* Header is the same in every file. */
175 struct tagcache_header
{
176 int32_t magic
; /* Header version number */
177 int32_t datasize
; /* Data size in bytes */
178 int32_t entry_count
; /* Number of entries in this file */
181 struct master_header
{
182 struct tagcache_header tch
;
183 int32_t serial
; /* Increasing counting number */
184 int32_t commitid
; /* Number of commits so far */
188 /* For the endianess correction */
189 static const char *tagfile_entry_ec
= "ss";
190 static const char *index_entry_ec
= "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
191 static const char *tagcache_header_ec
= "lll";
192 static const char *master_header_ec
= "llllll";
194 static struct master_header current_tcmh
;
196 #ifdef HAVE_TC_RAMCACHE
197 /* Header is created when loading database to ram. */
198 struct ramcache_header
{
199 struct master_header h
; /* Header from the master index */
200 struct index_entry
*indices
; /* Master index file content */
201 char *tags
[TAG_COUNT
]; /* Tag file content (not including filename tag) */
202 int entry_count
[TAG_COUNT
]; /* Number of entries in the indices. */
205 # ifdef HAVE_EEPROM_SETTINGS
206 struct statefile_header
{
207 struct ramcache_header
*hdr
;
208 struct tagcache_stat tc_stat
;
212 /* Pointer to allocated ramcache_header */
213 static struct ramcache_header
*hdr
;
217 * Full tag entries stored in a temporary file waiting
218 * for commit to the cache. */
219 struct temp_file_entry
{
220 long tag_offset
[TAG_COUNT
];
221 short tag_length
[TAG_COUNT
];
227 struct tempbuf_id_list
{
229 struct tempbuf_id_list
*next
;
232 struct tempbuf_searchidx
{
236 struct tempbuf_id_list idlist
;
239 /* Lookup buffer for fixing messed up index while after sorting. */
240 static long commit_entry_count
;
241 static long lookup_buffer_depth
;
242 static struct tempbuf_searchidx
**lookup
;
244 /* Used when building the temporary file. */
245 static int cachefd
= -1, filenametag_fd
;
246 static int total_entry_count
= 0;
247 static int data_size
= 0;
248 static int processed_dir_count
;
250 /* Thread safe locking */
251 static volatile int write_lock
;
252 static volatile int read_lock
;
254 static bool delete_entry(long idx_id
);
256 const char* tagcache_tag_to_str(int tag
)
258 return tags_str
[tag
];
261 bool tagcache_is_numeric_tag(int type
)
265 for (i
= 0; i
< (int)(sizeof(numeric_tags
)/sizeof(numeric_tags
[0])); i
++)
267 if (type
== numeric_tags
[i
])
274 bool tagcache_is_unique_tag(int type
)
278 for (i
= 0; i
< (int)(sizeof(unique_tags
)/sizeof(unique_tags
[0])); i
++)
280 if (type
== unique_tags
[i
])
287 bool tagcache_is_sorted_tag(int type
)
291 for (i
= 0; i
< (int)(sizeof(sorted_tags
)/sizeof(sorted_tags
[0])); i
++)
293 if (type
== sorted_tags
[i
])
302 * Returns true if specified flag is still present, i.e., dircache
303 * has not been reloaded.
305 static bool is_dircache_intact(void)
307 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE
);
311 static int open_tag_fd(struct tagcache_header
*hdr
, int tag
, bool write
)
317 if (tagcache_is_numeric_tag(tag
) || tag
< 0 || tag
>= TAG_COUNT
)
320 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag
);
322 fd
= open(buf
, write
? O_RDWR
: O_RDONLY
);
325 logf("tag file open failed: tag=%d write=%d file=%s", tag
, write
, buf
);
326 tc_stat
.ready
= false;
330 /* Check the header. */
331 rc
= ecread(fd
, hdr
, 1, tagcache_header_ec
, tc_stat
.econ
);
332 if (hdr
->magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct tagcache_header
))
334 logf("header error");
335 tc_stat
.ready
= false;
343 static int open_master_fd(struct master_header
*hdr
, bool write
)
348 fd
= open(TAGCACHE_FILE_MASTER
, write
? O_RDWR
: O_RDONLY
);
351 logf("master file open failed for R/W");
352 tc_stat
.ready
= false;
356 tc_stat
.econ
= false;
358 /* Check the header. */
359 rc
= read(fd
, hdr
, sizeof(struct master_header
));
360 if (hdr
->tch
.magic
== TAGCACHE_MAGIC
&& rc
== sizeof(struct master_header
))
366 /* Trying to read again, this time with endianess correction enabled. */
367 lseek(fd
, 0, SEEK_SET
);
369 rc
= ecread(fd
, hdr
, 1, master_header_ec
, true);
370 if (hdr
->tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct master_header
))
372 logf("header error");
373 tc_stat
.ready
= false;
384 static bool do_timed_yield(void)
386 /* Sorting can lock up for quite a while, so yield occasionally */
387 static long wakeup_tick
= 0;
388 if (current_tick
>= wakeup_tick
)
390 wakeup_tick
= current_tick
+ (HZ
/4);
398 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
399 static long find_entry_ram(const char *filename
,
400 const struct dirent
*dc
)
402 static long last_pos
= 0;
405 /* Check if we tagcache is loaded into ram. */
406 if (!tc_stat
.ramcache
)
410 dc
= dircache_get_entry_ptr(filename
);
414 logf("tagcache: file not found.");
425 for (; i
< hdr
->h
.tch
.entry_count
; i
++)
427 if (hdr
->indices
[i
].tag_seek
[tag_filename
] == (long)dc
)
429 last_pos
= MAX(0, i
- 3);
446 static long find_entry_disk(const char *filename
)
448 struct tagcache_header tch
;
449 static long last_pos
= -1;
450 long pos_history
[POS_HISTORY_COUNT
];
451 long pos_history_idx
= 0;
453 struct tagfile_entry tfe
;
455 char buf
[TAG_MAXLEN
+32];
466 if ( (fd
= open_tag_fd(&tch
, tag_filename
, false)) < 0)
473 lseek(fd
, last_pos
, SEEK_SET
);
475 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
479 pos
= lseek(fd
, 0, SEEK_CUR
);
480 for (i
= pos_history_idx
-1; i
>= 0; i
--)
481 pos_history
[i
+1] = pos_history
[i
];
482 pos_history
[0] = pos
;
484 if (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
485 != sizeof(struct tagfile_entry
))
490 if (tfe
.tag_length
>= (long)sizeof(buf
))
492 logf("too long tag #1");
498 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
500 logf("read error #2");
506 if (!strcasecmp(filename
, buf
))
508 last_pos
= pos_history
[pos_history_idx
];
513 if (pos_history_idx
< POS_HISTORY_COUNT
- 1)
527 if (fd
!= filenametag_fd
)
532 if (fd
!= filenametag_fd
)
538 static int find_index(const char *filename
)
542 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
543 if (tc_stat
.ramcache
&& is_dircache_intact())
544 idx_id
= find_entry_ram(filename
, NULL
);
548 idx_id
= find_entry_disk(filename
);
553 bool tagcache_find_index(struct tagcache_search
*tcs
, const char *filename
)
560 idx_id
= find_index(filename
);
564 if (!tagcache_search(tcs
, tag_filename
))
567 tcs
->entry_count
= 0;
568 tcs
->idx_id
= idx_id
;
573 static bool get_index(int masterfd
, int idxid
,
574 struct index_entry
*idx
, bool use_ram
)
576 bool localfd
= false;
580 logf("Incorrect idxid: %d", idxid
);
584 #ifdef HAVE_TC_RAMCACHE
585 if (tc_stat
.ramcache
&& use_ram
)
587 if (hdr
->indices
[idxid
].flag
& FLAG_DELETED
)
590 memcpy(idx
, &hdr
->indices
[idxid
], sizeof(struct index_entry
));
599 struct master_header tcmh
;
602 masterfd
= open_master_fd(&tcmh
, false);
607 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
608 + sizeof(struct master_header
), SEEK_SET
);
609 if (ecread(masterfd
, idx
, 1, index_entry_ec
, tc_stat
.econ
)
610 != sizeof(struct index_entry
))
612 logf("read error #3");
622 if (idx
->flag
& FLAG_DELETED
)
628 static bool write_index(int masterfd
, int idxid
, struct index_entry
*idx
)
630 /* We need to exclude all memory only flags & tags when writing to disk. */
631 if (idx
->flag
& FLAG_DIRCACHE
)
633 logf("memory only flags!");
637 #ifdef HAVE_TC_RAMCACHE
638 /* Only update numeric data. Writing the whole index to RAM by memcpy
639 * destroys dircache pointers!
641 if (tc_stat
.ramcache
)
644 struct index_entry
*idx_ram
= &hdr
->indices
[idxid
];
646 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
648 if (tagcache_is_numeric_tag(tag
))
650 idx_ram
->tag_seek
[tag
] = idx
->tag_seek
[tag
];
654 /* Don't touch the dircache flag. */
655 idx_ram
->flag
= idx
->flag
| (idx_ram
->flag
& FLAG_DIRCACHE
);
659 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
660 + sizeof(struct master_header
), SEEK_SET
);
661 if (ecwrite(masterfd
, idx
, 1, index_entry_ec
, tc_stat
.econ
)
662 != sizeof(struct index_entry
))
664 logf("write error #3");
665 logf("idxid: %d", idxid
);
672 static bool open_files(struct tagcache_search
*tcs
, int tag
)
674 if (tcs
->idxfd
[tag
] < 0)
678 snprintf(fn
, sizeof fn
, TAGCACHE_FILE_INDEX
, tag
);
679 tcs
->idxfd
[tag
] = open(fn
, O_RDONLY
);
682 if (tcs
->idxfd
[tag
] < 0)
684 logf("File not open!");
691 static bool retrieve(struct tagcache_search
*tcs
, struct index_entry
*idx
,
692 int tag
, char *buf
, long size
)
694 struct tagfile_entry tfe
;
699 if (tagcache_is_numeric_tag(tag
))
702 seek
= idx
->tag_seek
[tag
];
705 logf("Retrieve failed");
709 #ifdef HAVE_TC_RAMCACHE
712 struct tagfile_entry
*ep
;
714 # ifdef HAVE_DIRCACHE
715 if (tag
== tag_filename
&& (idx
->flag
& FLAG_DIRCACHE
)
716 && is_dircache_intact())
718 dircache_copy_path((struct dirent
*)seek
,
724 if (tag
!= tag_filename
)
726 ep
= (struct tagfile_entry
*)&hdr
->tags
[tag
][seek
];
727 strncpy(buf
, ep
->tag_data
, size
-1);
734 if (!open_files(tcs
, tag
))
737 lseek(tcs
->idxfd
[tag
], seek
, SEEK_SET
);
738 if (ecread(tcs
->idxfd
[tag
], &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
739 != sizeof(struct tagfile_entry
))
741 logf("read error #5");
745 if (tfe
.tag_length
>= size
)
747 logf("too small buffer");
751 if (read(tcs
->idxfd
[tag
], buf
, tfe
.tag_length
) !=
754 logf("read error #6");
758 buf
[tfe
.tag_length
] = '\0';
763 static long check_virtual_tags(int tag
, const struct index_entry
*idx
)
769 case tag_virt_length_sec
:
770 data
= (idx
->tag_seek
[tag_length
]/1000) % 60;
773 case tag_virt_length_min
:
774 data
= (idx
->tag_seek
[tag_length
]/1000) / 60;
777 case tag_virt_playtime_sec
:
778 data
= (idx
->tag_seek
[tag_playtime
]/1000) % 60;
781 case tag_virt_playtime_min
:
782 data
= (idx
->tag_seek
[tag_playtime
]/1000) / 60;
785 case tag_virt_autoscore
:
786 if (idx
->tag_seek
[tag_length
] == 0
787 || idx
->tag_seek
[tag_playcount
] == 0)
793 data
= 100 * idx
->tag_seek
[tag_playtime
]
794 / idx
->tag_seek
[tag_length
]
795 / idx
->tag_seek
[tag_playcount
];
799 /* How many commits before the file has been added to the DB. */
800 case tag_virt_entryage
:
801 data
= current_tcmh
.commitid
- idx
->tag_seek
[tag_commitid
] - 1;
805 data
= idx
->tag_seek
[tag
];
811 long tagcache_get_numeric(const struct tagcache_search
*tcs
, int tag
)
813 struct index_entry idx
;
818 if (!tagcache_is_numeric_tag(tag
))
821 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
824 return check_virtual_tags(tag
, &idx
);
827 inline static bool str_ends_with(const char *str1
, const char *str2
)
829 int str_len
= strlen(str1
);
830 int clause_len
= strlen(str2
);
832 if (clause_len
> str_len
)
835 return !strcasecmp(&str1
[str_len
- clause_len
], str2
);
838 inline static bool str_oneof(const char *str
, const char *list
)
841 int l
, len
= strlen(str
);
845 sep
= strchr(list
, '|');
846 l
= sep
? (long)sep
- (long)list
: (int)strlen(list
);
847 if ((l
==len
) && !strncasecmp(str
, list
, len
))
849 list
+= sep
? l
+ 1 : l
;
855 static bool check_against_clause(long numeric
, const char *str
,
856 const struct tagcache_search_clause
*clause
)
860 switch (clause
->type
)
863 return numeric
== clause
->numeric_data
;
865 return numeric
!= clause
->numeric_data
;
867 return numeric
> clause
->numeric_data
;
869 return numeric
>= clause
->numeric_data
;
871 return numeric
< clause
->numeric_data
;
873 return numeric
<= clause
->numeric_data
;
875 logf("Incorrect numeric tag: %d", clause
->type
);
880 switch (clause
->type
)
883 return !strcasecmp(clause
->str
, str
);
885 return strcasecmp(clause
->str
, str
);
887 return 0>strcasecmp(clause
->str
, str
);
889 return 0>=strcasecmp(clause
->str
, str
);
891 return 0<strcasecmp(clause
->str
, str
);
893 return 0<=strcasecmp(clause
->str
, str
);
894 case clause_contains
:
895 return (strcasestr(str
, clause
->str
) != NULL
);
896 case clause_not_contains
:
897 return (strcasestr(str
, clause
->str
) == NULL
);
898 case clause_begins_with
:
899 return (strcasestr(str
, clause
->str
) == str
);
900 case clause_not_begins_with
:
901 return (strcasestr(str
, clause
->str
) != str
);
902 case clause_ends_with
:
903 return str_ends_with(str
, clause
->str
);
904 case clause_not_ends_with
:
905 return !str_ends_with(str
, clause
->str
);
907 return str_oneof(str
, clause
->str
);
910 logf("Incorrect tag: %d", clause
->type
);
917 static bool check_clauses(struct tagcache_search
*tcs
,
918 struct index_entry
*idx
,
919 struct tagcache_search_clause
**clause
, int count
)
923 #ifdef HAVE_TC_RAMCACHE
926 /* Go through all conditional clauses. */
927 for (i
= 0; i
< count
; i
++)
929 struct tagfile_entry
*tfe
;
934 seek
= check_virtual_tags(clause
[i
]->tag
, idx
);
936 if (!tagcache_is_numeric_tag(clause
[i
]->tag
))
938 if (clause
[i
]->tag
== tag_filename
)
940 retrieve(tcs
, idx
, tag_filename
, buf
, sizeof buf
);
945 tfe
= (struct tagfile_entry
*)&hdr
->tags
[clause
[i
]->tag
][seek
];
950 if (!check_against_clause(seek
, str
, clause
[i
]))
957 /* Check for conditions. */
958 for (i
= 0; i
< count
; i
++)
960 struct tagfile_entry tfe
;
964 seek
= check_virtual_tags(clause
[i
]->tag
, idx
);
966 memset(str
, 0, sizeof str
);
967 if (!tagcache_is_numeric_tag(clause
[i
]->tag
))
969 int fd
= tcs
->idxfd
[clause
[i
]->tag
];
970 lseek(fd
, seek
, SEEK_SET
);
971 ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
972 if (tfe
.tag_length
>= (int)sizeof(str
))
974 logf("Too long tag read!");
978 read(fd
, str
, tfe
.tag_length
);
980 /* Check if entry has been deleted. */
985 if (!check_against_clause(seek
, str
, clause
[i
]))
993 bool tagcache_check_clauses(struct tagcache_search
*tcs
,
994 struct tagcache_search_clause
**clause
, int count
)
996 struct index_entry idx
;
1001 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
1004 return check_clauses(tcs
, &idx
, clause
, count
);
1007 static bool add_uniqbuf(struct tagcache_search
*tcs
, unsigned long id
)
1011 /* If uniq buffer is not defined we must return true for search to work. */
1012 if (tcs
->unique_list
== NULL
1013 || (!tagcache_is_unique_tag(tcs
->type
)
1014 && !tagcache_is_numeric_tag(tcs
->type
)))
1019 for (i
= 0; i
< tcs
->unique_list_count
; i
++)
1021 /* Return false if entry is found. */
1022 if (tcs
->unique_list
[i
] == id
)
1026 if (tcs
->unique_list_count
< tcs
->unique_list_capacity
)
1028 tcs
->unique_list
[i
] = id
;
1029 tcs
->unique_list_count
++;
1035 static bool build_lookup_list(struct tagcache_search
*tcs
)
1037 struct index_entry entry
;
1040 tcs
->seek_list_count
= 0;
1042 #ifdef HAVE_TC_RAMCACHE
1047 for (i
= tcs
->seek_pos
; i
< hdr
->h
.tch
.entry_count
; i
++)
1049 struct index_entry
*idx
= &hdr
->indices
[i
];
1050 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
1053 /* Skip deleted files. */
1054 if (idx
->flag
& FLAG_DELETED
)
1057 /* Go through all filters.. */
1058 for (j
= 0; j
< tcs
->filter_count
; j
++)
1060 if (idx
->tag_seek
[tcs
->filter_tag
[j
]] != tcs
->filter_seek
[j
])
1066 if (j
< tcs
->filter_count
)
1069 /* Check for conditions. */
1070 if (!check_clauses(tcs
, idx
, tcs
->clause
, tcs
->clause_count
))
1073 /* Add to the seek list if not already in uniq buffer. */
1074 if (!add_uniqbuf(tcs
, idx
->tag_seek
[tcs
->type
]))
1078 tcs
->seek_list
[tcs
->seek_list_count
] = idx
->tag_seek
[tcs
->type
];
1079 tcs
->seek_flags
[tcs
->seek_list_count
] = idx
->flag
;
1080 tcs
->seek_list_count
++;
1085 return tcs
->seek_list_count
> 0;
1089 lseek(tcs
->masterfd
, tcs
->seek_pos
* sizeof(struct index_entry
) +
1090 sizeof(struct master_header
), SEEK_SET
);
1092 while (ecread(tcs
->masterfd
, &entry
, 1, index_entry_ec
, tc_stat
.econ
)
1093 == sizeof(struct index_entry
))
1095 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
1100 /* Check if entry has been deleted. */
1101 if (entry
.flag
& FLAG_DELETED
)
1104 /* Go through all filters.. */
1105 for (i
= 0; i
< tcs
->filter_count
; i
++)
1107 if (entry
.tag_seek
[tcs
->filter_tag
[i
]] != tcs
->filter_seek
[i
])
1111 if (i
< tcs
->filter_count
)
1114 /* Check for conditions. */
1115 if (!check_clauses(tcs
, &entry
, tcs
->clause
, tcs
->clause_count
))
1118 /* Add to the seek list if not already in uniq buffer. */
1119 if (!add_uniqbuf(tcs
, entry
.tag_seek
[tcs
->type
]))
1123 tcs
->seek_list
[tcs
->seek_list_count
] = entry
.tag_seek
[tcs
->type
];
1124 tcs
->seek_flags
[tcs
->seek_list_count
] = entry
.flag
;
1125 tcs
->seek_list_count
++;
1130 return tcs
->seek_list_count
> 0;
1134 static void remove_files(void)
1139 tc_stat
.ready
= false;
1140 tc_stat
.ramcache
= false;
1141 tc_stat
.econ
= false;
1142 remove(TAGCACHE_FILE_MASTER
);
1143 for (i
= 0; i
< TAG_COUNT
; i
++)
1145 if (tagcache_is_numeric_tag(i
))
1148 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, i
);
1154 static bool check_all_headers(void)
1156 struct master_header myhdr
;
1157 struct tagcache_header tch
;
1161 if ( (fd
= open_master_fd(&myhdr
, false)) < 0)
1167 logf("tagcache is dirty!");
1171 memcpy(¤t_tcmh
, &myhdr
, sizeof(struct master_header
));
1173 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
1175 if (tagcache_is_numeric_tag(tag
))
1178 if ( (fd
= open_tag_fd(&tch
, tag
, false)) < 0)
1187 bool tagcache_search(struct tagcache_search
*tcs
, int tag
)
1189 struct tagcache_header tag_hdr
;
1190 struct master_header master_hdr
;
1193 if (tcs
->initialized
)
1194 tagcache_search_finish(tcs
);
1199 memset(tcs
, 0, sizeof(struct tagcache_search
));
1200 if (tc_stat
.commit_step
> 0 || !tc_stat
.ready
)
1203 tcs
->position
= sizeof(struct tagcache_header
);
1206 tcs
->seek_list_count
= 0;
1207 tcs
->filter_count
= 0;
1210 for (i
= 0; i
< TAG_COUNT
; i
++)
1213 #ifndef HAVE_TC_RAMCACHE
1214 tcs
->ramsearch
= false;
1216 tcs
->ramsearch
= tc_stat
.ramcache
;
1219 tcs
->entry_count
= hdr
->entry_count
[tcs
->type
];
1224 if (!tagcache_is_numeric_tag(tcs
->type
))
1226 tcs
->idxfd
[tcs
->type
] = open_tag_fd(&tag_hdr
, tcs
->type
, false);
1227 if (tcs
->idxfd
[tcs
->type
] < 0)
1231 /* Always open as R/W so we can pass tcs to functions that modify data also
1232 * without failing. */
1233 tcs
->masterfd
= open_master_fd(&master_hdr
, true);
1235 if (tcs
->masterfd
< 0)
1240 tcs
->initialized
= true;
1246 void tagcache_search_set_uniqbuf(struct tagcache_search
*tcs
,
1247 void *buffer
, long length
)
1249 tcs
->unique_list
= (unsigned long *)buffer
;
1250 tcs
->unique_list_capacity
= length
/ sizeof(*tcs
->unique_list
);
1251 tcs
->unique_list_count
= 0;
1254 bool tagcache_search_add_filter(struct tagcache_search
*tcs
,
1257 if (tcs
->filter_count
== TAGCACHE_MAX_FILTERS
)
1260 if (!tagcache_is_unique_tag(tag
) || tagcache_is_numeric_tag(tag
))
1263 tcs
->filter_tag
[tcs
->filter_count
] = tag
;
1264 tcs
->filter_seek
[tcs
->filter_count
] = seek
;
1265 tcs
->filter_count
++;
1270 bool tagcache_search_add_clause(struct tagcache_search
*tcs
,
1271 struct tagcache_search_clause
*clause
)
1275 if (tcs
->clause_count
>= TAGCACHE_MAX_CLAUSES
)
1277 logf("Too many clauses");
1281 /* Check if there is already a similar filter in present (filters are
1282 * much faster than clauses).
1284 for (i
= 0; i
< tcs
->filter_count
; i
++)
1286 if (tcs
->filter_tag
[i
] == clause
->tag
)
1290 if (!tagcache_is_numeric_tag(clause
->tag
) && tcs
->idxfd
[clause
->tag
] < 0)
1294 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, clause
->tag
);
1295 tcs
->idxfd
[clause
->tag
] = open(buf
, O_RDONLY
);
1298 tcs
->clause
[tcs
->clause_count
] = clause
;
1299 tcs
->clause_count
++;
1304 /* TODO: Remove this mess. */
1305 #ifdef HAVE_DIRCACHE
1306 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1307 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1309 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1312 static bool get_next(struct tagcache_search
*tcs
)
1314 static char buf
[TAG_MAXLEN
+32];
1315 struct tagfile_entry entry
;
1318 if (!tcs
->valid
|| !tc_stat
.ready
)
1321 if (tcs
->idxfd
[tcs
->type
] < 0 && !tagcache_is_numeric_tag(tcs
->type
)
1322 #ifdef HAVE_TC_RAMCACHE
1328 /* Relative fetch. */
1329 if (tcs
->filter_count
> 0 || tcs
->clause_count
> 0
1330 || tagcache_is_numeric_tag(tcs
->type
))
1332 /* Check for end of list. */
1333 if (tcs
->seek_list_count
== 0)
1335 /* Try to fetch more. */
1336 if (!build_lookup_list(tcs
))
1343 tcs
->seek_list_count
--;
1344 flag
= tcs
->seek_flags
[tcs
->seek_list_count
];
1346 /* Seek stream to the correct position and continue to direct fetch. */
1347 if ((!tcs
->ramsearch
|| !TAG_FILENAME_RAM(tcs
))
1348 && !tagcache_is_numeric_tag(tcs
->type
))
1350 if (!open_files(tcs
, tcs
->type
))
1353 lseek(tcs
->idxfd
[tcs
->type
], tcs
->seek_list
[tcs
->seek_list_count
], SEEK_SET
);
1356 tcs
->position
= tcs
->seek_list
[tcs
->seek_list_count
];
1359 if (tagcache_is_numeric_tag(tcs
->type
))
1361 snprintf(buf
, sizeof(buf
), "%d", tcs
->position
);
1362 tcs
->result_seek
= tcs
->position
;
1364 tcs
->result_len
= strlen(buf
) + 1;
1369 #ifdef HAVE_TC_RAMCACHE
1370 if (tcs
->ramsearch
&& TAG_FILENAME_RAM(tcs
))
1372 struct tagfile_entry
*ep
;
1374 if (tcs
->entry_count
== 0)
1381 tcs
->result_seek
= tcs
->position
;
1383 # ifdef HAVE_DIRCACHE
1384 if (tcs
->type
== tag_filename
)
1386 dircache_copy_path((struct dirent
*)tcs
->position
,
1389 tcs
->result_len
= strlen(buf
) + 1;
1390 tcs
->idx_id
= FLAG_GET_ATTR(flag
);
1391 tcs
->ramresult
= false;
1397 ep
= (struct tagfile_entry
*)&hdr
->tags
[tcs
->type
][tcs
->position
];
1398 tcs
->position
+= sizeof(struct tagfile_entry
) + ep
->tag_length
;
1399 tcs
->result
= ep
->tag_data
;
1400 tcs
->result_len
= strlen(tcs
->result
) + 1;
1401 tcs
->idx_id
= ep
->idx_id
;
1402 tcs
->ramresult
= true;
1409 if (!open_files(tcs
, tcs
->type
))
1412 tcs
->result_seek
= lseek(tcs
->idxfd
[tcs
->type
], 0, SEEK_CUR
);
1413 if (ecread(tcs
->idxfd
[tcs
->type
], &entry
, 1,
1414 tagfile_entry_ec
, tc_stat
.econ
) != sizeof(struct tagfile_entry
))
1422 if (entry
.tag_length
> (long)sizeof(buf
))
1425 logf("too long tag #2");
1429 if (read(tcs
->idxfd
[tcs
->type
], buf
, entry
.tag_length
) != entry
.tag_length
)
1432 logf("read error #4");
1437 tcs
->result_len
= strlen(tcs
->result
) + 1;
1438 tcs
->idx_id
= entry
.idx_id
;
1439 tcs
->ramresult
= false;
1444 bool tagcache_get_next(struct tagcache_search
*tcs
)
1446 while (get_next(tcs
))
1448 if (tcs
->result_len
> 1)
1455 bool tagcache_retrieve(struct tagcache_search
*tcs
, int idxid
,
1456 int tag
, char *buf
, long size
)
1458 struct index_entry idx
;
1461 if (!get_index(tcs
->masterfd
, idxid
, &idx
, true))
1464 return retrieve(tcs
, &idx
, tag
, buf
, size
);
1467 static bool update_master_header(void)
1469 struct master_header myhdr
;
1475 if ( (fd
= open_master_fd(&myhdr
, true)) < 0)
1478 myhdr
.serial
= current_tcmh
.serial
;
1479 myhdr
.commitid
= current_tcmh
.commitid
;
1482 lseek(fd
, 0, SEEK_SET
);
1483 ecwrite(fd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
1486 #ifdef HAVE_TC_RAMCACHE
1489 hdr
->h
.serial
= current_tcmh
.serial
;
1490 hdr
->h
.commitid
= current_tcmh
.commitid
;
1499 void tagcache_modify(struct tagcache_search
*tcs
, int type
, const char *text
)
1501 struct tagentry
*entry
;
1503 if (tcs
->type
!= tag_title
)
1506 /* We will need reserve buffer for this. */
1509 struct tagfile_entry
*ep
;
1511 ep
= (struct tagfile_entry
*)&hdr
->tags
[tcs
->type
][tcs
->result_seek
];
1512 tcs
->seek_list
[tcs
->seek_list_count
];
1515 entry
= find_entry_ram();
1520 void tagcache_search_finish(struct tagcache_search
*tcs
)
1524 if (!tcs
->initialized
)
1527 if (tcs
->masterfd
>= 0)
1529 close(tcs
->masterfd
);
1533 for (i
= 0; i
< TAG_COUNT
; i
++)
1535 if (tcs
->idxfd
[i
] >= 0)
1537 close(tcs
->idxfd
[i
]);
1542 tcs
->ramsearch
= false;
1544 tcs
->initialized
= 0;
1549 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1550 static struct tagfile_entry
*get_tag(const struct index_entry
*entry
, int tag
)
1552 return (struct tagfile_entry
*)&hdr
->tags
[tag
][entry
->tag_seek
[tag
]];
1555 static long get_tag_numeric(const struct index_entry
*entry
, int tag
)
1557 return check_virtual_tags(tag
, entry
);
1560 static char* get_tag_string(const struct index_entry
*entry
, int tag
)
1562 char* s
= get_tag(entry
, tag
)->tag_data
;
1563 return strcmp(s
, UNTAGGED
) ? s
: NULL
;
1566 bool tagcache_fill_tags(struct mp3entry
*id3
, const char *filename
)
1568 struct index_entry
*entry
;
1574 /* Find the corresponding entry in tagcache. */
1575 idx_id
= find_entry_ram(filename
, NULL
);
1576 if (idx_id
< 0 || !tc_stat
.ramcache
)
1579 entry
= &hdr
->indices
[idx_id
];
1581 id3
->title
= get_tag_string(entry
, tag_title
);
1582 id3
->artist
= get_tag_string(entry
, tag_artist
);
1583 id3
->album
= get_tag_string(entry
, tag_album
);
1584 id3
->genre_string
= get_tag_string(entry
, tag_genre
);
1585 id3
->composer
= get_tag_string(entry
, tag_composer
);
1586 id3
->comment
= get_tag_string(entry
, tag_comment
);
1587 id3
->albumartist
= get_tag_string(entry
, tag_albumartist
);
1588 id3
->grouping
= get_tag_string(entry
, tag_grouping
);
1590 id3
->playcount
= get_tag_numeric(entry
, tag_playcount
);
1591 id3
->rating
= get_tag_numeric(entry
, tag_rating
);
1592 id3
->lastplayed
= get_tag_numeric(entry
, tag_lastplayed
);
1593 id3
->score
= get_tag_numeric(entry
, tag_virt_autoscore
) / 10;
1594 id3
->year
= get_tag_numeric(entry
, tag_year
);
1596 id3
->discnum
= get_tag_numeric(entry
, tag_discnumber
);
1597 id3
->tracknum
= get_tag_numeric(entry
, tag_tracknumber
);
1598 id3
->bitrate
= get_tag_numeric(entry
, tag_bitrate
);
1599 if (id3
->bitrate
== 0)
1606 static inline void write_item(const char *item
)
1608 int len
= strlen(item
) + 1;
1611 write(cachefd
, item
, len
);
1614 static int check_if_empty(char **tag
)
1618 if (*tag
== NULL
|| **tag
== '\0')
1621 return sizeof(UNTAGGED
); /* Tag length */
1624 length
= strlen(*tag
);
1625 if (length
> TAG_MAXLEN
)
1627 logf("over length tag: %s", *tag
);
1628 length
= TAG_MAXLEN
;
1629 (*tag
)[length
] = '\0';
1635 #define ADD_TAG(entry,tag,data) \
1637 entry.tag_offset[tag] = offset; \
1638 entry.tag_length[tag] = check_if_empty(data); \
1639 offset += entry.tag_length[tag]
1641 static void add_tagcache(char *path
, unsigned long mtime
1642 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1643 ,const struct dirent
*dc
1647 struct mp3entry id3
;
1648 struct temp_file_entry entry
;
1652 char tracknumfix
[3];
1654 int path_length
= strlen(path
);
1655 bool has_albumartist
;
1661 /* Check for overlength file path. */
1662 if (path_length
> TAG_MAXLEN
)
1664 /* Path can't be shortened. */
1665 logf("Too long path: %s", path
);
1669 /* Check if the file is supported. */
1670 if (probe_file_format(path
) == AFMT_UNKNOWN
)
1673 /* Check if the file is already cached. */
1674 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1675 if (tc_stat
.ramcache
&& is_dircache_intact())
1677 idx_id
= find_entry_ram(path
, dc
);
1682 if (filenametag_fd
>= 0)
1684 idx_id
= find_entry_disk(path
);
1688 /* Check if file has been modified. */
1691 struct index_entry idx
;
1693 if (!get_index(-1, idx_id
, &idx
, true))
1695 logf("failed to retrieve index entry");
1699 if ((unsigned long)idx
.tag_seek
[tag_mtime
] == mtime
)
1701 /* No changes to file. */
1705 /* Metadata might have been changed. Delete the entry. */
1706 logf("Re-adding: %s", path
);
1707 if (!delete_entry(idx_id
))
1709 logf("delete_entry failed: %d", idx_id
);
1714 fd
= open(path
, O_RDONLY
);
1717 logf("open fail: %s", path
);
1721 memset(&id3
, 0, sizeof(struct mp3entry
));
1722 memset(&entry
, 0, sizeof(struct temp_file_entry
));
1723 memset(&tracknumfix
, 0, sizeof(tracknumfix
));
1724 ret
= get_metadata(&id3
, fd
, path
);
1730 logf("-> %s", path
);
1732 /* Generate track number if missing. */
1733 if (id3
.tracknum
<= 0)
1735 const char *p
= strrchr(path
, '.');
1738 p
= &path
[strlen(path
)-1];
1742 if (isdigit(*p
) && isdigit(*(p
-1)))
1744 tracknumfix
[1] = *p
--;
1745 tracknumfix
[0] = *p
;
1751 if (tracknumfix
[0] != '\0')
1753 id3
.tracknum
= atoi(tracknumfix
);
1754 /* Set a flag to indicate track number has been generated. */
1755 entry
.flag
|= FLAG_TRKNUMGEN
;
1759 /* Unable to generate track number. */
1765 entry
.tag_offset
[tag_year
] = id3
.year
;
1766 entry
.tag_offset
[tag_discnumber
] = id3
.discnum
;
1767 entry
.tag_offset
[tag_tracknumber
] = id3
.tracknum
;
1768 entry
.tag_offset
[tag_length
] = id3
.length
;
1769 entry
.tag_offset
[tag_bitrate
] = id3
.bitrate
;
1770 entry
.tag_offset
[tag_mtime
] = mtime
;
1773 has_albumartist
= id3
.albumartist
!= NULL
1774 && strlen(id3
.albumartist
) > 0;
1775 has_grouping
= id3
.grouping
!= NULL
1776 && strlen(id3
.grouping
) > 0;
1778 ADD_TAG(entry
, tag_filename
, &path
);
1779 ADD_TAG(entry
, tag_title
, &id3
.title
);
1780 ADD_TAG(entry
, tag_artist
, &id3
.artist
);
1781 ADD_TAG(entry
, tag_album
, &id3
.album
);
1782 ADD_TAG(entry
, tag_genre
, &id3
.genre_string
);
1783 ADD_TAG(entry
, tag_composer
, &id3
.composer
);
1784 ADD_TAG(entry
, tag_comment
, &id3
.comment
);
1785 if (has_albumartist
)
1787 ADD_TAG(entry
, tag_albumartist
, &id3
.albumartist
);
1791 ADD_TAG(entry
, tag_albumartist
, &id3
.artist
);
1795 ADD_TAG(entry
, tag_grouping
, &id3
.grouping
);
1799 ADD_TAG(entry
, tag_grouping
, &id3
.title
);
1801 entry
.data_length
= offset
;
1803 /* Write the header */
1804 write(cachefd
, &entry
, sizeof(struct temp_file_entry
));
1806 /* And tags also... Correct order is critical */
1808 write_item(id3
.title
);
1809 write_item(id3
.artist
);
1810 write_item(id3
.album
);
1811 write_item(id3
.genre_string
);
1812 write_item(id3
.composer
);
1813 write_item(id3
.comment
);
1814 if (has_albumartist
)
1816 write_item(id3
.albumartist
);
1820 write_item(id3
.artist
);
1824 write_item(id3
.grouping
);
1828 write_item(id3
.title
);
1830 total_entry_count
++;
1833 static bool tempbuf_insert(char *str
, int id
, int idx_id
, bool unique
)
1835 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1836 int len
= strlen(str
)+1;
1839 unsigned *crcbuf
= (unsigned *)&tempbuf
[tempbuf_size
-4];
1840 char buf
[TAG_MAXLEN
+32];
1842 for (i
= 0; str
[i
] != '\0' && i
< (int)sizeof(buf
)-1; i
++)
1843 buf
[i
] = tolower(str
[i
]);
1846 crc32
= crc_32(buf
, i
, 0xffffffff);
1850 /* Check if the crc does not exist -> entry does not exist for sure. */
1851 for (i
= 0; i
< tempbufidx
; i
++)
1853 if (crcbuf
[-i
] != crc32
)
1856 if (!strcasecmp(str
, index
[i
].str
))
1858 if (id
< 0 || id
>= lookup_buffer_depth
)
1860 logf("lookup buf overf.: %d", id
);
1864 lookup
[id
] = &index
[i
];
1870 /* Insert to CRC buffer. */
1871 crcbuf
[-tempbufidx
] = crc32
;
1874 /* Insert it to the buffer. */
1875 tempbuf_left
-= len
;
1876 if (tempbuf_left
- 4 < 0 || tempbufidx
>= commit_entry_count
-1)
1879 if (id
>= lookup_buffer_depth
)
1881 logf("lookup buf overf. #2: %d", id
);
1887 lookup
[id
] = &index
[tempbufidx
];
1888 index
[tempbufidx
].idlist
.id
= id
;
1891 index
[tempbufidx
].idlist
.id
= -1;
1893 index
[tempbufidx
].idlist
.next
= NULL
;
1894 index
[tempbufidx
].idx_id
= idx_id
;
1895 index
[tempbufidx
].seek
= -1;
1896 index
[tempbufidx
].str
= &tempbuf
[tempbuf_pos
];
1897 memcpy(index
[tempbufidx
].str
, str
, len
);
1904 static int compare(const void *p1
, const void *p2
)
1908 struct tempbuf_searchidx
*e1
= (struct tempbuf_searchidx
*)p1
;
1909 struct tempbuf_searchidx
*e2
= (struct tempbuf_searchidx
*)p2
;
1911 if (strcmp(e1
->str
, UNTAGGED
) == 0)
1913 if (strcmp(e2
->str
, UNTAGGED
) == 0)
1917 else if (strcmp(e2
->str
, UNTAGGED
) == 0)
1920 return strncasecmp(e1
->str
, e2
->str
, TAG_MAXLEN
);
1923 static int tempbuf_sort(int fd
)
1925 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1926 struct tagfile_entry fe
;
1930 /* Generate reverse lookup entries. */
1931 for (i
= 0; i
< lookup_buffer_depth
; i
++)
1933 struct tempbuf_id_list
*idlist
;
1938 if (lookup
[i
]->idlist
.id
== i
)
1941 idlist
= &lookup
[i
]->idlist
;
1942 while (idlist
->next
!= NULL
)
1943 idlist
= idlist
->next
;
1945 tempbuf_left
-= sizeof(struct tempbuf_id_list
);
1946 if (tempbuf_left
- 4 < 0)
1949 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
1950 if (tempbuf_pos
& 0x03)
1952 tempbuf_pos
= (tempbuf_pos
& ~0x03) + 0x04;
1954 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
1956 tempbuf_pos
+= sizeof(struct tempbuf_id_list
);
1958 idlist
= idlist
->next
;
1960 idlist
->next
= NULL
;
1965 qsort(index
, tempbufidx
, sizeof(struct tempbuf_searchidx
), compare
);
1966 memset(lookup
, 0, lookup_buffer_depth
* sizeof(struct tempbuf_searchidx
**));
1968 for (i
= 0; i
< tempbufidx
; i
++)
1970 struct tempbuf_id_list
*idlist
= &index
[i
].idlist
;
1972 /* Fix the lookup list. */
1973 while (idlist
!= NULL
)
1975 if (idlist
->id
>= 0)
1976 lookup
[idlist
->id
] = &index
[i
];
1977 idlist
= idlist
->next
;
1980 index
[i
].seek
= lseek(fd
, 0, SEEK_CUR
);
1981 length
= strlen(index
[i
].str
) + 1;
1982 fe
.tag_length
= length
;
1983 fe
.idx_id
= index
[i
].idx_id
;
1985 /* Check the chunk alignment. */
1986 if ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
1987 % TAGFILE_ENTRY_CHUNK_LENGTH
)
1989 fe
.tag_length
+= TAGFILE_ENTRY_CHUNK_LENGTH
-
1990 ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
1991 % TAGFILE_ENTRY_CHUNK_LENGTH
);
1994 #ifdef TAGCACHE_STRICT_ALIGN
1995 /* Make sure the entry is long aligned. */
1996 if (index
[i
].seek
& 0x03)
1998 logf("tempbuf_sort: alignment error!");
2003 if (ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
) !=
2004 sizeof(struct tagfile_entry
))
2006 logf("tempbuf_sort: write error #1");
2010 if (write(fd
, index
[i
].str
, length
) != length
)
2012 logf("tempbuf_sort: write error #2");
2016 /* Write some padding. */
2017 if (fe
.tag_length
- length
> 0)
2018 write(fd
, "XXXXXXXX", fe
.tag_length
- length
);
2024 inline static struct tempbuf_searchidx
* tempbuf_locate(int id
)
2026 if (id
< 0 || id
>= lookup_buffer_depth
)
2033 inline static int tempbuf_find_location(int id
)
2035 struct tempbuf_searchidx
*entry
;
2037 entry
= tempbuf_locate(id
);
2044 static bool build_numeric_indices(struct tagcache_header
*h
, int tmpfd
)
2046 struct master_header tcmh
;
2047 struct index_entry idx
;
2050 struct temp_file_entry
*entrybuf
= (struct temp_file_entry
*)tempbuf
;
2052 int entries_processed
= 0;
2054 char buf
[TAG_MAXLEN
];
2056 max_entries
= tempbuf_size
/ sizeof(struct temp_file_entry
) - 1;
2058 logf("Building numeric indices...");
2059 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2061 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
2064 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2066 if (masterfd_pos
== filesize(masterfd
))
2068 logf("we can't append!");
2073 while (entries_processed
< h
->entry_count
)
2075 int count
= MIN(h
->entry_count
- entries_processed
, max_entries
);
2077 /* Read in as many entries as possible. */
2078 for (i
= 0; i
< count
; i
++)
2080 struct temp_file_entry
*tfe
= &entrybuf
[i
];
2083 /* Read in numeric data. */
2084 if (read(tmpfd
, tfe
, sizeof(struct temp_file_entry
)) !=
2085 sizeof(struct temp_file_entry
))
2087 logf("read fail #1");
2092 datastart
= lseek(tmpfd
, 0, SEEK_CUR
);
2095 * Read string data from the following tags:
2101 * A crc32 hash is calculated from the read data
2102 * and stored back to the data offset field kept in memory.
2104 #define tmpdb_read_string_tag(tag) \
2105 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2106 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2108 logf("read fail: buffer overflow"); \
2113 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2114 tfe->tag_length[tag]) \
2116 logf("read fail #2"); \
2121 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2122 lseek(tmpfd, datastart, SEEK_SET)
2124 tmpdb_read_string_tag(tag_filename
);
2125 tmpdb_read_string_tag(tag_artist
);
2126 tmpdb_read_string_tag(tag_album
);
2127 tmpdb_read_string_tag(tag_title
);
2129 /* Seek to the end of the string data. */
2130 lseek(tmpfd
, tfe
->data_length
, SEEK_CUR
);
2133 /* Backup the master index position. */
2134 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2135 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2137 /* Check if we can resurrect some deleted runtime statistics data. */
2138 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
++)
2140 /* Read the index entry. */
2141 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2142 != sizeof(struct index_entry
))
2144 logf("read fail #3");
2150 * Skip unless the entry is marked as being deleted
2151 * or the data has already been resurrected.
2153 if (!(idx
.flag
& FLAG_DELETED
) || idx
.flag
& FLAG_RESURRECTED
)
2156 /* Now try to match the entry. */
2158 * To succesfully match a song, the following conditions
2161 * For numeric fields: tag_length
2162 * - Full identical match is required
2164 * If tag_filename matches, no further checking necessary.
2166 * For string hashes: tag_artist, tag_album, tag_title
2167 * - Two of these must match
2169 for (j
= 0; j
< count
; j
++)
2171 struct temp_file_entry
*tfe
= &entrybuf
[j
];
2173 /* Try to match numeric fields first. */
2174 if (tfe
->tag_offset
[tag_length
] != idx
.tag_seek
[tag_length
])
2177 /* Now it's time to do the hash matching. */
2178 if (tfe
->tag_offset
[tag_filename
] != idx
.tag_seek
[tag_filename
])
2180 int match_count
= 0;
2182 /* No filename match, check if we can match two other tags. */
2183 #define tmpdb_match(tag) \
2184 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2187 tmpdb_match(tag_artist
);
2188 tmpdb_match(tag_album
);
2189 tmpdb_match(tag_title
);
2191 if (match_count
< 2)
2193 /* Still no match found, give up. */
2198 /* A match found, now copy & resurrect the statistical data. */
2199 #define tmpdb_copy_tag(tag) \
2200 tfe->tag_offset[tag] = idx.tag_seek[tag]
2202 tmpdb_copy_tag(tag_playcount
);
2203 tmpdb_copy_tag(tag_rating
);
2204 tmpdb_copy_tag(tag_playtime
);
2205 tmpdb_copy_tag(tag_lastplayed
);
2206 tmpdb_copy_tag(tag_commitid
);
2208 /* Avoid processing this entry again. */
2209 idx
.flag
|= FLAG_RESURRECTED
;
2211 lseek(masterfd
, -sizeof(struct index_entry
), SEEK_CUR
);
2212 if (ecwrite(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2213 != sizeof(struct index_entry
))
2215 logf("masterfd writeback fail #1");
2220 logf("Entry resurrected");
2225 /* Restore the master index position. */
2226 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2228 /* Commit the data to the index. */
2229 for (i
= 0; i
< count
; i
++)
2231 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2233 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2234 != sizeof(struct index_entry
))
2236 logf("read fail #3");
2241 for (j
= 0; j
< TAG_COUNT
; j
++)
2243 if (!tagcache_is_numeric_tag(j
))
2246 idx
.tag_seek
[j
] = entrybuf
[i
].tag_offset
[j
];
2248 idx
.flag
= entrybuf
[i
].flag
;
2250 if (idx
.tag_seek
[tag_commitid
])
2252 /* Data has been resurrected. */
2253 idx
.flag
|= FLAG_DIRTYNUM
;
2255 else if (tc_stat
.ready
&& current_tcmh
.commitid
> 0)
2257 idx
.tag_seek
[tag_commitid
] = current_tcmh
.commitid
;
2258 idx
.flag
|= FLAG_DIRTYNUM
;
2261 /* Write back the updated index. */
2262 lseek(masterfd
, loc
, SEEK_SET
);
2263 if (ecwrite(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2264 != sizeof(struct index_entry
))
2272 entries_processed
+= count
;
2273 logf("%d/%ld entries processed", entries_processed
, h
->entry_count
);
2284 * == 0 temporary failure
2287 static int build_index(int index_type
, struct tagcache_header
*h
, int tmpfd
)
2290 struct tagcache_header tch
;
2291 struct master_header tcmh
;
2292 struct index_entry idxbuf
[IDX_BUF_DEPTH
];
2294 char buf
[TAG_MAXLEN
+32];
2295 int fd
= -1, masterfd
;
2300 logf("Building index: %d", index_type
);
2302 /* Check the number of entries we need to allocate ram for. */
2303 commit_entry_count
= h
->entry_count
+ 1;
2305 masterfd
= open_master_fd(&tcmh
, false);
2308 commit_entry_count
+= tcmh
.tch
.entry_count
;
2312 remove_files(); /* Just to be sure we are clean. */
2314 /* Open the index file, which contains the tag names. */
2315 fd
= open_tag_fd(&tch
, index_type
, true);
2318 logf("tch.datasize=%ld", tch
.datasize
);
2319 lookup_buffer_depth
= 1 +
2320 /* First part */ commit_entry_count
+
2321 /* Second part */ (tch
.datasize
/ TAGFILE_ENTRY_CHUNK_LENGTH
);
2325 lookup_buffer_depth
= 1 +
2326 /* First part */ commit_entry_count
+
2327 /* Second part */ 0;
2330 logf("lookup_buffer_depth=%ld", lookup_buffer_depth
);
2331 logf("commit_entry_count=%ld", commit_entry_count
);
2333 /* Allocate buffer for all index entries from both old and new
2336 tempbuf_pos
= commit_entry_count
* sizeof(struct tempbuf_searchidx
);
2338 /* Allocate lookup buffer. The first portion of commit_entry_count
2339 * contains the new tags in the temporary file and the second
2340 * part for locating entries already in the db.
2343 * +---------+---------------------------+
2344 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2345 * +---------+---------------------------+
2347 * Old tags are inserted to a temporary buffer with position:
2348 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2349 * And new tags with index:
2350 * tempbuf_insert(idx, ...);
2352 * The buffer is sorted and written into tag file:
2353 * tempbuf_sort(...);
2354 * leaving master index locations messed up.
2356 * That is fixed using the lookup buffer for old tags:
2357 * new_seek = tempbuf_find_location(old_seek, ...);
2359 * new_seek = tempbuf_find_location(idx);
2361 lookup
= (struct tempbuf_searchidx
**)&tempbuf
[tempbuf_pos
];
2362 tempbuf_pos
+= lookup_buffer_depth
* sizeof(void **);
2363 memset(lookup
, 0, lookup_buffer_depth
* sizeof(void **));
2365 /* And calculate the remaining data space used mainly for storing
2366 * tag data (strings). */
2367 tempbuf_left
= tempbuf_size
- tempbuf_pos
- 8;
2368 if (tempbuf_left
- TAGFILE_ENTRY_AVG_LENGTH
* commit_entry_count
< 0)
2370 logf("Buffer way too small!");
2377 * If tag file contains unique tags (sorted index), we will load
2378 * it entirely into memory so we can resort it later for use with
2381 if (tagcache_is_sorted_tag(index_type
))
2383 logf("loading tags...");
2384 for (i
= 0; i
< tch
.entry_count
; i
++)
2386 struct tagfile_entry entry
;
2387 int loc
= lseek(fd
, 0, SEEK_CUR
);
2390 if (ecread(fd
, &entry
, 1, tagfile_entry_ec
, tc_stat
.econ
)
2391 != sizeof(struct tagfile_entry
))
2393 logf("read error #7");
2398 if (entry
.tag_length
>= (int)sizeof(buf
))
2400 logf("too long tag #3");
2405 if (read(fd
, buf
, entry
.tag_length
) != entry
.tag_length
)
2407 logf("read error #8");
2412 /* Skip deleted entries. */
2417 * Save the tag and tag id in the memory buffer. Tag id
2418 * is saved so we can later reindex the master lookup
2419 * table when the index gets resorted.
2421 ret
= tempbuf_insert(buf
, loc
/TAGFILE_ENTRY_CHUNK_LENGTH
2422 + commit_entry_count
, entry
.idx_id
,
2423 tagcache_is_unique_tag(index_type
));
2434 tempbufidx
= tch
.entry_count
;
2439 * Creating new index file to store the tags. No need to preload
2440 * anything whether the index type is sorted or not.
2442 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, index_type
);
2443 fd
= open(buf
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2446 logf("%s open fail", buf
);
2450 tch
.magic
= TAGCACHE_MAGIC
;
2451 tch
.entry_count
= 0;
2454 if (ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
)
2455 != sizeof(struct tagcache_header
))
2457 logf("header write failed");
2463 /* Loading the tag lookup file as "master file". */
2464 logf("Loading index file");
2465 masterfd
= open(TAGCACHE_FILE_MASTER
, O_RDWR
);
2469 logf("Creating new DB");
2470 masterfd
= open(TAGCACHE_FILE_MASTER
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2474 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER
);
2479 /* Write the header (write real values later). */
2480 memset(&tcmh
, 0, sizeof(struct master_header
));
2482 tcmh
.tch
.entry_count
= 0;
2483 tcmh
.tch
.datasize
= 0;
2485 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2487 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2492 * Master file already exists so we need to process the current
2497 if (ecread(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
) !=
2498 sizeof(struct master_header
) || tcmh
.tch
.magic
!= TAGCACHE_MAGIC
)
2500 logf("header error");
2507 * If we reach end of the master file, we need to expand it to
2508 * hold new tags. If the current index is not sorted, we can
2509 * simply append new data to end of the file.
2510 * However, if the index is sorted, we need to update all tag
2511 * pointers in the master file for the current index.
2513 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2515 if (masterfd_pos
== filesize(masterfd
))
2517 logf("appending...");
2523 * Load new unique tags in memory to be sorted later and added
2524 * to the master lookup file.
2526 if (tagcache_is_sorted_tag(index_type
))
2528 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2529 /* h is the header of the temporary file containing new tags. */
2530 logf("inserting new tags...");
2531 for (i
= 0; i
< h
->entry_count
; i
++)
2533 struct temp_file_entry entry
;
2535 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2536 sizeof(struct temp_file_entry
))
2538 logf("read fail #3");
2544 if (entry
.tag_length
[index_type
] >= (long)sizeof(buf
))
2546 logf("too long entry!");
2551 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2552 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2553 entry
.tag_length
[index_type
])
2555 logf("read fail #4");
2560 if (tagcache_is_unique_tag(index_type
))
2561 error
= !tempbuf_insert(buf
, i
, -1, true);
2563 error
= !tempbuf_insert(buf
, i
, tcmh
.tch
.entry_count
+ i
, false);
2567 logf("insert error");
2572 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2573 entry
.tag_length
[index_type
], SEEK_CUR
);
2578 /* Sort the buffer data and write it to the index file. */
2579 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
2580 i
= tempbuf_sort(fd
);
2583 logf("sorted %d tags", i
);
2586 * Now update all indexes in the master lookup file.
2588 logf("updating indices...");
2589 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2590 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
+= idxbuf_pos
)
2593 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2595 idxbuf_pos
= MIN(tcmh
.tch
.entry_count
- i
, IDX_BUF_DEPTH
);
2597 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2598 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2600 logf("read fail #5");
2604 lseek(masterfd
, loc
, SEEK_SET
);
2606 for (j
= 0; j
< idxbuf_pos
; j
++)
2608 if (idxbuf
[j
].flag
& FLAG_DELETED
)
2610 /* We can just ignore deleted entries. */
2611 // idxbuf[j].tag_seek[index_type] = 0;
2615 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(
2616 idxbuf
[j
].tag_seek
[index_type
]/TAGFILE_ENTRY_CHUNK_LENGTH
2617 + commit_entry_count
);
2619 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2621 logf("update error: %d/%d/%ld",
2622 idxbuf
[j
].flag
, i
+j
, tcmh
.tch
.entry_count
);
2630 /* Write back the updated index. */
2631 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2632 index_entry_ec
, tc_stat
.econ
) !=
2633 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2644 * Walk through the temporary file containing the new tags.
2646 // build_normal_index(h, tmpfd, masterfd, idx);
2647 logf("updating new indices...");
2648 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2649 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2650 lseek(fd
, 0, SEEK_END
);
2651 for (i
= 0; i
< h
->entry_count
; i
+= idxbuf_pos
)
2655 idxbuf_pos
= MIN(h
->entry_count
- i
, IDX_BUF_DEPTH
);
2658 memset(idxbuf
, 0, sizeof(struct index_entry
)*IDX_BUF_DEPTH
);
2662 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2664 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2665 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2667 logf("read fail #6");
2671 lseek(masterfd
, loc
, SEEK_SET
);
2674 /* Read entry headers. */
2675 for (j
= 0; j
< idxbuf_pos
; j
++)
2677 if (!tagcache_is_sorted_tag(index_type
))
2679 struct temp_file_entry entry
;
2680 struct tagfile_entry fe
;
2682 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2683 sizeof(struct temp_file_entry
))
2685 logf("read fail #7");
2691 if (entry
.tag_length
[index_type
] >= (int)sizeof(buf
))
2693 logf("too long entry!");
2694 logf("length=%d", entry
.tag_length
[index_type
]);
2695 logf("pos=0x%02lx", lseek(tmpfd
, 0, SEEK_CUR
));
2700 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2701 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2702 entry
.tag_length
[index_type
])
2704 logf("read fail #8");
2705 logf("offset=0x%02lx", entry
.tag_offset
[index_type
]);
2706 logf("length=0x%02x", entry
.tag_length
[index_type
]);
2711 /* Write to index file. */
2712 idxbuf
[j
].tag_seek
[index_type
] = lseek(fd
, 0, SEEK_CUR
);
2713 fe
.tag_length
= entry
.tag_length
[index_type
];
2714 fe
.idx_id
= tcmh
.tch
.entry_count
+ i
+ j
;
2715 ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
2716 write(fd
, buf
, fe
.tag_length
);
2720 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2721 entry
.tag_length
[index_type
], SEEK_CUR
);
2725 /* Locate the correct entry from the sorted array. */
2726 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(i
+ j
);
2727 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2729 logf("entry not found (%d)", j
);
2737 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2738 index_entry_ec
, tc_stat
.econ
) !=
2739 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2741 logf("tagcache: write fail #4");
2750 /* Finally write the header. */
2751 tch
.magic
= TAGCACHE_MAGIC
;
2752 tch
.entry_count
= tempbufidx
;
2753 tch
.datasize
= lseek(fd
, 0, SEEK_END
) - sizeof(struct tagcache_header
);
2754 lseek(fd
, 0, SEEK_SET
);
2755 ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
);
2757 if (index_type
!= tag_filename
)
2758 h
->datasize
+= tch
.datasize
;
2759 logf("s:%d/%ld/%ld", index_type
, tch
.datasize
, h
->datasize
);
2771 static bool commit(void)
2773 struct tagcache_header tch
;
2774 struct master_header tcmh
;
2778 #ifdef HAVE_DIRCACHE
2779 bool dircache_buffer_stolen
= false;
2781 bool local_allocation
= false;
2783 logf("committing tagcache");
2788 tmpfd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
2791 logf("nothing to commit");
2796 /* Load the header. */
2797 len
= sizeof(struct tagcache_header
);
2798 rc
= read(tmpfd
, &tch
, len
);
2800 if (tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= len
)
2802 logf("incorrect header");
2804 remove(TAGCACHE_FILE_TEMP
);
2808 if (tch
.entry_count
== 0)
2810 logf("nothing to commit");
2812 remove(TAGCACHE_FILE_TEMP
);
2816 #ifdef HAVE_EEPROM_SETTINGS
2817 remove(TAGCACHE_STATEFILE
);
2820 /* At first be sure to unload the ramcache! */
2821 #ifdef HAVE_TC_RAMCACHE
2822 tc_stat
.ramcache
= false;
2827 /* Try to steal every buffer we can :) */
2828 if (tempbuf_size
== 0)
2829 local_allocation
= true;
2831 #ifdef HAVE_DIRCACHE
2832 if (tempbuf_size
== 0)
2834 /* Try to steal the dircache buffer. */
2835 tempbuf
= dircache_steal_buffer(&tempbuf_size
);
2836 tempbuf_size
&= ~0x03;
2838 if (tempbuf_size
> 0)
2840 dircache_buffer_stolen
= true;
2845 #ifdef HAVE_TC_RAMCACHE
2846 if (tempbuf_size
== 0 && tc_stat
.ramcache_allocated
> 0)
2848 tempbuf
= (char *)(hdr
+ 1);
2849 tempbuf_size
= tc_stat
.ramcache_allocated
- sizeof(struct ramcache_header
) - 128;
2850 tempbuf_size
&= ~0x03;
2854 /* And finally fail if there are no buffers available. */
2855 if (tempbuf_size
== 0)
2857 logf("delaying commit until next boot");
2858 tc_stat
.commit_delayed
= true;
2864 logf("commit %ld entries...", tch
.entry_count
);
2866 /* Mark DB dirty so it will stay disabled if commit fails. */
2867 current_tcmh
.dirty
= true;
2868 update_master_header();
2870 /* Now create the index files. */
2871 tc_stat
.commit_step
= 0;
2873 tc_stat
.commit_delayed
= false;
2875 for (i
= 0; i
< TAG_COUNT
; i
++)
2879 if (tagcache_is_numeric_tag(i
))
2882 tc_stat
.commit_step
++;
2883 ret
= build_index(i
, &tch
, tmpfd
);
2887 logf("tagcache failed init");
2891 tc_stat
.commit_delayed
= true;
2892 tc_stat
.commit_step
= 0;
2898 if (!build_numeric_indices(&tch
, tmpfd
))
2900 logf("Failure to commit numeric indices");
2903 tc_stat
.commit_step
= 0;
2909 tc_stat
.commit_step
= 0;
2911 /* Update the master index headers. */
2912 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
2918 tcmh
.tch
.entry_count
+= tch
.entry_count
;
2919 tcmh
.tch
.datasize
= sizeof(struct master_header
)
2920 + sizeof(struct index_entry
) * tcmh
.tch
.entry_count
2925 lseek(masterfd
, 0, SEEK_SET
);
2926 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2929 logf("tagcache committed");
2930 remove(TAGCACHE_FILE_TEMP
);
2931 tc_stat
.ready
= check_all_headers();
2932 tc_stat
.readyvalid
= true;
2934 if (local_allocation
)
2940 #ifdef HAVE_DIRCACHE
2941 /* Rebuild the dircache, if we stole the buffer. */
2942 if (dircache_buffer_stolen
)
2946 #ifdef HAVE_TC_RAMCACHE
2947 /* Reload tagcache. */
2948 if (tc_stat
.ramcache_allocated
> 0)
2949 tagcache_start_scan();
2957 static void allocate_tempbuf(void)
2959 /* Yeah, malloc would be really nice now :) */
2961 tempbuf_size
= 32*1024*1024;
2962 tempbuf
= malloc(tempbuf_size
);
2964 tempbuf
= (char *)(((long)audiobuf
& ~0x03) + 0x04);
2965 tempbuf_size
= (long)audiobufend
- (long)audiobuf
- 4;
2966 audiobuf
+= tempbuf_size
;
2970 static void free_tempbuf(void)
2972 if (tempbuf_size
== 0)
2978 audiobuf
-= tempbuf_size
;
2984 static bool modify_numeric_entry(int masterfd
, int idx_id
, int tag
, long data
)
2986 struct index_entry idx
;
2991 if (!tagcache_is_numeric_tag(tag
))
2994 if (!get_index(masterfd
, idx_id
, &idx
, false))
2997 idx
.tag_seek
[tag
] = data
;
2998 idx
.flag
|= FLAG_DIRTYNUM
;
3000 return write_index(masterfd
, idx_id
, &idx
);
3004 bool tagcache_modify_numeric_entry(struct tagcache_search
*tcs
,
3007 struct master_header myhdr
;
3009 if (tcs
->masterfd
< 0)
3011 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, true)) < 0)
3015 return modify_numeric_entry(tcs
->masterfd
, tcs
->idx_id
, tag
, data
);
3019 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3021 static bool command_queue_is_full(void)
3025 next
= command_queue_widx
+ 1;
3026 if (next
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3029 return (next
== command_queue_ridx
);
3031 static bool command_queue_sync_callback(void)
3034 struct master_header myhdr
;
3037 mutex_lock(&command_queue_mutex
);
3039 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3042 while (command_queue_ridx
!= command_queue_widx
)
3044 struct tagcache_command_entry
*ce
= &command_queue
[command_queue_ridx
];
3046 switch (ce
->command
)
3048 case CMD_UPDATE_MASTER_HEADER
:
3051 update_master_header();
3053 /* Re-open the masterfd. */
3054 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3059 case CMD_UPDATE_NUMERIC
:
3061 modify_numeric_entry(masterfd
, ce
->idx_id
, ce
->tag
, ce
->data
);
3066 if (++command_queue_ridx
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3067 command_queue_ridx
= 0;
3072 tc_stat
.queue_length
= 0;
3073 mutex_unlock(&command_queue_mutex
);
3077 static void run_command_queue(bool force
)
3079 if (COMMAND_QUEUE_IS_EMPTY
)
3082 if (force
|| command_queue_is_full())
3083 command_queue_sync_callback();
3085 register_ata_idle_func(command_queue_sync_callback
);
3088 static void queue_command(int cmd
, long idx_id
, int tag
, long data
)
3094 mutex_lock(&command_queue_mutex
);
3095 next
= command_queue_widx
+ 1;
3096 if (next
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3099 /* Make sure queue is not full. */
3100 if (next
!= command_queue_ridx
)
3102 struct tagcache_command_entry
*ce
= &command_queue
[command_queue_widx
];
3105 ce
->idx_id
= idx_id
;
3109 command_queue_widx
= next
;
3111 tc_stat
.queue_length
++;
3113 mutex_unlock(&command_queue_mutex
);
3117 /* Queue is full, try again later... */
3118 mutex_unlock(&command_queue_mutex
);
3123 long tagcache_increase_serial(void)
3133 old
= current_tcmh
.serial
++;
3134 queue_command(CMD_UPDATE_MASTER_HEADER
, 0, 0, 0);
3139 void tagcache_update_numeric(int idx_id
, int tag
, long data
)
3141 queue_command(CMD_UPDATE_NUMERIC
, idx_id
, tag
, data
);
3144 long tagcache_get_serial(void)
3146 return current_tcmh
.serial
;
3149 long tagcache_get_commitid(void)
3151 return current_tcmh
.commitid
;
3154 static bool write_tag(int fd
, const char *tagstr
, const char *datastr
)
3159 snprintf(buf
, sizeof buf
, "%s=\"", tagstr
);
3160 for (i
= strlen(buf
); i
< (long)sizeof(buf
)-4; i
++)
3162 if (*datastr
== '\0')
3165 if (*datastr
== '"' || *datastr
== '\\')
3168 buf
[i
] = *(datastr
++);
3171 strcpy(&buf
[i
], "\" ");
3173 write(fd
, buf
, i
+ 2);
3178 static bool read_tag(char *dest
, long size
,
3179 const char *src
, const char *tagstr
)
3182 char current_tag
[32];
3184 while (*src
!= '\0')
3186 /* Skip all whitespace */
3194 /* Read in tag name */
3195 while (*src
!= '=' && *src
!= ' ')
3197 current_tag
[pos
] = *src
;
3201 if (*src
== '\0' || pos
>= (long)sizeof(current_tag
))
3204 current_tag
[pos
] = '\0';
3206 /* Read in tag data */
3208 /* Find the start. */
3209 while (*src
!= '"' && *src
!= '\0')
3212 if (*src
== '\0' || *(++src
) == '\0')
3215 /* Read the data. */
3216 for (pos
= 0; pos
< size
; pos
++)
3223 dest
[pos
] = *(src
+1);
3243 if (!strcasecmp(tagstr
, current_tag
))
3250 static int parse_changelog_line(int line_n
, const char *buf
, void *parameters
)
3252 struct index_entry idx
;
3253 char tag_data
[TAG_MAXLEN
+32];
3255 long masterfd
= (long)parameters
;
3256 const int import_tags
[] = { tag_playcount
, tag_rating
, tag_playtime
, tag_lastplayed
,
3264 logf("%d/%s", line_n
, buf
);
3265 if (!read_tag(tag_data
, sizeof tag_data
, buf
, "filename"))
3267 logf("filename missing");
3272 idx_id
= find_index(tag_data
);
3275 logf("entry not found");
3279 if (!get_index(masterfd
, idx_id
, &idx
, false))
3281 logf("failed to retrieve index entry");
3285 /* Stop if tag has already been modified. */
3286 if (idx
.flag
& FLAG_DIRTYNUM
)
3289 logf("import: %s", tag_data
);
3291 idx
.flag
|= FLAG_DIRTYNUM
;
3292 for (i
= 0; i
< (long)(sizeof(import_tags
)/sizeof(import_tags
[0])); i
++)
3296 if (!read_tag(tag_data
, sizeof tag_data
, buf
,
3297 tagcache_tag_to_str(import_tags
[i
])))
3302 data
= atoi(tag_data
);
3306 idx
.tag_seek
[import_tags
[i
]] = data
;
3308 if (import_tags
[i
] == tag_lastplayed
&& data
> current_tcmh
.serial
)
3309 current_tcmh
.serial
= data
;
3310 else if (import_tags
[i
] == tag_commitid
&& data
>= current_tcmh
.commitid
)
3311 current_tcmh
.commitid
= data
+ 1;
3314 return write_index(masterfd
, idx_id
, &idx
) ? 0 : -5;
3318 bool tagcache_import_changelog(void)
3320 struct master_header myhdr
;
3321 struct tagcache_header tch
;
3332 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_RDONLY
);
3335 logf("failure to open changelog");
3339 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3347 filenametag_fd
= open_tag_fd(&tch
, tag_filename
, false);
3349 fast_readline(clfd
, buf
, sizeof buf
, (long *)masterfd
,
3350 parse_changelog_line
);
3355 if (filenametag_fd
>= 0)
3356 close(filenametag_fd
);
3360 update_master_header();
3366 bool tagcache_create_changelog(struct tagcache_search
*tcs
)
3368 struct master_header myhdr
;
3369 struct index_entry idx
;
3370 char buf
[TAG_MAXLEN
+32];
3378 if (!tagcache_search(tcs
, tag_filename
))
3381 /* Initialize the changelog */
3382 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_WRONLY
| O_CREAT
| O_TRUNC
);
3385 logf("failure to open changelog");
3389 if (tcs
->masterfd
< 0)
3391 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, false)) < 0)
3396 lseek(tcs
->masterfd
, 0, SEEK_SET
);
3397 ecread(tcs
->masterfd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
3400 write(clfd
, "## Changelog version 1\n", 23);
3402 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3404 if (ecread(tcs
->masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
3405 != sizeof(struct index_entry
))
3407 logf("read error #9");
3408 tagcache_search_finish(tcs
);
3413 /* Skip until the entry found has been modified. */
3414 if (! (idx
.flag
& FLAG_DIRTYNUM
) )
3417 /* Skip deleted entries too. */
3418 if (idx
.flag
& FLAG_DELETED
)
3421 /* Now retrieve all tags. */
3422 for (j
= 0; j
< TAG_COUNT
; j
++)
3424 if (tagcache_is_numeric_tag(j
))
3426 snprintf(temp
, sizeof temp
, "%d", idx
.tag_seek
[j
]);
3427 write_tag(clfd
, tagcache_tag_to_str(j
), temp
);
3432 tagcache_retrieve(tcs
, i
, tcs
->type
, buf
, sizeof buf
);
3433 write_tag(clfd
, tagcache_tag_to_str(j
), buf
);
3436 write(clfd
, "\n", 1);
3442 tagcache_search_finish(tcs
);
3447 static bool delete_entry(long idx_id
)
3452 struct index_entry idx
, myidx
;
3453 struct master_header myhdr
;
3454 char buf
[TAG_MAXLEN
+32];
3455 int in_use
[TAG_COUNT
];
3457 logf("delete_entry(): %ld", idx_id
);
3459 #ifdef HAVE_TC_RAMCACHE
3460 /* At first mark the entry removed from ram cache. */
3461 if (tc_stat
.ramcache
)
3462 hdr
->indices
[idx_id
].flag
|= FLAG_DELETED
;
3465 if ( (masterfd
= open_master_fd(&myhdr
, true) ) < 0)
3468 lseek(masterfd
, idx_id
* sizeof(struct index_entry
), SEEK_CUR
);
3469 if (ecread(masterfd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3470 != sizeof(struct index_entry
))
3472 logf("delete_entry(): read error");
3476 if (myidx
.flag
& FLAG_DELETED
)
3478 logf("delete_entry(): already deleted!");
3482 myidx
.flag
|= FLAG_DELETED
;
3483 lseek(masterfd
, -sizeof(struct index_entry
), SEEK_CUR
);
3484 if (ecwrite(masterfd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3485 != sizeof(struct index_entry
))
3487 logf("delete_entry(): write_error #1");
3491 /* Now check which tags are no longer in use (if any) */
3492 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3495 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
3496 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3498 struct index_entry
*idxp
;
3500 #ifdef HAVE_TC_RAMCACHE
3501 /* Use RAM DB if available for greater speed */
3502 if (tc_stat
.ramcache
)
3503 idxp
= &hdr
->indices
[i
];
3507 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
3508 != sizeof(struct index_entry
))
3510 logf("delete_entry(): read error #2");
3516 if (idxp
->flag
& FLAG_DELETED
)
3519 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3521 if (tagcache_is_numeric_tag(tag
))
3524 if (idxp
->tag_seek
[tag
] == myidx
.tag_seek
[tag
])
3529 /* Now delete all tags no longer in use. */
3530 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3532 struct tagcache_header tch
;
3533 int oldseek
= myidx
.tag_seek
[tag
];
3535 if (tagcache_is_numeric_tag(tag
))
3539 * Replace tag seek with a hash value of the field string data.
3540 * That way runtime statistics of moved or altered files can be
3543 #ifdef HAVE_TC_RAMCACHE
3544 if (tc_stat
.ramcache
&& tag
!= tag_filename
)
3546 struct tagfile_entry
*tfe
;
3547 int32_t *seek
= &hdr
->indices
[idx_id
].tag_seek
[tag
];
3549 tfe
= (struct tagfile_entry
*)&hdr
->tags
[tag
][*seek
];
3550 *seek
= crc_32(tfe
->tag_data
, strlen(tfe
->tag_data
), 0xffffffff);
3551 myidx
.tag_seek
[tag
] = *seek
;
3556 struct tagfile_entry tfe
;
3558 /* Open the index file, which contains the tag names. */
3559 if ((fd
= open_tag_fd(&tch
, tag
, true)) < 0)
3562 /* Skip the header block */
3563 lseek(fd
, myidx
.tag_seek
[tag
], SEEK_SET
);
3564 if (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
3565 != sizeof(struct tagfile_entry
))
3567 logf("delete_entry(): read error #3");
3571 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
3573 logf("delete_entry(): read error #4");
3577 myidx
.tag_seek
[tag
] = crc_32(buf
, strlen(buf
), 0xffffffff);
3582 logf("in use: %d/%d", tag
, in_use
[tag
]);
3591 #ifdef HAVE_TC_RAMCACHE
3592 /* Delete from ram. */
3593 if (tc_stat
.ramcache
&& tag
!= tag_filename
)
3595 struct tagfile_entry
*tagentry
= (struct tagfile_entry
*)&hdr
->tags
[tag
][oldseek
];
3596 tagentry
->tag_data
[0] = '\0';
3600 /* Open the index file, which contains the tag names. */
3603 if ((fd
= open_tag_fd(&tch
, tag
, true)) < 0)
3607 /* Skip the header block */
3608 lseek(fd
, oldseek
+ sizeof(struct tagfile_entry
), SEEK_SET
);
3610 /* Debug, print 10 first characters of the tag
3613 logf("TAG:%s", buf);
3614 lseek(fd, -10, SEEK_CUR);
3617 /* Write first data byte in tag as \0 */
3620 /* Now tag data has been removed */
3625 /* Write index entry back into master index. */
3626 lseek(masterfd
, sizeof(struct master_header
) +
3627 (idx_id
* sizeof(struct index_entry
)), SEEK_SET
);
3628 if (ecwrite(masterfd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3629 != sizeof(struct index_entry
))
3631 logf("delete_entry(): write_error #2");
3650 * Returns true if there is an event waiting in the queue
3651 * that requires the current operation to be aborted.
3653 static bool check_event_queue(void)
3655 struct queue_event ev
;
3657 queue_wait_w_tmo(&tagcache_queue
, &ev
, 0);
3662 case SYS_USB_CONNECTED
:
3663 /* Put the event back into the queue. */
3664 queue_post(&tagcache_queue
, ev
.id
, ev
.data
);
3672 #ifdef HAVE_TC_RAMCACHE
3673 static bool allocate_tagcache(void)
3675 struct master_header tcmh
;
3678 /* Load the header. */
3679 if ( (fd
= open_master_fd(&tcmh
, false)) < 0)
3688 * Now calculate the required cache size plus
3689 * some extra space for alignment fixes.
3691 tc_stat
.ramcache_allocated
= tcmh
.tch
.datasize
+ 128 + TAGCACHE_RESERVE
+
3692 sizeof(struct ramcache_header
) + TAG_COUNT
*sizeof(void *);
3693 hdr
= buffer_alloc(tc_stat
.ramcache_allocated
+ 128);
3694 memset(hdr
, 0, sizeof(struct ramcache_header
));
3695 memcpy(&hdr
->h
, &tcmh
, sizeof(struct master_header
));
3696 hdr
->indices
= (struct index_entry
*)(hdr
+ 1);
3697 logf("tagcache: %d bytes allocated.", tc_stat
.ramcache_allocated
);
3702 # ifdef HAVE_EEPROM_SETTINGS
3703 static bool tagcache_dumpload(void)
3705 struct statefile_header shdr
;
3710 fd
= open(TAGCACHE_STATEFILE
, O_RDONLY
);
3713 logf("no tagcache statedump");
3717 /* Check the statefile memory placement */
3718 hdr
= buffer_alloc(0);
3719 rc
= read(fd
, &shdr
, sizeof(struct statefile_header
));
3720 if (rc
!= sizeof(struct statefile_header
)
3721 /* || (long)hdr != (long)shdr.hdr */)
3723 logf("incorrect statefile");
3729 offpos
= (long)hdr
- (long)shdr
.hdr
;
3731 /* Lets allocate real memory and load it */
3732 hdr
= buffer_alloc(shdr
.tc_stat
.ramcache_allocated
);
3733 rc
= read(fd
, hdr
, shdr
.tc_stat
.ramcache_allocated
);
3736 if (rc
!= shdr
.tc_stat
.ramcache_allocated
)
3738 logf("read failure!");
3743 memcpy(&tc_stat
, &shdr
.tc_stat
, sizeof(struct tagcache_stat
));
3745 /* Now fix the pointers */
3746 hdr
->indices
= (struct index_entry
*)((long)hdr
->indices
+ offpos
);
3747 for (i
= 0; i
< TAG_COUNT
; i
++)
3748 hdr
->tags
[i
] += offpos
;
3753 static bool tagcache_dumpsave(void)
3755 struct statefile_header shdr
;
3758 if (!tc_stat
.ramcache
)
3761 fd
= open(TAGCACHE_STATEFILE
, O_WRONLY
| O_CREAT
| O_TRUNC
);
3764 logf("failed to create a statedump");
3768 /* Create the header */
3770 memcpy(&shdr
.tc_stat
, &tc_stat
, sizeof(struct tagcache_stat
));
3771 write(fd
, &shdr
, sizeof(struct statefile_header
));
3773 /* And dump the data too */
3774 write(fd
, hdr
, tc_stat
.ramcache_allocated
);
3781 static bool load_tagcache(void)
3783 struct tagcache_header
*tch
;
3784 long bytesleft
= tc_stat
.ramcache_allocated
;
3785 struct index_entry
*idx
;
3790 # ifdef HAVE_DIRCACHE
3791 while (dircache_is_initializing())
3794 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE
);
3797 logf("loading tagcache to ram...");
3799 fd
= open(TAGCACHE_FILE_MASTER
, O_RDONLY
);
3802 logf("tagcache open failed");
3806 if (ecread(fd
, &hdr
->h
, 1, master_header_ec
, tc_stat
.econ
)
3807 != sizeof(struct master_header
)
3808 || hdr
->h
.tch
.magic
!= TAGCACHE_MAGIC
)
3810 logf("incorrect header");
3816 /* Load the master index table. */
3817 for (i
= 0; i
< hdr
->h
.tch
.entry_count
; i
++)
3819 rc
= ecread(fd
, idx
, 1, index_entry_ec
, tc_stat
.econ
);
3820 if (rc
!= sizeof(struct index_entry
))
3822 logf("read error #10");
3827 bytesleft
-= sizeof(struct index_entry
);
3828 if (bytesleft
< 0 || ((long)idx
- (long)hdr
->indices
) >= tc_stat
.ramcache_allocated
)
3830 logf("too big tagcache.");
3840 /* Load the tags. */
3842 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3844 struct tagfile_entry
*fe
;
3845 char buf
[TAG_MAXLEN
+32];
3847 if (tagcache_is_numeric_tag(tag
))
3850 //p = ((void *)p+1);
3851 p
= (char *)((long)p
& ~0x03) + 0x04;
3854 /* Check the header. */
3855 tch
= (struct tagcache_header
*)p
;
3856 p
+= sizeof(struct tagcache_header
);
3858 if ( (fd
= open_tag_fd(tch
, tag
, false)) < 0)
3861 for (hdr
->entry_count
[tag
] = 0;
3862 hdr
->entry_count
[tag
] < tch
->entry_count
;
3863 hdr
->entry_count
[tag
]++)
3867 if (do_timed_yield())
3869 /* Abort if we got a critical event in queue */
3870 if (check_event_queue())
3874 fe
= (struct tagfile_entry
*)p
;
3875 pos
= lseek(fd
, 0, SEEK_CUR
);
3876 rc
= ecread(fd
, fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
3877 if (rc
!= sizeof(struct tagfile_entry
))
3879 /* End of lookup table. */
3880 logf("read error #11");
3885 /* We have a special handling for the filename tags. */
3886 if (tag
== tag_filename
)
3888 # ifdef HAVE_DIRCACHE
3889 const struct dirent
*dc
;
3892 // FIXME: This is wrong!
3893 // idx = &hdr->indices[hdr->entry_count[i]];
3894 idx
= &hdr
->indices
[fe
->idx_id
];
3896 if (fe
->tag_length
>= (long)sizeof(buf
)-1)
3900 logf("TAG:%s", buf
);
3901 logf("too long filename");
3906 rc
= read(fd
, buf
, fe
->tag_length
);
3907 if (rc
!= fe
->tag_length
)
3909 logf("read error #12");
3914 /* Check if the entry has already been removed */
3915 if (idx
->flag
& FLAG_DELETED
)
3918 /* This flag must not be used yet. */
3919 if (idx
->flag
& FLAG_DIRCACHE
)
3921 logf("internal error!");
3926 if (idx
->tag_seek
[tag
] != pos
)
3928 logf("corrupt data structures!");
3933 # ifdef HAVE_DIRCACHE
3934 if (dircache_is_enabled())
3936 dc
= dircache_get_entry_ptr(buf
);
3939 logf("Entry no longer valid.");
3941 delete_entry(fe
->idx_id
);
3945 idx
->flag
|= FLAG_DIRCACHE
;
3946 FLAG_SET_ATTR(idx
->flag
, fe
->idx_id
);
3947 idx
->tag_seek
[tag_filename
] = (long)dc
;
3952 /* This will be very slow unless dircache is enabled
3953 or target is flash based, but do it anyway for
3955 /* Check if entry has been removed. */
3956 if (global_settings
.tagcache_autoupdate
)
3958 if (!file_exists(buf
))
3960 logf("Entry no longer valid.");
3962 delete_entry(fe
->idx_id
);
3971 bytesleft
-= sizeof(struct tagfile_entry
) + fe
->tag_length
;
3974 logf("too big tagcache #2");
3975 logf("tl: %d", fe
->tag_length
);
3976 logf("bl: %ld", bytesleft
);
3982 rc
= read(fd
, fe
->tag_data
, fe
->tag_length
);
3985 if (rc
!= fe
->tag_length
)
3987 logf("read error #13");
3988 logf("rc=0x%04x", rc
); // 0x431
3989 logf("len=0x%04x", fe
->tag_length
); // 0x4000
3990 logf("pos=0x%04lx", lseek(fd
, 0, SEEK_CUR
)); // 0x433
3991 logf("tag=0x%02x", tag
); // 0x00
3999 tc_stat
.ramcache_used
= tc_stat
.ramcache_allocated
- bytesleft
;
4000 logf("tagcache loaded into ram!");
4004 #endif /* HAVE_TC_RAMCACHE */
4006 static bool check_deleted_files(void)
4009 char buf
[TAG_MAXLEN
+32];
4010 struct tagfile_entry tfe
;
4012 logf("reverse scan...");
4013 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag_filename
);
4014 fd
= open(buf
, O_RDONLY
);
4018 logf("%s open fail", buf
);
4022 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
4023 while (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
4024 == sizeof(struct tagfile_entry
)
4026 && !check_event_queue()
4030 if (tfe
.tag_length
>= (long)sizeof(buf
)-1)
4032 logf("too long tag");
4037 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
4039 logf("read error #14");
4044 /* Check if the file has already deleted from the db. */
4048 /* Now check if the file exists. */
4049 if (!file_exists(buf
))
4051 logf("Entry no longer valid.");
4052 logf("-> %s / %d", buf
, tfe
.tag_length
);
4053 delete_entry(tfe
.idx_id
);
4064 static bool check_dir(const char *dirname
, int add_files
)
4068 int success
= false;
4069 int ignore
, unignore
;
4070 char newpath
[MAX_PATH
];
4072 dir
= opendir(dirname
);
4075 logf("tagcache: opendir() failed");
4079 /* check for a database.ignore file */
4080 snprintf(newpath
, MAX_PATH
, "%s/database.ignore", dirname
);
4081 ignore
= file_exists(newpath
);
4082 /* check for a database.unignore file */
4083 snprintf(newpath
, MAX_PATH
, "%s/database.unignore", dirname
);
4084 unignore
= file_exists(newpath
);
4086 /* don't do anything if both ignore and unignore are there */
4087 if (ignore
!= unignore
)
4088 add_files
= unignore
;
4090 /* Recursively scan the dir. */
4094 while (!check_event_queue())
4097 struct dirent
*entry
;
4099 entry
= readdir(dir
);
4107 if (!strcmp((char *)entry
->d_name
, ".") ||
4108 !strcmp((char *)entry
->d_name
, ".."))
4113 len
= strlen(curpath
);
4114 snprintf(&curpath
[len
], curpath_size
- len
, "/%s",
4117 processed_dir_count
++;
4118 if (entry
->attribute
& ATTR_DIRECTORY
)
4119 check_dir(curpath
, add_files
);
4122 tc_stat
.curentry
= curpath
;
4124 /* Add a new entry to the temporary db file. */
4125 add_tagcache(curpath
, (entry
->wrtdate
<< 16) | entry
->wrttime
4126 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4127 , dir
->internal_entry
4131 /* Wait until current path for debug screen is read and unset. */
4132 while (tc_stat
.syncscreen
&& tc_stat
.curentry
!= NULL
)
4135 tc_stat
.curentry
= NULL
;
4138 curpath
[len
] = '\0';
4146 void tagcache_screensync_event(void)
4148 tc_stat
.curentry
= NULL
;
4151 void tagcache_screensync_enable(bool state
)
4153 tc_stat
.syncscreen
= state
;
4156 void build_tagcache(const char *path
)
4158 struct tagcache_header header
;
4163 total_entry_count
= 0;
4164 processed_dir_count
= 0;
4166 #ifdef HAVE_DIRCACHE
4167 while (dircache_is_initializing())
4171 logf("updating tagcache");
4173 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
4176 logf("skipping, cache already waiting for commit");
4181 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDWR
| O_CREAT
| O_TRUNC
);
4184 logf("master file open failed: %s", TAGCACHE_FILE_TEMP
);
4188 filenametag_fd
= open_tag_fd(&header
, tag_filename
, false);
4192 logf("Scanning files...");
4193 /* Scan for new files. */
4194 memset(&header
, 0, sizeof(struct tagcache_header
));
4195 write(cachefd
, &header
, sizeof(struct tagcache_header
));
4197 if (strcmp("/", path
) != 0)
4198 strcpy(curpath
, path
);
4199 ret
= check_dir(path
, true);
4201 /* Write the header. */
4202 header
.magic
= TAGCACHE_MAGIC
;
4203 header
.datasize
= data_size
;
4204 header
.entry_count
= total_entry_count
;
4205 lseek(cachefd
, 0, SEEK_SET
);
4206 write(cachefd
, &header
, sizeof(struct tagcache_header
));
4209 if (filenametag_fd
>= 0)
4211 close(filenametag_fd
);
4212 filenametag_fd
= -1;
4222 /* Commit changes to the database. */
4228 remove(TAGCACHE_FILE_TEMP
);
4229 logf("tagcache built!");
4235 #ifdef HAVE_TC_RAMCACHE
4238 /* Import runtime statistics if we just initialized the db. */
4239 if (hdr
->h
.serial
== 0)
4240 queue_post(&tagcache_queue
, Q_IMPORT_CHANGELOG
, 0);
4247 #ifdef HAVE_TC_RAMCACHE
4248 static void load_ramcache(void)
4255 /* At first we should load the cache (if exists). */
4256 tc_stat
.ramcache
= load_tagcache();
4258 if (!tc_stat
.ramcache
)
4260 /* If loading failed, it must indicate some problem with the db
4261 * so disable it entirely to prevent further issues. */
4262 tc_stat
.ready
= false;
4269 void tagcache_unload_ramcache(void)
4271 tc_stat
.ramcache
= false;
4272 /* Just to make sure there is no statefile present. */
4273 // remove(TAGCACHE_STATEFILE);
4278 static void tagcache_thread(void)
4280 struct queue_event ev
;
4281 bool check_done
= false;
4283 /* If the previous cache build/update was interrupted, commit
4284 * the changes first in foreground. */
4290 #ifdef HAVE_TC_RAMCACHE
4291 # ifdef HAVE_EEPROM_SETTINGS
4292 if (firmware_settings
.initialized
&& firmware_settings
.disk_clean
)
4293 check_done
= tagcache_dumpload();
4295 remove(TAGCACHE_STATEFILE
);
4298 /* Allocate space for the tagcache if found on disk. */
4299 if (global_settings
.tagcache_ram
&& !tc_stat
.ramcache
)
4300 allocate_tagcache();
4304 tc_stat
.initialized
= true;
4306 /* Don't delay bootup with the header check but do it on background. */
4308 tc_stat
.ready
= check_all_headers();
4309 tc_stat
.readyvalid
= true;
4313 run_command_queue(false);
4315 queue_wait_w_tmo(&tagcache_queue
, &ev
, HZ
);
4319 case Q_IMPORT_CHANGELOG
:
4320 tagcache_import_changelog();
4325 build_tagcache("/");
4329 build_tagcache("/");
4330 #ifdef HAVE_TC_RAMCACHE
4333 check_deleted_files();
4339 if (check_done
|| !tc_stat
.ready
)
4342 #ifdef HAVE_TC_RAMCACHE
4343 if (!tc_stat
.ramcache
&& global_settings
.tagcache_ram
)
4346 if (tc_stat
.ramcache
&& global_settings
.tagcache_autoupdate
)
4347 build_tagcache("/");
4351 if (global_settings
.tagcache_autoupdate
)
4353 build_tagcache("/");
4355 /* This will be very slow unless dircache is enabled
4356 or target is flash based, but do it anyway for
4358 check_deleted_files();
4361 logf("tagcache check done");
4373 case SYS_USB_CONNECTED
:
4374 logf("USB: TagCache");
4375 usb_acknowledge(SYS_USB_CONNECTED_ACK
);
4376 usb_wait_for_disconnect(&tagcache_queue
);
4383 bool tagcache_prepare_shutdown(void)
4385 if (tagcache_get_commit_step() > 0)
4388 tagcache_stop_scan();
4389 while (read_lock
|| write_lock
)
4395 void tagcache_shutdown(void)
4397 /* Flush the command queue. */
4398 run_command_queue(true);
4400 #ifdef HAVE_EEPROM_SETTINGS
4401 if (tc_stat
.ramcache
)
4402 tagcache_dumpsave();
4406 static int get_progress(void)
4408 int total_count
= -1;
4410 #ifdef HAVE_DIRCACHE
4411 if (dircache_is_enabled())
4413 total_count
= dircache_get_entry_count();
4417 #ifdef HAVE_TC_RAMCACHE
4419 if (hdr
&& tc_stat
.ramcache
)
4420 total_count
= hdr
->h
.tch
.entry_count
;
4424 if (total_count
< 0)
4427 return processed_dir_count
* 100 / total_count
;
4430 struct tagcache_stat
* tagcache_get_stat(void)
4432 tc_stat
.progress
= get_progress();
4433 tc_stat
.processed_entries
= processed_dir_count
;
4438 void tagcache_start_scan(void)
4440 queue_post(&tagcache_queue
, Q_START_SCAN
, 0);
4443 bool tagcache_update(void)
4448 queue_post(&tagcache_queue
, Q_UPDATE
, 0);
4452 bool tagcache_rebuild()
4454 queue_post(&tagcache_queue
, Q_REBUILD
, 0);
4458 void tagcache_stop_scan(void)
4460 queue_post(&tagcache_queue
, Q_STOP_SCAN
, 0);
4463 #ifdef HAVE_TC_RAMCACHE
4464 bool tagcache_is_ramcache(void)
4466 return tc_stat
.ramcache
;
4470 #endif /* !__PCTOOL__ */
4473 void tagcache_init(void)
4475 memset(&tc_stat
, 0, sizeof(struct tagcache_stat
));
4476 memset(¤t_tcmh
, 0, sizeof(struct master_header
));
4477 filenametag_fd
= -1;
4478 write_lock
= read_lock
= 0;
4481 mutex_init(&command_queue_mutex
);
4482 queue_init(&tagcache_queue
, true);
4483 create_thread(tagcache_thread
, tagcache_stack
,
4484 sizeof(tagcache_stack
), 0, tagcache_thread_name
4485 IF_PRIO(, PRIORITY_BACKGROUND
)
4488 tc_stat
.initialized
= true;
4492 tc_stat
.ready
= check_all_headers();
4497 void tagcache_reverse_scan(void)
4499 logf("Checking for deleted files");
4500 check_deleted_files();
4504 bool tagcache_is_initialized(void)
4506 return tc_stat
.initialized
;
4508 bool tagcache_is_usable(void)
4510 return tc_stat
.initialized
&& tc_stat
.ready
;
4512 int tagcache_get_commit_step(void)
4514 return tc_stat
.commit_step
;
4516 int tagcache_get_max_commit_step(void)
4518 return (int)(sizeof(sorted_tags
)/sizeof(sorted_tags
[0]))+1;