?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Sim I/O and threading that runs more like on target. Tweakable if any genuine slownes...
[Rockbox.git] / apps / tagcache.c
blobb1811061b2360c6febc2f9d59018302c241dbb2c
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "thread.h"
61 #include "kernel.h"
62 #include "system.h"
63 #include "logf.h"
64 #include "string.h"
65 #include "usb.h"
66 #include "metadata.h"
67 #include "id3.h"
68 #include "tagcache.h"
69 #include "buffer.h"
70 #include "crc32.h"
71 #include "misc.h"
72 #include "settings.h"
73 #include "dir.h"
74 #include "structec.h"
75 #ifndef __PCTOOL__
76 #include "atoi.h"
77 #include "splash.h"
78 #include "lang.h"
79 #include "eeprom_settings.h"
80 #endif
81
82 #ifdef __PCTOOL__
83 #define yield() do { } while(0)
84 #define sim_sleep(timeout) do { } while(0)
85 #define do_timed_yield() do { } while(0)
86 #endif
87
88
89 #ifndef __PCTOOL__
90 /* Tag Cache thread. */
91 static struct event_queue tagcache_queue;
92 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
93 static const char tagcache_thread_name[] = "tagcache";
94 #endif
95
96 #define UNTAGGED "<Untagged>"
97
98 /* Previous path when scanning directory tree recursively. */
99 static char curpath[TAG_MAXLEN+32];
100 static long curpath_size = sizeof(curpath);
101
102 /* Used when removing duplicates. */
103 static char *tempbuf; /* Allocated when needed. */
104 static long tempbufidx; /* Current location in buffer. */
105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
106 static long tempbuf_left; /* Buffer space left. */
107 static long tempbuf_pos;
108
109 /* Tags we want to get sorted (loaded to the tempbuf). */
110 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
111 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
112
113 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
114 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
115 tag_composer, tag_comment, tag_albumartist, tag_grouping };
116
117 /* Numeric tags (we can use these tags with conditional clauses). */
118 static const int numeric_tags[] = { tag_year, tag_discnumber, tag_tracknumber, tag_length,
119 tag_bitrate, tag_playcount, tag_rating, tag_playtime, tag_lastplayed, tag_commitid,
120 tag_virt_length_min, tag_virt_length_sec,
121 tag_virt_playtime_min, tag_virt_playtime_sec,
122 tag_virt_entryage, tag_virt_autoscore };
123
124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
125 static const char *tags_str[] = { "artist", "album", "genre", "title",
126 "filename", "composer", "comment", "albumartist", "grouping", "year", "discnumber", "tracknumber",
127 "bitrate", "length", "playcount", "rating", "playtime", "lastplayed", "commitid" };
128
129 /* Status information of the tagcache. */
130 static struct tagcache_stat tc_stat;
131
132 /* Queue commands. */
133 enum tagcache_queue {
134 Q_STOP_SCAN = 0,
135 Q_START_SCAN,
136 Q_IMPORT_CHANGELOG,
137 Q_UPDATE,
138 Q_REBUILD,
139
140 /* Internal tagcache command queue. */
141 CMD_UPDATE_MASTER_HEADER,
142 CMD_UPDATE_NUMERIC,
143 };
144
145 struct tagcache_command_entry {
146 long command;
147 long idx_id;
148 long tag;
149 long data;
150 };
151
152 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
153 static volatile int command_queue_widx = 0;
154 static volatile int command_queue_ridx = 0;
155 static struct mutex command_queue_mutex;
156 /* Timestamp of the last added event, so we can wait a bit before committing the
157 * whole queue at once. */
158 static long command_queue_timestamp = 0;
159
160 /* Tag database structures. */
161
162 /* Variable-length tag entry in tag files. */
163 struct tagfile_entry {
164 short tag_length; /* Length of the data in bytes including '\0' */
165 short idx_id; /* Corresponding entry location in index file of not unique tags */
166 char tag_data[0]; /* Begin of the tag data */
167 };
168
169 /* Fixed-size tag entry in master db index. */
170 struct index_entry {
171 long tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
172 long flag; /* Status flags */
173 };
174
175 /* Header is the same in every file. */
176 struct tagcache_header {
177 long magic; /* Header version number */
178 long datasize; /* Data size in bytes */
179 long entry_count; /* Number of entries in this file */
180 };
181
182 struct master_header {
183 struct tagcache_header tch;
184 long serial; /* Increasing counting number */
185 long commitid; /* Number of commits so far */
186 long dirty;
187 };
188
189 /* For the endianess correction */
190 static const char *tagfile_entry_ec = "ss";
191 static const char *index_entry_ec = "llllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
192 static const char *tagcache_header_ec = "lll";
193 static const char *master_header_ec = "llllll";
194
195 static struct master_header current_tcmh;
196
197 #ifdef HAVE_TC_RAMCACHE
198 /* Header is created when loading database to ram. */
199 struct ramcache_header {
200 struct master_header h; /* Header from the master index */
201 struct index_entry *indices; /* Master index file content */
202 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
203 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
204 };
205
206 # ifdef HAVE_EEPROM_SETTINGS
207 struct statefile_header {
208 struct ramcache_header *hdr;
209 struct tagcache_stat tc_stat;
210 };
211 # endif
212
213 /* Pointer to allocated ramcache_header */
214 static struct ramcache_header *hdr;
215 #endif
216
217 /**
218 * Full tag entries stored in a temporary file waiting
219 * for commit to the cache. */
220 struct temp_file_entry {
221 long tag_offset[TAG_COUNT];
222 short tag_length[TAG_COUNT];
223 long flag;
224
225 long data_length;
226 };
227
228 struct tempbuf_id_list {
229 long id;
230 struct tempbuf_id_list *next;
231 };
232
233 struct tempbuf_searchidx {
234 long idx_id;
235 char *str;
236 int seek;
237 struct tempbuf_id_list idlist;
238 };
239
240 /* Lookup buffer for fixing messed up index while after sorting. */
241 static long commit_entry_count;
242 static long lookup_buffer_depth;
243 static struct tempbuf_searchidx **lookup;
244
245 /* Used when building the temporary file. */
246 static int cachefd = -1, filenametag_fd;
247 static int total_entry_count = 0;
248 static int data_size = 0;
249 static int processed_dir_count;
250
251 /* Thread safe locking */
252 static volatile int write_lock;
253 static volatile int read_lock;
254
255 const char* tagcache_tag_to_str(int tag)
256 {
257 return tags_str[tag];
258 }
259
260 bool tagcache_is_numeric_tag(int type)
261 {
262 int i;
263
264 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
265 {
266 if (type == numeric_tags[i])
267 return true;
268 }
269
270 return false;
271 }
272
273 bool tagcache_is_unique_tag(int type)
274 {
275 int i;
276
277 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
278 {
279 if (type == unique_tags[i])
280 return true;
281 }
282
283 return false;
284 }
285
286 bool tagcache_is_sorted_tag(int type)
287 {
288 int i;
289
290 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
291 {
292 if (type == sorted_tags[i])
293 return true;
294 }
295
296 return false;
297 }
298
299 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
300 {
301 int fd;
302 char buf[MAX_PATH];
303 int rc;
304
305 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
306 return -1;
307
308 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
309
310 fd = open(buf, write ? O_RDWR : O_RDONLY);
311 if (fd < 0)
312 {
313 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
314 tc_stat.ready = false;
315 return fd;
316 }
317
318 /* Check the header. */
319 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
320 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
321 {
322 logf("header error");
323 tc_stat.ready = false;
324 close(fd);
325 return -2;
326 }
327
328 return fd;
329 }
330
331 #ifndef __PCTOOL__
332 static bool do_timed_yield(void)
333 {
334 /* Sorting can lock up for quite a while, so yield occasionally */
335 static long wakeup_tick = 0;
336 if (current_tick >= wakeup_tick)
337 {
338 wakeup_tick = current_tick + (HZ/4);
339 yield();
340 return true;
341 }
342 return false;
343 }
344 #endif
345
346 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
347 static long find_entry_ram(const char *filename,
348 const struct dirent *dc)
349 {
350 static long last_pos = 0;
351 int i;
352
353 /* Check if we tagcache is loaded into ram. */
354 if (!tc_stat.ramcache)
355 return -1;
356
357 if (dc == NULL)
358 dc = dircache_get_entry_ptr(filename);
359
360 if (dc == NULL)
361 {
362 logf("tagcache: file not found.");
363 return -1;
364 }
365
366 try_again:
367
368 if (last_pos > 0)
369 i = last_pos;
370 else
371 i = 0;
372
373 for (; i < hdr->h.tch.entry_count; i++)
374 {
375 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
376 {
377 last_pos = MAX(0, i - 3);
378 return i;
379 }
380
381 do_timed_yield();
382 }
383
384 if (last_pos > 0)
385 {
386 last_pos = 0;
387 goto try_again;
388 }
389
390 return -1;
391 }
392 #endif
393
394 static long find_entry_disk(const char *filename)
395 {
396 struct tagcache_header tch;
397 static long last_pos = -1;
398 long pos_history[POS_HISTORY_COUNT];
399 long pos_history_idx = 0;
400 bool found = false;
401 struct tagfile_entry tfe;
402 int fd;
403 char buf[TAG_MAXLEN+32];
404 int i;
405 int pos = -1;
406
407 if (!tc_stat.ready)
408 return -2;
409
410 fd = filenametag_fd;
411 if (fd < 0)
412 {
413 last_pos = -1;
414 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
415 return -1;
416 }
417
418 check_again:
419
420 if (last_pos > 0)
421 lseek(fd, last_pos, SEEK_SET);
422 else
423 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
424
425 while (true)
426 {
427 pos = lseek(fd, 0, SEEK_CUR);
428 for (i = pos_history_idx-1; i >= 0; i--)
429 pos_history[i+1] = pos_history[i];
430 pos_history[0] = pos;
431
432 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
433 != sizeof(struct tagfile_entry))
434 {
435 break ;
436 }
437
438 if (tfe.tag_length >= (long)sizeof(buf))
439 {
440 logf("too long tag #1");
441 close(fd);
442 last_pos = -1;
443 return -2;
444 }
445
446 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
447 {
448 logf("read error #2");
449 close(fd);
450 last_pos = -1;
451 return -3;
452 }
453
454 if (!strcasecmp(filename, buf))
455 {
456 last_pos = pos_history[pos_history_idx];
457 found = true;
458 break ;
459 }
460
461 if (pos_history_idx < POS_HISTORY_COUNT - 1)
462 pos_history_idx++;
463 }
464
465 /* Not found? */
466 if (!found)
467 {
468 if (last_pos > 0)
469 {
470 last_pos = -1;
471 logf("seek again");
472 goto check_again;
473 }
474
475 if (fd != filenametag_fd)
476 close(fd);
477 return -4;
478 }
479
480 if (fd != filenametag_fd)
481 close(fd);
482
483 return tfe.idx_id;
484 }
485
486 static int find_index(const char *filename)
487 {
488 long idx_id = -1;
489
490 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
491 if (tc_stat.ramcache && dircache_is_enabled())
492 idx_id = find_entry_ram(filename, NULL);
493 #endif
494
495 if (idx_id < 0)
496 idx_id = find_entry_disk(filename);
497
498 return idx_id;
499 }
500
501 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
502 {
503 int idx_id;
504
505 if (!tc_stat.ready)
506 return false;
507
508 idx_id = find_index(filename);
509 if (idx_id < 0)
510 return false;
511
512 if (!tagcache_search(tcs, tag_filename))
513 return false;
514
515 tcs->entry_count = 0;
516 tcs->idx_id = idx_id;
517
518 return true;
519 }
520
521 static bool get_index(int masterfd, int idxid,
522 struct index_entry *idx, bool use_ram)
523 {
524 if (idxid < 0)
525 {
526 logf("Incorrect idxid: %d", idxid);
527 return false;
528 }
529
530 #ifdef HAVE_TC_RAMCACHE
531 if (tc_stat.ramcache && use_ram)
532 {
533 if (hdr->indices[idxid].flag & FLAG_DELETED)
534 return false;
535
536 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
537 return true;
538 }
539 #else
540 (void)use_ram;
541 #endif
542
543 lseek(masterfd, idxid * sizeof(struct index_entry)
544 + sizeof(struct master_header), SEEK_SET);
545 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
546 != sizeof(struct index_entry))
547 {
548 logf("read error #3");
549 return false;
550 }
551
552 if (idx->flag & FLAG_DELETED)
553 return false;
554
555 return true;
556 }
557
558 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
559 {
560 /* We need to exclude all memory only flags & tags when writing to disk. */
561 if (idx->flag & FLAG_DIRCACHE)
562 {
563 logf("memory only flags!");
564 return false;
565 }
566
567 #ifdef HAVE_TC_RAMCACHE
568 /* Only update numeric data. Writing the whole index to RAM by memcpy
569 * destroys dircache pointers!
570 */
571 if (tc_stat.ramcache)
572 {
573 int tag;
574 struct index_entry *idx_ram = &hdr->indices[idxid];
575
576 for (tag = 0; tag < TAG_COUNT; tag++)
577 {
578 if (tagcache_is_numeric_tag(tag))
579 {
580 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
581 }
582 }
583
584 /* Don't touch the dircache flag. */
585 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
586 }
587 #endif
588
589 lseek(masterfd, idxid * sizeof(struct index_entry)
590 + sizeof(struct master_header), SEEK_SET);
591 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
592 != sizeof(struct index_entry))
593 {
594 logf("write error #3");
595 logf("idxid: %d", idxid);
596 return false;
597 }
598
599 return true;
600 }
601
602 static bool open_files(struct tagcache_search *tcs, int tag)
603 {
604 if (tcs->idxfd[tag] < 0)
605 {
606 char fn[MAX_PATH];
607
608 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
609 tcs->idxfd[tag] = open(fn, O_RDONLY);
610 }
611
612 if (tcs->idxfd[tag] < 0)
613 {
614 logf("File not open!");
615 return false;
616 }
617
618 return true;
619 }
620
621 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
622 int tag, char *buf, long size)
623 {
624 struct tagfile_entry tfe;
625 long seek;
626
627 *buf = '\0';
628
629 if (tagcache_is_numeric_tag(tag))
630 return false;
631
632 seek = idx->tag_seek[tag];
633 if (seek < 0)
634 {
635 logf("Retrieve failed");
636 return false;
637 }
638
639 #ifdef HAVE_TC_RAMCACHE
640 if (tcs->ramsearch)
641 {
642 struct tagfile_entry *ep;
643
644 # ifdef HAVE_DIRCACHE
645 if (tag == tag_filename && idx->flag & FLAG_DIRCACHE)
646 {
647 dircache_copy_path((struct dirent *)seek,
648 buf, size);
649 return true;
650 }
651 else
652 # endif
653 if (tag != tag_filename)
654 {
655 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
656 strncpy(buf, ep->tag_data, size-1);
657
658 return true;
659 }
660 }
661 #endif
662
663 if (!open_files(tcs, tag))
664 return false;
665
666 lseek(tcs->idxfd[tag], seek, SEEK_SET);
667 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
668 != sizeof(struct tagfile_entry))
669 {
670 logf("read error #5");
671 return false;
672 }
673
674 if (tfe.tag_length >= size)
675 {
676 logf("too small buffer");
677 return false;
678 }
679
680 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
681 tfe.tag_length)
682 {
683 logf("read error #6");
684 return false;
685 }
686
687 buf[tfe.tag_length] = '\0';
688
689 return true;
690 }
691
692 static long check_virtual_tags(int tag, const struct index_entry *idx)
693 {
694 long data = 0;
695
696 switch (tag)
697 {
698 case tag_virt_length_sec:
699 data = (idx->tag_seek[tag_length]/1000) % 60;
700 break;
701
702 case tag_virt_length_min:
703 data = (idx->tag_seek[tag_length]/1000) / 60;
704 break;
705
706 case tag_virt_playtime_sec:
707 data = (idx->tag_seek[tag_playtime]/1000) % 60;
708 break;
709
710 case tag_virt_playtime_min:
711 data = (idx->tag_seek[tag_playtime]/1000) / 60;
712 break;
713
714 case tag_virt_autoscore:
715 if (idx->tag_seek[tag_length] == 0
716 || idx->tag_seek[tag_playcount] == 0)
717 {
718 data = 0;
719 }
720 else
721 {
722 data = 100 * idx->tag_seek[tag_playtime]
723 / idx->tag_seek[tag_length]
724 / idx->tag_seek[tag_playcount];
725 }
726 break;
727
728 /* How many commits before the file has been added to the DB. */
729 case tag_virt_entryage:
730 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
731 break;
732
733 default:
734 data = idx->tag_seek[tag];
735 }
736
737 return data;
738 }
739
740 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
741 {
742 struct index_entry idx;
743
744 if (!tc_stat.ready)
745 return false;
746
747 if (!tagcache_is_numeric_tag(tag))
748 return -1;
749
750 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
751 return -2;
752
753 return check_virtual_tags(tag, &idx);
754 }
755
756 inline static bool str_ends_with(const char *str1, const char *str2)
757 {
758 int str_len = strlen(str1);
759 int clause_len = strlen(str2);
760
761 if (clause_len > str_len)
762 return false;
763
764 return !strcasecmp(&str1[str_len - clause_len], str2);
765 }
766
767 inline static bool str_oneof(const char *str, const char *list)
768 {
769 const char *sep;
770 int l, len = strlen(str);
771
772 while (*list)
773 {
774 sep = strchr(list, '|');
775 l = sep ? (long)sep - (long)list : (int)strlen(list);
776 if ((l==len) && !strncasecmp(str, list, len))
777 return true;
778 list += sep ? l + 1 : l;
779 }
780
781 return false;
782 }
783
784 static bool check_against_clause(long numeric, const char *str,
785 const struct tagcache_search_clause *clause)
786 {
787 if (clause->numeric)
788 {
789 switch (clause->type)
790 {
791 case clause_is:
792 return numeric == clause->numeric_data;
793 case clause_is_not:
794 return numeric != clause->numeric_data;
795 case clause_gt:
796 return numeric > clause->numeric_data;
797 case clause_gteq:
798 return numeric >= clause->numeric_data;
799 case clause_lt:
800 return numeric < clause->numeric_data;
801 case clause_lteq:
802 return numeric <= clause->numeric_data;
803 default:
804 logf("Incorrect numeric tag: %d", clause->type);
805 }
806 }
807 else
808 {
809 switch (clause->type)
810 {
811 case clause_is:
812 return !strcasecmp(clause->str, str);
813 case clause_is_not:
814 return strcasecmp(clause->str, str);
815 case clause_gt:
816 return 0>strcasecmp(clause->str, str);
817 case clause_gteq:
818 return 0>=strcasecmp(clause->str, str);
819 case clause_lt:
820 return 0<strcasecmp(clause->str, str);
821 case clause_lteq:
822 return 0<=strcasecmp(clause->str, str);
823 case clause_contains:
824 return (strcasestr(str, clause->str) != NULL);
825 case clause_not_contains:
826 return (strcasestr(str, clause->str) == NULL);
827 case clause_begins_with:
828 return (strcasestr(str, clause->str) == str);
829 case clause_not_begins_with:
830 return (strcasestr(str, clause->str) != str);
831 case clause_ends_with:
832 return str_ends_with(str, clause->str);
833 case clause_not_ends_with:
834 return !str_ends_with(str, clause->str);
835 case clause_oneof:
836 return str_oneof(str, clause->str);
837
838 default:
839 logf("Incorrect tag: %d", clause->type);
840 }
841 }
842
843 return false;
844 }
845
846 static bool check_clauses(struct tagcache_search *tcs,
847 struct index_entry *idx,
848 struct tagcache_search_clause **clause, int count)
849 {
850 int i;
851
852 #ifdef HAVE_TC_RAMCACHE
853 if (tcs->ramsearch)
854 {
855 /* Go through all conditional clauses. */
856 for (i = 0; i < count; i++)
857 {
858 struct tagfile_entry *tfe;
859 int seek;
860 char buf[256];
861 char *str = NULL;
862
863 seek = check_virtual_tags(clause[i]->tag, idx);
864
865 if (!tagcache_is_numeric_tag(clause[i]->tag))
866 {
867 if (clause[i]->tag == tag_filename)
868 {
869 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
870 str = buf;
871 }
872 else
873 {
874 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
875 str = tfe->tag_data;
876 }
877 }
878
879 if (!check_against_clause(seek, str, clause[i]))
880 return false;
881 }
882 }
883 else
884 #endif
885 {
886 /* Check for conditions. */
887 for (i = 0; i < count; i++)
888 {
889 struct tagfile_entry tfe;
890 int seek;
891 char str[256];
892
893 seek = check_virtual_tags(clause[i]->tag, idx);
894
895 memset(str, 0, sizeof str);
896 if (!tagcache_is_numeric_tag(clause[i]->tag))
897 {
898 int fd = tcs->idxfd[clause[i]->tag];
899 lseek(fd, seek, SEEK_SET);
900 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
901 if (tfe.tag_length >= (int)sizeof(str))
902 {
903 logf("Too long tag read!");
904 break ;
905 }
906
907 read(fd, str, tfe.tag_length);
908
909 /* Check if entry has been deleted. */
910 if (str[0] == '\0')
911 break;
912 }
913
914 if (!check_against_clause(seek, str, clause[i]))
915 return false;
916 }
917 }
918
919 return true;
920 }
921
922 bool tagcache_check_clauses(struct tagcache_search *tcs,
923 struct tagcache_search_clause **clause, int count)
924 {
925 struct index_entry idx;
926
927 if (count == 0)
928 return true;
929
930 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
931 return false;
932
933 return check_clauses(tcs, &idx, clause, count);
934 }
935
936 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
937 {
938 int i;
939
940 /* If uniq buffer is not defined we must return true for search to work. */
941 if (tcs->unique_list == NULL
942 || (!tagcache_is_unique_tag(tcs->type)
943 && !tagcache_is_numeric_tag(tcs->type)))
944 {
945 return true;
946 }
947
948 for (i = 0; i < tcs->unique_list_count; i++)
949 {
950 /* Return false if entry is found. */
951 if (tcs->unique_list[i] == id)
952 return false;
953 }
954
955 if (tcs->unique_list_count < tcs->unique_list_capacity)
956 {
957 tcs->unique_list[i] = id;
958 tcs->unique_list_count++;
959 }
960
961 return true;
962 }
963
964 static bool build_lookup_list(struct tagcache_search *tcs)
965 {
966 struct index_entry entry;
967 int i;
968
969 tcs->seek_list_count = 0;
970
971 #ifdef HAVE_TC_RAMCACHE
972 if (tcs->ramsearch)
973 {
974 int j;
975
976 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
977 {
978 struct index_entry *idx = &hdr->indices[i];
979 if (tcs->seek_list_count == SEEK_LIST_SIZE)
980 break ;
981
982 /* Skip deleted files. */
983 if (idx->flag & FLAG_DELETED)
984 continue;
985
986 /* Go through all filters.. */
987 for (j = 0; j < tcs->filter_count; j++)
988 {
989 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
990 {
991 break ;
992 }
993 }
994
995 if (j < tcs->filter_count)
996 continue ;
997
998 /* Check for conditions. */
999 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1000 continue;
1001
1002 /* Add to the seek list if not already in uniq buffer. */
1003 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1004 continue;
1005
1006 /* Lets add it. */
1007 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1008 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1009 tcs->seek_list_count++;
1010 }
1011
1012 tcs->seek_pos = i;
1013
1014 return tcs->seek_list_count > 0;
1015 }
1016 #endif
1017
1018 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1019 sizeof(struct master_header), SEEK_SET);
1020
1021 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1022 == sizeof(struct index_entry))
1023 {
1024 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1025 break ;
1026
1027 tcs->seek_pos++;
1028
1029 /* Check if entry has been deleted. */
1030 if (entry.flag & FLAG_DELETED)
1031 continue;
1032
1033 /* Go through all filters.. */
1034 for (i = 0; i < tcs->filter_count; i++)
1035 {
1036 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1037 break ;
1038 }
1039
1040 if (i < tcs->filter_count)
1041 continue ;
1042
1043 /* Check for conditions. */
1044 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1045 continue;
1046
1047 /* Add to the seek list if not already in uniq buffer. */
1048 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1049 continue;
1050
1051 /* Lets add it. */
1052 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1053 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1054 tcs->seek_list_count++;
1055
1056 yield();
1057 }
1058
1059 return tcs->seek_list_count > 0;
1060 }
1061
1062
1063 static void remove_files(void)
1064 {
1065 int i;
1066 char buf[MAX_PATH];
1067
1068 tc_stat.ready = false;
1069 tc_stat.ramcache = false;
1070 tc_stat.econ = false;
1071 remove(TAGCACHE_FILE_MASTER);
1072 for (i = 0; i < TAG_COUNT; i++)
1073 {
1074 if (tagcache_is_numeric_tag(i))
1075 continue;
1076
1077 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1078 remove(buf);
1079 }
1080 }
1081
1082
1083 static int open_master_fd(struct master_header *hdr, bool write)
1084 {
1085 int fd;
1086 int rc;
1087
1088 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
1089 if (fd < 0)
1090 {
1091 logf("master file open failed for R/W");
1092 tc_stat.ready = false;
1093 return fd;
1094 }
1095
1096 tc_stat.econ = false;
1097
1098 /* Check the header. */
1099 rc = read(fd, hdr, sizeof(struct master_header));
1100 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
1101 {
1102 /* Success. */
1103 return fd;
1104 }
1105
1106 /* Trying to read again, this time with endianess correction enabled. */
1107 lseek(fd, 0, SEEK_SET);
1108
1109 rc = ecread(fd, hdr, 1, master_header_ec, true);
1110 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
1111 {
1112 logf("header error");
1113 tc_stat.ready = false;
1114 close(fd);
1115 return -2;
1116 }
1117
1118 tc_stat.econ = true;
1119
1120 return fd;
1121 }
1122
1123 static bool check_all_headers(void)
1124 {
1125 struct master_header myhdr;
1126 struct tagcache_header tch;
1127 int tag;
1128 int fd;
1129
1130 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1131 return false;
1132
1133 close(fd);
1134 if (myhdr.dirty)
1135 {
1136 logf("tagcache is dirty!");
1137 return false;
1138 }
1139
1140 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1141
1142 for (tag = 0; tag < TAG_COUNT; tag++)
1143 {
1144 if (tagcache_is_numeric_tag(tag))
1145 continue;
1146
1147 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1148 return false;
1149
1150 close(fd);
1151 }
1152
1153 return true;
1154 }
1155
1156 bool tagcache_search(struct tagcache_search *tcs, int tag)
1157 {
1158 struct tagcache_header tag_hdr;
1159 struct master_header master_hdr;
1160 int i;
1161
1162 if (tcs->initialized)
1163 tagcache_search_finish(tcs);
1164
1165 while (read_lock)
1166 sleep(1);
1167
1168 memset(tcs, 0, sizeof(struct tagcache_search));
1169 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1170 return false;
1171
1172 tcs->position = sizeof(struct tagcache_header);
1173 tcs->type = tag;
1174 tcs->seek_pos = 0;
1175 tcs->seek_list_count = 0;
1176 tcs->filter_count = 0;
1177 tcs->masterfd = -1;
1178
1179 for (i = 0; i < TAG_COUNT; i++)
1180 tcs->idxfd[i] = -1;
1181
1182 #ifndef HAVE_TC_RAMCACHE
1183 tcs->ramsearch = false;
1184 #else
1185 tcs->ramsearch = tc_stat.ramcache;
1186 if (tcs->ramsearch)
1187 {
1188 tcs->entry_count = hdr->entry_count[tcs->type];
1189 }
1190 else
1191 #endif
1192 {
1193 if (!tagcache_is_numeric_tag(tcs->type))
1194 {
1195 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1196 if (tcs->idxfd[tcs->type] < 0)
1197 return false;
1198 }
1199
1200 /* Always open as R/W so we can pass tcs to functions that modify data also
1201 * without failing. */
1202 tcs->masterfd = open_master_fd(&master_hdr, true);
1203
1204 if (tcs->masterfd < 0)
1205 return false;
1206 }
1207
1208 tcs->valid = true;
1209 tcs->initialized = true;
1210 write_lock++;
1211
1212 return true;
1213 }
1214
1215 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1216 void *buffer, long length)
1217 {
1218 tcs->unique_list = (unsigned long *)buffer;
1219 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1220 tcs->unique_list_count = 0;
1221 }
1222
1223 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1224 int tag, int seek)
1225 {
1226 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1227 return false;
1228
1229 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1230 return false;
1231
1232 tcs->filter_tag[tcs->filter_count] = tag;
1233 tcs->filter_seek[tcs->filter_count] = seek;
1234 tcs->filter_count++;
1235
1236 return true;
1237 }
1238
1239 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1240 struct tagcache_search_clause *clause)
1241 {
1242 int i;
1243
1244 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1245 {
1246 logf("Too many clauses");
1247 return false;
1248 }
1249
1250 /* Check if there is already a similar filter in present (filters are
1251 * much faster than clauses).
1252 */
1253 for (i = 0; i < tcs->filter_count; i++)
1254 {
1255 if (tcs->filter_tag[i] == clause->tag)
1256 return true;
1257 }
1258
1259 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1260 {
1261 char buf[MAX_PATH];
1262
1263 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1264 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1265 }
1266
1267 tcs->clause[tcs->clause_count] = clause;
1268 tcs->clause_count++;
1269
1270 return true;
1271 }
1272
1273 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1274 ? (flag & FLAG_DIRCACHE) : 1)
1275
1276 static bool get_next(struct tagcache_search *tcs)
1277 {
1278 static char buf[TAG_MAXLEN+32];
1279 struct tagfile_entry entry;
1280 long flag = 0;
1281
1282 if (!tcs->valid || !tc_stat.ready)
1283 return false;
1284
1285 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1286 #ifdef HAVE_TC_RAMCACHE
1287 && !tcs->ramsearch
1288 #endif
1289 )
1290 return false;
1291
1292 /* Relative fetch. */
1293 if (tcs->filter_count > 0 || tcs->clause_count > 0
1294 || tagcache_is_numeric_tag(tcs->type))
1295 {
1296 /* Check for end of list. */
1297 if (tcs->seek_list_count == 0)
1298 {
1299 /* Try to fetch more. */
1300 if (!build_lookup_list(tcs))
1301 {
1302 tcs->valid = false;
1303 return false;
1304 }
1305 }
1306
1307 tcs->seek_list_count--;
1308 flag = tcs->seek_flags[tcs->seek_list_count];
1309
1310 /* Seek stream to the correct position and continue to direct fetch. */
1311 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1312 && !tagcache_is_numeric_tag(tcs->type))
1313 {
1314 if (!open_files(tcs, tcs->type))
1315 return false;
1316
1317 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1318 }
1319 else
1320 tcs->position = tcs->seek_list[tcs->seek_list_count];
1321 }
1322
1323 if (tagcache_is_numeric_tag(tcs->type))
1324 {
1325 snprintf(buf, sizeof(buf), "%d", tcs->position);
1326 tcs->result_seek = tcs->position;
1327 tcs->result = buf;
1328 tcs->result_len = strlen(buf) + 1;
1329 return true;
1330 }
1331
1332 /* Direct fetch. */
1333 #ifdef HAVE_TC_RAMCACHE
1334 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1335 {
1336 struct tagfile_entry *ep;
1337
1338 if (tcs->entry_count == 0)
1339 {
1340 tcs->valid = false;
1341 return false;
1342 }
1343 tcs->entry_count--;
1344
1345 tcs->result_seek = tcs->position;
1346
1347 # ifdef HAVE_DIRCACHE
1348 if (tcs->type == tag_filename)
1349 {
1350 dircache_copy_path((struct dirent *)tcs->position,
1351 buf, sizeof buf);
1352 tcs->result = buf;
1353 tcs->result_len = strlen(buf) + 1;
1354 tcs->idx_id = FLAG_GET_ATTR(flag);
1355 tcs->ramresult = false;
1356
1357 return true;
1358 }
1359 # endif
1360
1361 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1362 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1363 tcs->result = ep->tag_data;
1364 tcs->result_len = strlen(tcs->result) + 1;
1365 tcs->idx_id = ep->idx_id;
1366 tcs->ramresult = true;
1367
1368 return true;
1369 }
1370 else
1371 #endif
1372 {
1373 if (!open_files(tcs, tcs->type))
1374 return false;
1375
1376 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1377 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1378 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1379 {
1380 /* End of data. */
1381 tcs->valid = false;
1382 return false;
1383 }
1384 }
1385
1386 if (entry.tag_length > (long)sizeof(buf))
1387 {
1388 tcs->valid = false;
1389 logf("too long tag #2");
1390 return false;
1391 }
1392
1393 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1394 {
1395 tcs->valid = false;
1396 logf("read error #4");
1397 return false;
1398 }
1399
1400 tcs->result = buf;
1401 tcs->result_len = strlen(tcs->result) + 1;
1402 tcs->idx_id = entry.idx_id;
1403 tcs->ramresult = false;
1404
1405 return true;
1406 }
1407
1408 bool tagcache_get_next(struct tagcache_search *tcs)
1409 {
1410 while (get_next(tcs))
1411 {
1412 if (tcs->result_len > 1)
1413 return true;
1414 }
1415
1416 return false;
1417 }
1418
1419 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1420 int tag, char *buf, long size)
1421 {
1422 struct index_entry idx;
1423
1424 *buf = '\0';
1425 if (!get_index(tcs->masterfd, idxid, &idx, true))
1426 return false;
1427
1428 return retrieve(tcs, &idx, tag, buf, size);
1429 }
1430
1431 static bool update_master_header(void)
1432 {
1433 struct master_header myhdr;
1434 int fd;
1435
1436 if (!tc_stat.ready)
1437 return false;
1438
1439 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1440 return false;
1441
1442 myhdr.serial = current_tcmh.serial;
1443 myhdr.commitid = current_tcmh.commitid;
1444
1445 /* Write it back */
1446 lseek(fd, 0, SEEK_SET);
1447 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1448 close(fd);
1449
1450 #ifdef HAVE_TC_RAMCACHE
1451 if (hdr)
1452 {
1453 hdr->h.serial = current_tcmh.serial;
1454 hdr->h.commitid = current_tcmh.commitid;
1455 }
1456 #endif
1457
1458 return true;
1459 }
1460
1461 #if 0
1462
1463 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1464 {
1465 struct tagentry *entry;
1466
1467 if (tcs->type != tag_title)
1468 return ;
1469
1470 /* We will need reserve buffer for this. */
1471 if (tcs->ramcache)
1472 {
1473 struct tagfile_entry *ep;
1474
1475 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1476 tcs->seek_list[tcs->seek_list_count];
1477 }
1478
1479 entry = find_entry_ram();
1480
1481 }
1482 #endif
1483
1484 void tagcache_search_finish(struct tagcache_search *tcs)
1485 {
1486 int i;
1487
1488 if (!tcs->initialized)
1489 return;
1490
1491 if (tcs->masterfd >= 0)
1492 {
1493 close(tcs->masterfd);
1494 tcs->masterfd = -1;
1495 }
1496
1497 for (i = 0; i < TAG_COUNT; i++)
1498 {
1499 if (tcs->idxfd[i] >= 0)
1500 {
1501 close(tcs->idxfd[i]);
1502 tcs->idxfd[i] = -1;
1503 }
1504 }
1505
1506 tcs->ramsearch = false;
1507 tcs->valid = false;
1508 tcs->initialized = 0;
1509 if (write_lock > 0)
1510 write_lock--;
1511 }
1512
1513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1514 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1515 {
1516 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1517 }
1518
1519 static long get_tag_numeric(const struct index_entry *entry, int tag)
1520 {
1521 return entry->tag_seek[tag];
1522 }
1523
1524 static char* get_tag_string(const struct index_entry *entry, int tag)
1525 {
1526 char* s = get_tag(entry, tag)->tag_data;
1527 return strcmp(s, UNTAGGED) ? s : NULL;
1528 }
1529
1530 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1531 {
1532 struct index_entry *entry;
1533 int idx_id;
1534
1535 if (!tc_stat.ready)
1536 return false;
1537
1538 /* Find the corresponding entry in tagcache. */
1539 idx_id = find_entry_ram(filename, NULL);
1540 if (idx_id < 0 || !tc_stat.ramcache)
1541 return false;
1542
1543 entry = &hdr->indices[idx_id];
1544
1545 id3->title = get_tag_string(entry, tag_title);
1546 id3->artist = get_tag_string(entry, tag_artist);
1547 id3->album = get_tag_string(entry, tag_album);
1548 id3->genre_string = get_tag_string(entry, tag_genre);
1549 id3->composer = get_tag_string(entry, tag_composer);
1550 id3->comment = get_tag_string(entry, tag_comment);
1551 id3->albumartist = get_tag_string(entry, tag_albumartist);
1552 id3->grouping = get_tag_string(entry, tag_grouping);
1553
1554 id3->playcount = get_tag_numeric(entry, tag_playcount);
1555 id3->rating = get_tag_numeric(entry, tag_rating);
1556 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1557 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1558 id3->year = get_tag_numeric(entry, tag_year);
1559
1560 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1561 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1562 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1563 if (id3->bitrate == 0)
1564 id3->bitrate = 1;
1565
1566 return true;
1567 }
1568 #endif
1569
1570 static inline void write_item(const char *item)
1571 {
1572 int len = strlen(item) + 1;
1573
1574 data_size += len;
1575 write(cachefd, item, len);
1576 }
1577
1578 static int check_if_empty(char **tag)
1579 {
1580 int length;
1581
1582 if (*tag == NULL || *tag[0] == '\0')
1583 {
1584 *tag = UNTAGGED;
1585 return sizeof(UNTAGGED); /* Tag length */
1586 }
1587
1588 length = strlen(*tag);
1589 if (length > TAG_MAXLEN)
1590 {
1591 logf("over length tag: %s", *tag);
1592 length = TAG_MAXLEN;
1593 *tag[length] = '\0';
1594 }
1595
1596 return length + 1;
1597 }
1598
1599 #define ADD_TAG(entry,tag,data) \
1600 /* Adding tag */ \
1601 entry.tag_offset[tag] = offset; \
1602 entry.tag_length[tag] = check_if_empty(data); \
1603 offset += entry.tag_length[tag]
1604
1605 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1606 static void add_tagcache(char *path, const struct dirent *dc)
1607 #else
1608 static void add_tagcache(char *path)
1609 #endif
1610 {
1611 struct track_info track;
1612 struct temp_file_entry entry;
1613 bool ret;
1614 int fd;
1615 char tracknumfix[3];
1616 int offset = 0;
1617 int path_length = strlen(path);
1618 bool has_albumartist;
1619 bool has_grouping;
1620
1621 if (cachefd < 0)
1622 return ;
1623
1624 /* Check for overlength file path. */
1625 if (path_length > TAG_MAXLEN)
1626 {
1627 /* Path can't be shortened. */
1628 logf("Too long path: %s", path);
1629 return ;
1630 }
1631
1632 /* Check if the file is supported. */
1633 if (probe_file_format(path) == AFMT_UNKNOWN)
1634 return ;
1635
1636 /* Check if the file is already cached. */
1637 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1638 if (tc_stat.ramcache && dircache_is_enabled())
1639 {
1640 if (find_entry_ram(path, dc) >= 0)
1641 return ;
1642 }
1643 else
1644 #endif
1645 {
1646 if (filenametag_fd >= 0)
1647 {
1648 if (find_entry_disk(path) >= 0)
1649 return ;
1650 }
1651 }
1652
1653 fd = open(path, O_RDONLY);
1654 if (fd < 0)
1655 {
1656 logf("open fail: %s", path);
1657 return ;
1658 }
1659
1660 memset(&track, 0, sizeof(struct track_info));
1661 memset(&entry, 0, sizeof(struct temp_file_entry));
1662 memset(&tracknumfix, 0, sizeof(tracknumfix));
1663 ret = get_metadata(&(track.id3), fd, path, false);
1664 close(fd);
1665
1666 if (!ret)
1667 return ;
1668
1669 logf("-> %s", path);
1670
1671 /* Generate track number if missing. */
1672 if (track.id3.tracknum <= 0)
1673 {
1674 const char *p = strrchr(path, '.');
1675
1676 if (p == NULL)
1677 p = &path[strlen(path)-1];
1678
1679 while (*p != '/')
1680 {
1681 if (isdigit(*p) && isdigit(*(p-1)))
1682 {
1683 tracknumfix[1] = *p--;
1684 tracknumfix[0] = *p;
1685 break;
1686 }
1687 p--;
1688 }
1689
1690 if (tracknumfix[0] != '\0')
1691 {
1692 track.id3.tracknum = atoi(tracknumfix);
1693 /* Set a flag to indicate track number has been generated. */
1694 entry.flag |= FLAG_TRKNUMGEN;
1695 }
1696 else
1697 {
1698 /* Unable to generate track number. */
1699 track.id3.tracknum = -1;
1700 }
1701 }
1702
1703 /* Numeric tags */
1704 entry.tag_offset[tag_year] = track.id3.year;
1705 entry.tag_offset[tag_discnumber] = track.id3.discnum;
1706 entry.tag_offset[tag_tracknumber] = track.id3.tracknum;
1707 entry.tag_offset[tag_length] = track.id3.length;
1708 entry.tag_offset[tag_bitrate] = track.id3.bitrate;
1709
1710 /* String tags. */
1711 has_albumartist = track.id3.albumartist != NULL
1712 && strlen(track.id3.albumartist) > 0;
1713 has_grouping = track.id3.grouping != NULL
1714 && strlen(track.id3.grouping) > 0;
1715
1716 ADD_TAG(entry, tag_filename, &path);
1717 ADD_TAG(entry, tag_title, &track.id3.title);
1718 ADD_TAG(entry, tag_artist, &track.id3.artist);
1719 ADD_TAG(entry, tag_album, &track.id3.album);
1720 ADD_TAG(entry, tag_genre, &track.id3.genre_string);
1721 ADD_TAG(entry, tag_composer, &track.id3.composer);
1722 ADD_TAG(entry, tag_comment, &track.id3.comment);
1723 if (has_albumartist)
1724 {
1725 ADD_TAG(entry, tag_albumartist, &track.id3.albumartist);
1726 }
1727 else
1728 {
1729 ADD_TAG(entry, tag_albumartist, &track.id3.artist);
1730 }
1731 if (has_grouping)
1732 {
1733 ADD_TAG(entry, tag_grouping, &track.id3.grouping);
1734 }
1735 else
1736 {
1737 ADD_TAG(entry, tag_grouping, &track.id3.title);
1738 }
1739 entry.data_length = offset;
1740
1741 /* Write the header */
1742 write(cachefd, &entry, sizeof(struct temp_file_entry));
1743
1744 /* And tags also... Correct order is critical */
1745 write_item(path);
1746 write_item(track.id3.title);
1747 write_item(track.id3.artist);
1748 write_item(track.id3.album);
1749 write_item(track.id3.genre_string);
1750 write_item(track.id3.composer);
1751 write_item(track.id3.comment);
1752 if (has_albumartist)
1753 {
1754 write_item(track.id3.albumartist);
1755 }
1756 else
1757 {
1758 write_item(track.id3.artist);
1759 }
1760 if (has_grouping)
1761 {
1762 write_item(track.id3.grouping);
1763 }
1764 else
1765 {
1766 write_item(track.id3.title);
1767 }
1768 total_entry_count++;
1769 }
1770
1771 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1772 {
1773 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1774 int len = strlen(str)+1;
1775 int i;
1776 unsigned crc32;
1777 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1778 char buf[TAG_MAXLEN+32];
1779
1780 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1781 buf[i] = tolower(str[i]);
1782 buf[i] = '\0';
1783
1784 crc32 = crc_32(buf, i, 0xffffffff);
1785
1786 if (unique)
1787 {
1788 /* Check if the crc does not exist -> entry does not exist for sure. */
1789 for (i = 0; i < tempbufidx; i++)
1790 {
1791 if (crcbuf[-i] != crc32)
1792 continue;
1793
1794 if (!strcasecmp(str, index[i].str))
1795 {
1796 if (id < 0 || id >= lookup_buffer_depth)
1797 {
1798 logf("lookup buf overf.: %d", id);
1799 return false;
1800 }
1801
1802 lookup[id] = &index[i];
1803 return true;
1804 }
1805 }
1806 }
1807
1808 /* Insert to CRC buffer. */
1809 crcbuf[-tempbufidx] = crc32;
1810 tempbuf_left -= 4;
1811
1812 /* Insert it to the buffer. */
1813 tempbuf_left -= len;
1814 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1815 return false;
1816
1817 if (id >= lookup_buffer_depth)
1818 {
1819 logf("lookup buf overf. #2: %d", id);
1820 return false;
1821 }
1822
1823 if (id >= 0)
1824 {
1825 lookup[id] = &index[tempbufidx];
1826 index[tempbufidx].idlist.id = id;
1827 }
1828 else
1829 index[tempbufidx].idlist.id = -1;
1830
1831 index[tempbufidx].idlist.next = NULL;
1832 index[tempbufidx].idx_id = idx_id;
1833 index[tempbufidx].seek = -1;
1834 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1835 memcpy(index[tempbufidx].str, str, len);
1836 tempbuf_pos += len;
1837 tempbufidx++;
1838
1839 return true;
1840 }
1841
1842 static int compare(const void *p1, const void *p2)
1843 {
1844 do_timed_yield();
1845
1846 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1847 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1848
1849 if (strcmp(e1->str, UNTAGGED) == 0)
1850 {
1851 if (strcmp(e2->str, UNTAGGED) == 0)
1852 return 0;
1853 return -1;
1854 }
1855 else if (strcmp(e2->str, UNTAGGED) == 0)
1856 return 1;
1857
1858 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1859 }
1860
1861 static int tempbuf_sort(int fd)
1862 {
1863 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1864 struct tagfile_entry fe;
1865 int i;
1866 int length;
1867
1868 /* Generate reverse lookup entries. */
1869 for (i = 0; i < lookup_buffer_depth; i++)
1870 {
1871 struct tempbuf_id_list *idlist;
1872
1873 if (!lookup[i])
1874 continue;
1875
1876 if (lookup[i]->idlist.id == i)
1877 continue;
1878
1879 idlist = &lookup[i]->idlist;
1880 while (idlist->next != NULL)
1881 idlist = idlist->next;
1882
1883 tempbuf_left -= sizeof(struct tempbuf_id_list);
1884 if (tempbuf_left - 4 < 0)
1885 return -1;
1886
1887 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1888 if (tempbuf_pos & 0x03)
1889 {
1890 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1891 tempbuf_left -= 3;
1892 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1893 }
1894 tempbuf_pos += sizeof(struct tempbuf_id_list);
1895
1896 idlist = idlist->next;
1897 idlist->id = i;
1898 idlist->next = NULL;
1899
1900 do_timed_yield();
1901 }
1902
1903 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1904 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1905
1906 for (i = 0; i < tempbufidx; i++)
1907 {
1908 struct tempbuf_id_list *idlist = &index[i].idlist;
1909
1910 /* Fix the lookup list. */
1911 while (idlist != NULL)
1912 {
1913 if (idlist->id >= 0)
1914 lookup[idlist->id] = &index[i];
1915 idlist = idlist->next;
1916 }
1917
1918 index[i].seek = lseek(fd, 0, SEEK_CUR);
1919 length = strlen(index[i].str) + 1;
1920 fe.tag_length = length;
1921 fe.idx_id = index[i].idx_id;
1922
1923 /* Check the chunk alignment. */
1924 if ((fe.tag_length + sizeof(struct tagfile_entry))
1925 % TAGFILE_ENTRY_CHUNK_LENGTH)
1926 {
1927 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1928 ((fe.tag_length + sizeof(struct tagfile_entry))
1929 % TAGFILE_ENTRY_CHUNK_LENGTH);
1930 }
1931
1932 #ifdef TAGCACHE_STRICT_ALIGN
1933 /* Make sure the entry is long aligned. */
1934 if (index[i].seek & 0x03)
1935 {
1936 logf("tempbuf_sort: alignment error!");
1937 return -3;
1938 }
1939 #endif
1940
1941 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
1942 sizeof(struct tagfile_entry))
1943 {
1944 logf("tempbuf_sort: write error #1");
1945 return -1;
1946 }
1947
1948 if (write(fd, index[i].str, length) != length)
1949 {
1950 logf("tempbuf_sort: write error #2");
1951 return -2;
1952 }
1953
1954 /* Write some padding. */
1955 if (fe.tag_length - length > 0)
1956 write(fd, "XXXXXXXX", fe.tag_length - length);
1957 }
1958
1959 return i;
1960 }
1961
1962 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
1963 {
1964 if (id < 0 || id >= lookup_buffer_depth)
1965 return NULL;
1966
1967 return lookup[id];
1968 }
1969
1970
1971 inline static int tempbuf_find_location(int id)
1972 {
1973 struct tempbuf_searchidx *entry;
1974
1975 entry = tempbuf_locate(id);
1976 if (entry == NULL)
1977 return -1;
1978
1979 return entry->seek;
1980 }
1981
1982 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
1983 {
1984 struct master_header tcmh;
1985 struct index_entry idx;
1986 int masterfd;
1987 int masterfd_pos;
1988 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
1989 int max_entries;
1990 int entries_processed = 0;
1991 int i, j;
1992
1993 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
1994
1995 logf("Building numeric indices...");
1996 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
1997
1998 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
1999 return false;
2000
2001 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2002 SEEK_CUR);
2003 if (masterfd_pos == filesize(masterfd))
2004 {
2005 logf("we can't append!");
2006 close(masterfd);
2007 return false;
2008 }
2009
2010 while (entries_processed < h->entry_count)
2011 {
2012 int count = MIN(h->entry_count - entries_processed, max_entries);
2013
2014 /* Read in as many entries as possible. */
2015 for (i = 0; i < count; i++)
2016 {
2017 /* Read in numeric data. */
2018 if (read(tmpfd, &entrybuf[i], sizeof(struct temp_file_entry)) !=
2019 sizeof(struct temp_file_entry))
2020 {
2021 logf("read fail #1");
2022 close(masterfd);
2023 return false;
2024 }
2025
2026 /* Skip string data. */
2027 lseek(tmpfd, entrybuf[i].data_length, SEEK_CUR);
2028 }
2029
2030 /* Commit the data to the index. */
2031 for (i = 0; i < count; i++)
2032 {
2033 int loc = lseek(masterfd, 0, SEEK_CUR);
2034
2035 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2036 != sizeof(struct index_entry))
2037 {
2038 logf("read fail #2");
2039 close(masterfd);
2040 return false;
2041 }
2042
2043 for (j = 0; j < TAG_COUNT; j++)
2044 {
2045 if (!tagcache_is_numeric_tag(j))
2046 continue;
2047
2048 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2049 }
2050 idx.flag = entrybuf[i].flag;
2051
2052 if (tc_stat.ready && current_tcmh.commitid > 0)
2053 {
2054 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2055 idx.flag |= FLAG_DIRTYNUM;
2056 }
2057
2058 /* Write back the updated index. */
2059 lseek(masterfd, loc, SEEK_SET);
2060 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2061 != sizeof(struct index_entry))
2062 {
2063 logf("write fail");
2064 close(masterfd);
2065 return false;
2066 }
2067 }
2068
2069 entries_processed += count;
2070 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2071 }
2072
2073 close(masterfd);
2074
2075 return true;
2076 }
2077
2078 /**
2079 * Return values:
2080 * > 0 success
2081 * == 0 temporary failure
2082 * < 0 fatal error
2083 */
2084 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2085 {
2086 int i;
2087 struct tagcache_header tch;
2088 struct master_header tcmh;
2089 struct index_entry idxbuf[IDX_BUF_DEPTH];
2090 int idxbuf_pos;
2091 char buf[TAG_MAXLEN+32];
2092 int fd = -1, masterfd;
2093 bool error = false;
2094 int init;
2095 int masterfd_pos;
2096
2097 logf("Building index: %d", index_type);
2098
2099 /* Check the number of entries we need to allocate ram for. */
2100 commit_entry_count = h->entry_count + 1;
2101
2102 masterfd = open_master_fd(&tcmh, false);
2103 if (masterfd >= 0)
2104 {
2105 commit_entry_count += tcmh.tch.entry_count;
2106 close(masterfd);
2107 }
2108 else
2109 remove_files(); /* Just to be sure we are clean. */
2110
2111 /* Open the index file, which contains the tag names. */
2112 fd = open_tag_fd(&tch, index_type, true);
2113 if (fd >= 0)
2114 {
2115 logf("tch.datasize=%ld", tch.datasize);
2116 lookup_buffer_depth = 1 +
2117 /* First part */ commit_entry_count +
2118 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2119 }
2120 else
2121 {
2122 lookup_buffer_depth = 1 +
2123 /* First part */ commit_entry_count +
2124 /* Second part */ 0;
2125 }
2126
2127 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2128 logf("commit_entry_count=%ld", commit_entry_count);
2129
2130 /* Allocate buffer for all index entries from both old and new
2131 * tag files. */
2132 tempbufidx = 0;
2133 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2134
2135 /* Allocate lookup buffer. The first portion of commit_entry_count
2136 * contains the new tags in the temporary file and the second
2137 * part for locating entries already in the db.
2138 *
2139 * New tags Old tags
2140 * +---------+---------------------------+
2141 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2142 * +---------+---------------------------+
2143 *
2144 * Old tags are inserted to a temporary buffer with position:
2145 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2146 * And new tags with index:
2147 * tempbuf_insert(idx, ...);
2148 *
2149 * The buffer is sorted and written into tag file:
2150 * tempbuf_sort(...);
2151 * leaving master index locations messed up.
2152 *
2153 * That is fixed using the lookup buffer for old tags:
2154 * new_seek = tempbuf_find_location(old_seek, ...);
2155 * and for new tags:
2156 * new_seek = tempbuf_find_location(idx);
2157 */
2158 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2159 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2160 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2161
2162 /* And calculate the remaining data space used mainly for storing
2163 * tag data (strings). */
2164 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2165 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2166 {
2167 logf("Buffer way too small!");
2168 return 0;
2169 }
2170
2171 if (fd >= 0)
2172 {
2173 /**
2174 * If tag file contains unique tags (sorted index), we will load
2175 * it entirely into memory so we can resort it later for use with
2176 * chunked browsing.
2177 */
2178 if (tagcache_is_sorted_tag(index_type))
2179 {
2180 logf("loading tags...");
2181 for (i = 0; i < tch.entry_count; i++)
2182 {
2183 struct tagfile_entry entry;
2184 int loc = lseek(fd, 0, SEEK_CUR);
2185 bool ret;
2186
2187 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2188 != sizeof(struct tagfile_entry))
2189 {
2190 logf("read error #7");
2191 close(fd);
2192 return -2;
2193 }
2194
2195 if (entry.tag_length >= (int)sizeof(buf))
2196 {
2197 logf("too long tag #3");
2198 close(fd);
2199 return -2;
2200 }
2201
2202 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2203 {
2204 logf("read error #8");
2205 close(fd);
2206 return -2;
2207 }
2208
2209 /* Skip deleted entries. */
2210 if (buf[0] == '\0')
2211 continue;
2212
2213 /**
2214 * Save the tag and tag id in the memory buffer. Tag id
2215 * is saved so we can later reindex the master lookup
2216 * table when the index gets resorted.
2217 */
2218 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2219 + commit_entry_count, entry.idx_id,
2220 tagcache_is_unique_tag(index_type));
2221 if (!ret)
2222 {
2223 close(fd);
2224 return -3;
2225 }
2226 do_timed_yield();
2227 }
2228 logf("done");
2229 }
2230 else
2231 tempbufidx = tch.entry_count;
2232 }
2233 else
2234 {
2235 /**
2236 * Creating new index file to store the tags. No need to preload
2237 * anything whether the index type is sorted or not.
2238 */
2239 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2240 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2241 if (fd < 0)
2242 {
2243 logf("%s open fail", buf);
2244 return -2;
2245 }
2246
2247 tch.magic = TAGCACHE_MAGIC;
2248 tch.entry_count = 0;
2249 tch.datasize = 0;
2250
2251 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2252 != sizeof(struct tagcache_header))
2253 {
2254 logf("header write failed");
2255 close(fd);
2256 return -2;
2257 }
2258 }
2259
2260 /* Loading the tag lookup file as "master file". */
2261 logf("Loading index file");
2262 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2263
2264 if (masterfd < 0)
2265 {
2266 logf("Creating new DB");
2267 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2268
2269 if (masterfd < 0)
2270 {
2271 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2272 close(fd);
2273 return -2;
2274 }
2275
2276 /* Write the header (write real values later). */
2277 memset(&tcmh, 0, sizeof(struct master_header));
2278 tcmh.tch = *h;
2279 tcmh.tch.entry_count = 0;
2280 tcmh.tch.datasize = 0;
2281 tcmh.dirty = true;
2282 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2283 init = true;
2284 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2285 }
2286 else
2287 {
2288 /**
2289 * Master file already exists so we need to process the current
2290 * file first.
2291 */
2292 init = false;
2293
2294 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2295 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2296 {
2297 logf("header error");
2298 close(fd);
2299 close(masterfd);
2300 return -2;
2301 }
2302
2303 /**
2304 * If we reach end of the master file, we need to expand it to
2305 * hold new tags. If the current index is not sorted, we can
2306 * simply append new data to end of the file.
2307 * However, if the index is sorted, we need to update all tag
2308 * pointers in the master file for the current index.
2309 */
2310 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2311 SEEK_CUR);
2312 if (masterfd_pos == filesize(masterfd))
2313 {
2314 logf("appending...");
2315 init = true;
2316 }
2317 }
2318
2319 /**
2320 * Load new unique tags in memory to be sorted later and added
2321 * to the master lookup file.
2322 */
2323 if (tagcache_is_sorted_tag(index_type))
2324 {
2325 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2326 /* h is the header of the temporary file containing new tags. */
2327 logf("inserting new tags...");
2328 for (i = 0; i < h->entry_count; i++)
2329 {
2330 struct temp_file_entry entry;
2331
2332 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2333 sizeof(struct temp_file_entry))
2334 {
2335 logf("read fail #3");
2336 error = true;
2337 goto error_exit;
2338 }
2339
2340 /* Read data. */
2341 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2342 {
2343 logf("too long entry!");
2344 error = true;
2345 goto error_exit;
2346 }
2347
2348 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2349 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2350 entry.tag_length[index_type])
2351 {
2352 logf("read fail #4");
2353 error = true;
2354 goto error_exit;
2355 }
2356
2357 if (tagcache_is_unique_tag(index_type))
2358 error = !tempbuf_insert(buf, i, -1, true);
2359 else
2360 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2361
2362 if (error)
2363 {
2364 logf("insert error");
2365 goto error_exit;
2366 }
2367
2368 /* Skip to next. */
2369 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2370 entry.tag_length[index_type], SEEK_CUR);
2371 do_timed_yield();
2372 }
2373 logf("done");
2374
2375 /* Sort the buffer data and write it to the index file. */
2376 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2377 i = tempbuf_sort(fd);
2378 if (i < 0)
2379 goto error_exit;
2380 logf("sorted %d tags", i);
2381
2382 /**
2383 * Now update all indexes in the master lookup file.
2384 */
2385 logf("updating indices...");
2386 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2387 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2388 {
2389 int j;
2390 int loc = lseek(masterfd, 0, SEEK_CUR);
2391
2392 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2393
2394 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2395 != (int)sizeof(struct index_entry)*idxbuf_pos)
2396 {
2397 logf("read fail #5");
2398 error = true;
2399 goto error_exit ;
2400 }
2401 lseek(masterfd, loc, SEEK_SET);
2402
2403 for (j = 0; j < idxbuf_pos; j++)
2404 {
2405 if (idxbuf[j].flag & FLAG_DELETED)
2406 {
2407 /* We can just ignore deleted entries. */
2408 idxbuf[j].tag_seek[index_type] = 0;
2409 continue;
2410 }
2411
2412 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2413 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2414 + commit_entry_count);
2415
2416 if (idxbuf[j].tag_seek[index_type] < 0)
2417 {
2418 logf("update error: %d/%ld", i+j, tcmh.tch.entry_count);
2419 error = true;
2420 goto error_exit;
2421 }
2422
2423 do_timed_yield();
2424 }
2425
2426 /* Write back the updated index. */
2427 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2428 index_entry_ec, tc_stat.econ) !=
2429 (int)sizeof(struct index_entry)*idxbuf_pos)
2430 {
2431 logf("write fail");
2432 error = true;
2433 goto error_exit;
2434 }
2435 }
2436 logf("done");
2437 }
2438
2439 /**
2440 * Walk through the temporary file containing the new tags.
2441 */
2442 // build_normal_index(h, tmpfd, masterfd, idx);
2443 logf("updating new indices...");
2444 lseek(masterfd, masterfd_pos, SEEK_SET);
2445 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2446 lseek(fd, 0, SEEK_END);
2447 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2448 {
2449 int j;
2450
2451 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2452 if (init)
2453 {
2454 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2455 }
2456 else
2457 {
2458 int loc = lseek(masterfd, 0, SEEK_CUR);
2459
2460 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2461 != (int)sizeof(struct index_entry)*idxbuf_pos)
2462 {
2463 logf("read fail #6");
2464 error = true;
2465 break ;
2466 }
2467 lseek(masterfd, loc, SEEK_SET);
2468 }
2469
2470 /* Read entry headers. */
2471 for (j = 0; j < idxbuf_pos; j++)
2472 {
2473 if (!tagcache_is_sorted_tag(index_type))
2474 {
2475 struct temp_file_entry entry;
2476 struct tagfile_entry fe;
2477
2478 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2479 sizeof(struct temp_file_entry))
2480 {
2481 logf("read fail #7");
2482 error = true;
2483 break ;
2484 }
2485
2486 /* Read data. */
2487 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2488 {
2489 logf("too long entry!");
2490 logf("length=%d", entry.tag_length[index_type]);
2491 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2492 error = true;
2493 break ;
2494 }
2495
2496 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2497 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2498 entry.tag_length[index_type])
2499 {
2500 logf("read fail #8");
2501 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2502 logf("length=0x%02x", entry.tag_length[index_type]);
2503 error = true;
2504 break ;
2505 }
2506
2507 /* Write to index file. */
2508 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2509 fe.tag_length = entry.tag_length[index_type];
2510 fe.idx_id = tcmh.tch.entry_count + i + j;
2511 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2512 write(fd, buf, fe.tag_length);
2513 tempbufidx++;
2514
2515 /* Skip to next. */
2516 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2517 entry.tag_length[index_type], SEEK_CUR);
2518 }
2519 else
2520 {
2521 /* Locate the correct entry from the sorted array. */
2522 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2523 if (idxbuf[j].tag_seek[index_type] < 0)
2524 {
2525 logf("entry not found (%d)", j);
2526 error = true;
2527 break ;
2528 }
2529 }
2530 }
2531
2532 /* Write index. */
2533 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2534 index_entry_ec, tc_stat.econ) !=
2535 (int)sizeof(struct index_entry)*idxbuf_pos)
2536 {
2537 logf("tagcache: write fail #4");
2538 error = true;
2539 break ;
2540 }
2541
2542 do_timed_yield();
2543 }
2544 logf("done");
2545
2546 /* Finally write the header. */
2547 tch.magic = TAGCACHE_MAGIC;
2548 tch.entry_count = tempbufidx;
2549 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2550 lseek(fd, 0, SEEK_SET);
2551 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2552
2553 if (index_type != tag_filename)
2554 h->datasize += tch.datasize;
2555 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2556 error_exit:
2557
2558 close(fd);
2559 close(masterfd);
2560
2561 if (error)
2562 return -2;
2563
2564 return 1;
2565 }
2566
2567 static bool commit(void)
2568 {
2569 struct tagcache_header tch;
2570 struct master_header tcmh;
2571 int i, len, rc;
2572 int tmpfd;
2573 int masterfd;
2574 #ifdef HAVE_DIRCACHE
2575 bool dircache_buffer_stolen = false;
2576 #endif
2577 bool local_allocation = false;
2578
2579 logf("committing tagcache");
2580
2581 while (write_lock)
2582 sleep(1);
2583
2584 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2585 if (tmpfd < 0)
2586 {
2587 logf("nothing to commit");
2588 return true;
2589 }
2590
2591
2592 /* Load the header. */
2593 len = sizeof(struct tagcache_header);
2594 rc = read(tmpfd, &tch, len);
2595
2596 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2597 {
2598 logf("incorrect header");
2599 close(tmpfd);
2600 remove(TAGCACHE_FILE_TEMP);
2601 return false;
2602 }
2603
2604 if (tch.entry_count == 0)
2605 {
2606 logf("nothing to commit");
2607 close(tmpfd);
2608 remove(TAGCACHE_FILE_TEMP);
2609 return true;
2610 }
2611
2612 #ifdef HAVE_EEPROM_SETTINGS
2613 remove(TAGCACHE_STATEFILE);
2614 #endif
2615
2616 /* At first be sure to unload the ramcache! */
2617 #ifdef HAVE_TC_RAMCACHE
2618 tc_stat.ramcache = false;
2619 #endif
2620
2621 read_lock++;
2622
2623 /* Try to steal every buffer we can :) */
2624 if (tempbuf_size == 0)
2625 local_allocation = true;
2626
2627 #ifdef HAVE_DIRCACHE
2628 if (tempbuf_size == 0)
2629 {
2630 /* Try to steal the dircache buffer. */
2631 tempbuf = dircache_steal_buffer(&tempbuf_size);
2632 tempbuf_size &= ~0x03;
2633
2634 if (tempbuf_size > 0)
2635 {
2636 dircache_buffer_stolen = true;
2637 }
2638 }
2639 #endif
2640
2641 #ifdef HAVE_TC_RAMCACHE
2642 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2643 {
2644 tempbuf = (char *)(hdr + 1);
2645 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2646 tempbuf_size &= ~0x03;
2647 }
2648 #endif
2649
2650 /* And finally fail if there are no buffers available. */
2651 if (tempbuf_size == 0)
2652 {
2653 logf("delaying commit until next boot");
2654 tc_stat.commit_delayed = true;
2655 close(tmpfd);
2656 read_lock--;
2657 return false;
2658 }
2659
2660 logf("commit %ld entries...", tch.entry_count);
2661
2662 /* Mark DB dirty so it will stay disabled if commit fails. */
2663 current_tcmh.dirty = true;
2664 update_master_header();
2665
2666 /* Now create the index files. */
2667 tc_stat.commit_step = 0;
2668 tch.datasize = 0;
2669 tc_stat.commit_delayed = false;
2670
2671 for (i = 0; i < TAG_COUNT; i++)
2672 {
2673 int ret;
2674
2675 if (tagcache_is_numeric_tag(i))
2676 continue;
2677
2678 tc_stat.commit_step++;
2679 ret = build_index(i, &tch, tmpfd);
2680 if (ret <= 0)
2681 {
2682 close(tmpfd);
2683 logf("tagcache failed init");
2684 if (ret < 0)
2685 remove_files();
2686 else
2687 tc_stat.commit_delayed = true;
2688 tc_stat.commit_step = 0;
2689 read_lock--;
2690 return false;
2691 }
2692 }
2693
2694 if (!build_numeric_indices(&tch, tmpfd))
2695 {
2696 logf("Failure to commit numeric indices");
2697 close(tmpfd);
2698 remove_files();
2699 tc_stat.commit_step = 0;
2700 read_lock--;
2701 return false;
2702 }
2703
2704 close(tmpfd);
2705 tc_stat.commit_step = 0;
2706
2707 /* Update the master index headers. */
2708 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2709 {
2710 read_lock--;
2711 return false;
2712 }
2713
2714 tcmh.tch.entry_count += tch.entry_count;
2715 tcmh.tch.datasize = sizeof(struct master_header)
2716 + sizeof(struct index_entry) * tcmh.tch.entry_count
2717 + tch.datasize;
2718 tcmh.dirty = false;
2719 tcmh.commitid++;
2720
2721 lseek(masterfd, 0, SEEK_SET);
2722 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2723 close(masterfd);
2724
2725 logf("tagcache committed");
2726 remove(TAGCACHE_FILE_TEMP);
2727 tc_stat.ready = check_all_headers();
2728 tc_stat.readyvalid = true;
2729
2730 if (local_allocation)
2731 {
2732 tempbuf = NULL;
2733 tempbuf_size = 0;
2734 }
2735
2736 #ifdef HAVE_DIRCACHE
2737 /* Rebuild the dircache, if we stole the buffer. */
2738 if (dircache_buffer_stolen)
2739 dircache_build(0);
2740 #endif
2741
2742 #ifdef HAVE_TC_RAMCACHE
2743 /* Reload tagcache. */
2744 if (tc_stat.ramcache_allocated > 0)
2745 tagcache_start_scan();
2746 #endif
2747
2748 read_lock--;
2749
2750 return true;
2751 }
2752
2753 static void allocate_tempbuf(void)
2754 {
2755 /* Yeah, malloc would be really nice now :) */
2756 #ifdef __PCTOOL__
2757 tempbuf_size = 32*1024*1024;
2758 tempbuf = malloc(tempbuf_size);
2759 #else
2760 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2761 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2762 audiobuf += tempbuf_size;
2763 #endif
2764 }
2765
2766 static void free_tempbuf(void)
2767 {
2768 if (tempbuf_size == 0)
2769 return ;
2770
2771 #ifdef __PCTOOL__
2772 free(tempbuf);
2773 #else
2774 audiobuf -= tempbuf_size;
2775 #endif
2776 tempbuf = NULL;
2777 tempbuf_size = 0;
2778 }
2779
2780 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2781 {
2782 struct index_entry idx;
2783
2784 if (!tc_stat.ready)
2785 return false;
2786
2787 if (!tagcache_is_numeric_tag(tag))
2788 return false;
2789
2790 if (!get_index(masterfd, idx_id, &idx, false))
2791 return false;
2792
2793 idx.tag_seek[tag] = data;
2794 idx.flag |= FLAG_DIRTYNUM;
2795
2796 return write_index(masterfd, idx_id, &idx);
2797 }
2798
2799 #if 0
2800 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
2801 int tag, long data)
2802 {
2803 struct master_header myhdr;
2804
2805 if (tcs->masterfd < 0)
2806 {
2807 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
2808 return false;
2809 }
2810
2811 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
2812 }
2813 #endif
2814
2815 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
2816
2817 static bool command_queue_is_full(void)
2818 {
2819 int next;
2820
2821 next = command_queue_widx + 1;
2822 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
2823 next = 0;
2824
2825 return (next == command_queue_ridx);
2826 }
2827
2828 void run_command_queue(bool force)
2829 {
2830 struct master_header myhdr;
2831 int masterfd;
2832
2833 if (COMMAND_QUEUE_IS_EMPTY)
2834 return;
2835
2836 if (!force && !command_queue_is_full()
2837 && current_tick - TAGCACHE_COMMAND_QUEUE_COMMIT_DELAY
2838 < command_queue_timestamp)
2839 {
2840 return;
2841 }
2842
2843 mutex_lock(&command_queue_mutex);
2844
2845 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
2846 return;
2847
2848 while (command_queue_ridx != command_queue_widx)
2849 {
2850 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
2851
2852 switch (ce->command)
2853 {
2854 case CMD_UPDATE_MASTER_HEADER:
2855 {
2856 close(masterfd);
2857 update_master_header();
2858
2859 /* Re-open the masterfd. */
2860 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
2861 return;
2862
2863 break;
2864 }
2865 case CMD_UPDATE_NUMERIC:
2866 {
2867 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
2868 break;
2869 }
2870 }
2871
2872 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
2873 command_queue_ridx = 0;
2874 }
2875
2876 close(masterfd);
2877
2878 mutex_unlock(&command_queue_mutex);
2879 }
2880
2881 static void queue_command(int cmd, long idx_id, int tag, long data)
2882 {
2883 while (1)
2884 {
2885 int next;
2886
2887 mutex_lock(&command_queue_mutex);
2888 next = command_queue_widx + 1;
2889 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
2890 next = 0;
2891
2892 /* Make sure queue is not full. */
2893 if (next != command_queue_ridx)
2894 {
2895 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
2896
2897 ce->command = cmd;
2898 ce->idx_id = idx_id;
2899 ce->tag = tag;
2900 ce->data = data;
2901
2902 command_queue_widx = next;
2903 command_queue_timestamp = current_tick;
2904 mutex_unlock(&command_queue_mutex);
2905 break;
2906 }
2907
2908 /* Queue is full, try again later... */
2909 mutex_unlock(&command_queue_mutex);
2910 sleep(1);
2911 }
2912 }
2913
2914 long tagcache_increase_serial(void)
2915 {
2916 long old;
2917
2918 if (!tc_stat.ready)
2919 return -2;
2920
2921 while (read_lock)
2922 sleep(1);
2923
2924 old = current_tcmh.serial++;
2925 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
2926
2927 return old;
2928 }
2929
2930 void tagcache_update_numeric(int idx_id, int tag, long data)
2931 {
2932 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
2933 }
2934
2935 long tagcache_get_serial(void)
2936 {
2937 return current_tcmh.serial;
2938 }
2939
2940 long tagcache_get_commitid(void)
2941 {
2942 return current_tcmh.commitid;
2943 }
2944
2945 static bool write_tag(int fd, const char *tagstr, const char *datastr)
2946 {
2947 char buf[512];
2948 int i;
2949
2950 snprintf(buf, sizeof buf, "%s=\"", tagstr);
2951 for (i = strlen(buf); i < (long)sizeof(buf)-3; i++)
2952 {
2953 if (*datastr == '\0')
2954 break;
2955
2956 if (*datastr == '"')
2957 {
2958 buf[i] = '\\';
2959 datastr++;
2960 continue;
2961 }
2962
2963 buf[i] = *(datastr++);
2964 }
2965
2966 strcpy(&buf[i], "\" ");
2967
2968 write(fd, buf, i + 2);
2969
2970 return true;
2971 }
2972
2973 static bool read_tag(char *dest, long size,
2974 const char *src, const char *tagstr)
2975 {
2976 int pos;
2977 char current_tag[32];
2978
2979 while (*src != '\0')
2980 {
2981 /* Skip all whitespace */
2982 while (*src == ' ')
2983 src++;
2984
2985 if (*src == '\0')
2986 break;
2987
2988 pos = 0;
2989 /* Read in tag name */
2990 while (*src != '=' && *src != ' ')
2991 {
2992 current_tag[pos] = *src;
2993 src++;
2994 pos++;
2995
2996 if (*src == '\0' || pos >= (long)sizeof(current_tag))
2997 return false;
2998 }
2999 current_tag[pos] = '\0';
3000
3001 /* Read in tag data */
3002
3003 /* Find the start. */
3004 while (*src != '"' && *src != '\0')
3005 src++;
3006
3007 if (*src == '\0' || *(++src) == '\0')
3008 return false;
3009
3010 /* Read the data. */
3011 for (pos = 0; pos < size; pos++)
3012 {
3013 if (*src == '\0')
3014 break;
3015
3016 if (*src == '\\' && *(src+1) == '"')
3017 {
3018 dest[pos] = '"';
3019 src += 2;
3020 continue;
3021 }
3022
3023 dest[pos] = *src;
3024
3025 if (*src == '"')
3026 {
3027 src++;
3028 break;
3029 }
3030
3031 if (*src == '\0')
3032 break;
3033
3034 src++;
3035 }
3036 dest[pos] = '\0';
3037
3038 if (!strcasecmp(tagstr, current_tag))
3039 return true;
3040 }
3041
3042 return false;
3043 }
3044
3045 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3046 {
3047 struct index_entry idx;
3048 char tag_data[TAG_MAXLEN+32];
3049 int idx_id;
3050 long masterfd = (long)parameters;
3051 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3052 tag_commitid };
3053 int i;
3054 (void)line_n;
3055
3056 if (*buf == '#')
3057 return 0;
3058
3059 logf("%d/%s", line_n, buf);
3060 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3061 {
3062 logf("filename missing");
3063 logf("-> %s", buf);
3064 return 0;
3065 }
3066
3067 idx_id = find_index(tag_data);
3068 if (idx_id < 0)
3069 {
3070 logf("entry not found");
3071 return 0;
3072 }
3073
3074 if (!get_index(masterfd, idx_id, &idx, false))
3075 {
3076 logf("failed to retrieve index entry");
3077 return 0;
3078 }
3079
3080 /* Stop if tag has already been modified. */
3081 if (idx.flag & FLAG_DIRTYNUM)
3082 return 0;
3083
3084 logf("import: %s", tag_data);
3085
3086 idx.flag |= FLAG_DIRTYNUM;
3087 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3088 {
3089 int data;
3090
3091 if (!read_tag(tag_data, sizeof tag_data, buf,
3092 tagcache_tag_to_str(import_tags[i])))
3093 {
3094 continue;
3095 }
3096
3097 data = atoi(tag_data);
3098 if (data < 0)
3099 continue;
3100
3101 idx.tag_seek[import_tags[i]] = data;
3102
3103 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3104 current_tcmh.serial = data;
3105 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3106 current_tcmh.commitid = data + 1;
3107 }
3108
3109 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3110 }
3111
3112 #ifndef __PCTOOL__
3113 bool tagcache_import_changelog(void)
3114 {
3115 struct master_header myhdr;
3116 struct tagcache_header tch;
3117 int clfd;
3118 long masterfd;
3119 char buf[2048];
3120
3121 if (!tc_stat.ready)
3122 return false;
3123
3124 while (read_lock)
3125 sleep(1);
3126
3127 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3128 if (clfd < 0)
3129 {
3130 logf("failure to open changelog");
3131 return false;
3132 }
3133
3134 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3135 {
3136 close(clfd);
3137 return false;
3138 }
3139
3140 write_lock++;
3141
3142 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3143
3144 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3145 parse_changelog_line);
3146
3147 close(clfd);
3148 close(masterfd);
3149
3150 if (filenametag_fd >= 0)
3151 close(filenametag_fd);
3152
3153 write_lock--;
3154
3155 update_master_header();
3156
3157 return true;
3158 }
3159 #endif
3160
3161 bool tagcache_create_changelog(struct tagcache_search *tcs)
3162 {
3163 struct master_header myhdr;
3164 struct index_entry idx;
3165 char buf[TAG_MAXLEN+32];
3166 char temp[32];
3167 int clfd;
3168 int i, j;
3169
3170 if (!tc_stat.ready)
3171 return false;
3172
3173 if (!tagcache_search(tcs, tag_filename))
3174 return false;
3175
3176 /* Initialize the changelog */
3177 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3178 if (clfd < 0)
3179 {
3180 logf("failure to open changelog");
3181 return false;
3182 }
3183
3184 if (tcs->masterfd < 0)
3185 {
3186 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3187 return false;
3188 }
3189 else
3190 {
3191 lseek(tcs->masterfd, 0, SEEK_SET);
3192 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3193 }
3194
3195 write(clfd, "## Changelog version 1\n", 23);
3196
3197 for (i = 0; i < myhdr.tch.entry_count; i++)
3198 {
3199 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3200 != sizeof(struct index_entry))
3201 {
3202 logf("read error #9");
3203 tagcache_search_finish(tcs);
3204 close(clfd);
3205 return false;
3206 }
3207
3208 /* Skip until the entry found has been modified. */
3209 if (! (idx.flag & FLAG_DIRTYNUM) )
3210 continue;
3211
3212 /* Skip deleted entries too. */
3213 if (idx.flag & FLAG_DELETED)
3214 continue;
3215
3216 /* Now retrieve all tags. */
3217 for (j = 0; j < TAG_COUNT; j++)
3218 {
3219 if (tagcache_is_numeric_tag(j))
3220 {
3221 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3222 write_tag(clfd, tagcache_tag_to_str(j), temp);
3223 continue;
3224 }
3225
3226 tcs->type = j;
3227 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3228 write_tag(clfd, tagcache_tag_to_str(j), buf);
3229 }
3230
3231 write(clfd, "\n", 1);
3232 do_timed_yield();
3233 }
3234
3235 close(clfd);
3236
3237 tagcache_search_finish(tcs);
3238
3239 return true;
3240 }
3241
3242 static bool delete_entry(long idx_id)
3243 {
3244 int fd;
3245 int tag, i;
3246 struct index_entry idx, myidx;
3247 struct master_header myhdr;
3248 char buf[TAG_MAXLEN+32];
3249 int in_use[TAG_COUNT];
3250
3251 logf("delete_entry(): %ld", idx_id);
3252
3253 #ifdef HAVE_TC_RAMCACHE
3254 /* At first mark the entry removed from ram cache. */
3255 if (tc_stat.ramcache)
3256 hdr->indices[idx_id].flag |= FLAG_DELETED;
3257 #endif
3258
3259 if ( (fd = open_master_fd(&myhdr, true) ) < 0)
3260 return false;
3261
3262 lseek(fd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3263 if (ecread(fd, &myidx, 1, index_entry_ec, tc_stat.econ)
3264 != sizeof(struct index_entry))
3265 {
3266 logf("delete_entry(): read error");
3267 close(fd);
3268 return false;
3269 }
3270
3271 myidx.flag |= FLAG_DELETED;
3272 lseek(fd, -sizeof(struct index_entry), SEEK_CUR);
3273 if (ecwrite(fd, &myidx, 1, index_entry_ec, tc_stat.econ)
3274 != sizeof(struct index_entry))
3275 {
3276 logf("delete_entry(): write_error");
3277 close(fd);
3278 return false;
3279 }
3280
3281 /* Now check which tags are no longer in use (if any) */
3282 for (tag = 0; tag < TAG_COUNT; tag++)
3283 in_use[tag] = 0;
3284
3285 lseek(fd, sizeof(struct master_header), SEEK_SET);
3286 for (i = 0; i < myhdr.tch.entry_count; i++)
3287 {
3288 struct index_entry *idxp;
3289
3290 #ifdef HAVE_TC_RAMCACHE
3291 /* Use RAM DB if available for greater speed */
3292 if (tc_stat.ramcache)
3293 idxp = &hdr->indices[i];
3294 else
3295 #endif
3296 {
3297 if (ecread(fd, &idx, 1, index_entry_ec, tc_stat.econ)
3298 != sizeof(struct index_entry))
3299 {
3300 logf("delete_entry(): read error #2");
3301 close(fd);
3302 return false;
3303 }
3304 idxp = &idx;
3305 }
3306
3307 if (idxp->flag & FLAG_DELETED)
3308 continue;
3309
3310 for (tag = 0; tag < TAG_COUNT; tag++)
3311 {
3312 if (tagcache_is_numeric_tag(tag))
3313 continue;
3314
3315 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3316 in_use[tag]++;
3317 }
3318 }
3319
3320 close(fd);
3321
3322 /* Now delete all tags no longer in use. */
3323 for (tag = 0; tag < TAG_COUNT; tag++)
3324 {
3325 if (tagcache_is_numeric_tag(tag))
3326 continue;
3327
3328 if (in_use[tag])
3329 {
3330 logf("in use: %d/%d", tag, in_use[tag]);
3331 continue;
3332 }
3333
3334 #ifdef HAVE_TC_RAMCACHE
3335 /* Delete from ram. */
3336 if (tc_stat.ramcache && tag != tag_filename)
3337 {
3338 struct tagfile_entry *tagentry = get_tag(&myidx, tag);
3339 tagentry->tag_data[0] = '\0';
3340 }
3341 #endif
3342
3343 /* Open the index file, which contains the tag names. */
3344 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
3345 fd = open(buf, O_RDWR);
3346
3347 if (fd < 0)
3348 {
3349 logf("open failed");
3350 return false;
3351 }
3352
3353 /* Skip the header block */
3354 lseek(fd, myidx.tag_seek[tag] + sizeof(struct tagfile_entry), SEEK_SET);
3355
3356 /* Debug, print 10 first characters of the tag
3357 read(fd, buf, 10);
3358 buf[10]='\0';
3359 logf("TAG:%s", buf);
3360 lseek(fd, -10, SEEK_CUR);
3361 */
3362
3363 /* Write first data byte in tag as \0 */
3364 write(fd, "", 1);
3365
3366 /* Now tag data has been removed */
3367 close(fd);
3368 }
3369
3370 return true;
3371 }
3372
3373 #ifndef __PCTOOL__
3374 /**
3375 * Returns true if there is an event waiting in the queue
3376 * that requires the current operation to be aborted.
3377 */
3378 static bool check_event_queue(void)
3379 {
3380 struct event ev;
3381
3382 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3383 switch (ev.id)
3384 {
3385 case Q_STOP_SCAN:
3386 case SYS_POWEROFF:
3387 case SYS_USB_CONNECTED:
3388 /* Put the event back into the queue. */
3389 queue_post(&tagcache_queue, ev.id, ev.data);
3390 return true;
3391 }
3392
3393 return false;
3394 }
3395 #endif
3396
3397 #ifdef HAVE_TC_RAMCACHE
3398 static bool allocate_tagcache(void)
3399 {
3400 struct master_header tcmh;
3401 int fd;
3402
3403 /* Load the header. */
3404 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3405 {
3406 hdr = NULL;
3407 return false;
3408 }
3409
3410 close(fd);
3411
3412 /**
3413 * Now calculate the required cache size plus
3414 * some extra space for alignment fixes.
3415 */
3416 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3417 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3418 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3419 memset(hdr, 0, sizeof(struct ramcache_header));
3420 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3421 hdr->indices = (struct index_entry *)(hdr + 1);
3422 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3423
3424 return true;
3425 }
3426
3427 # ifdef HAVE_EEPROM_SETTINGS
3428 static bool tagcache_dumpload(void)
3429 {
3430 struct statefile_header shdr;
3431 int fd, rc;
3432 long offpos;
3433 int i;
3434
3435 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3436 if (fd < 0)
3437 {
3438 logf("no tagcache statedump");
3439 return false;
3440 }
3441
3442 /* Check the statefile memory placement */
3443 hdr = buffer_alloc(0);
3444 rc = read(fd, &shdr, sizeof(struct statefile_header));
3445 if (rc != sizeof(struct statefile_header)
3446 /* || (long)hdr != (long)shdr.hdr */)
3447 {
3448 logf("incorrect statefile");
3449 hdr = NULL;
3450 close(fd);
3451 return false;
3452 }
3453
3454 offpos = (long)hdr - (long)shdr.hdr;
3455
3456 /* Lets allocate real memory and load it */
3457 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3458 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3459 close(fd);
3460
3461 if (rc != shdr.tc_stat.ramcache_allocated)
3462 {
3463 logf("read failure!");
3464 hdr = NULL;
3465 return false;
3466 }
3467
3468 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3469
3470 /* Now fix the pointers */
3471 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3472 for (i = 0; i < TAG_COUNT; i++)
3473 hdr->tags[i] += offpos;
3474
3475 return true;
3476 }
3477
3478 static bool tagcache_dumpsave(void)
3479 {
3480 struct statefile_header shdr;
3481 int fd;
3482
3483 if (!tc_stat.ramcache)
3484 return false;
3485
3486 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3487 if (fd < 0)
3488 {
3489 logf("failed to create a statedump");
3490 return false;
3491 }
3492
3493 /* Create the header */
3494 shdr.hdr = hdr;
3495 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3496 write(fd, &shdr, sizeof(struct statefile_header));
3497
3498 /* And dump the data too */
3499 write(fd, hdr, tc_stat.ramcache_allocated);
3500 close(fd);
3501
3502 return true;
3503 }
3504 # endif
3505
3506 static bool load_tagcache(void)
3507 {
3508 struct tagcache_header *tch;
3509 long bytesleft = tc_stat.ramcache_allocated;
3510 struct index_entry *idx;
3511 int rc, fd;
3512 char *p;
3513 int i, tag;
3514
3515 # ifdef HAVE_DIRCACHE
3516 while (dircache_is_initializing())
3517 sleep(1);
3518 # endif
3519
3520 logf("loading tagcache to ram...");
3521
3522 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3523 if (fd < 0)
3524 {
3525 logf("tagcache open failed");
3526 return false;
3527 }
3528
3529 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3530 != sizeof(struct master_header)
3531 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3532 {
3533 logf("incorrect header");
3534 return false;
3535 }
3536
3537 idx = hdr->indices;
3538
3539 /* Load the master index table. */
3540 for (i = 0; i < hdr->h.tch.entry_count; i++)
3541 {
3542 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3543 if (rc != sizeof(struct index_entry))
3544 {
3545 logf("read error #10");
3546 close(fd);
3547 return false;
3548 }
3549
3550 bytesleft -= sizeof(struct index_entry);
3551 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3552 {
3553 logf("too big tagcache.");
3554 close(fd);
3555 return false;
3556 }
3557
3558 idx++;
3559 }
3560
3561 close(fd);
3562
3563 /* Load the tags. */
3564 p = (char *)idx;
3565 for (tag = 0; tag < TAG_COUNT; tag++)
3566 {
3567 struct tagfile_entry *fe;
3568 char buf[TAG_MAXLEN+32];
3569
3570 if (tagcache_is_numeric_tag(tag))
3571 continue ;
3572
3573 //p = ((void *)p+1);
3574 p = (char *)((long)p & ~0x03) + 0x04;
3575 hdr->tags[tag] = p;
3576
3577 /* Check the header. */
3578 tch = (struct tagcache_header *)p;
3579 p += sizeof(struct tagcache_header);
3580
3581 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3582 return false;
3583
3584 for (hdr->entry_count[tag] = 0;
3585 hdr->entry_count[tag] < tch->entry_count;
3586 hdr->entry_count[tag]++)
3587 {
3588 long pos;
3589
3590 if (do_timed_yield())
3591 {
3592 /* Abort if we got a critical event in queue */
3593 if (check_event_queue())
3594 return false;
3595 }
3596
3597 fe = (struct tagfile_entry *)p;
3598 pos = lseek(fd, 0, SEEK_CUR);
3599 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3600 if (rc != sizeof(struct tagfile_entry))
3601 {
3602 /* End of lookup table. */
3603 logf("read error #11");
3604 close(fd);
3605 return false;
3606 }
3607
3608 /* We have a special handling for the filename tags. */
3609 if (tag == tag_filename)
3610 {
3611 # ifdef HAVE_DIRCACHE
3612 const struct dirent *dc;
3613 # endif
3614
3615 // FIXME: This is wrong!
3616 // idx = &hdr->indices[hdr->entry_count[i]];
3617 idx = &hdr->indices[fe->idx_id];
3618
3619 if (fe->tag_length >= (long)sizeof(buf)-1)
3620 {
3621 read(fd, buf, 10);
3622 buf[10] = '\0';
3623 logf("TAG:%s", buf);
3624 logf("too long filename");
3625 close(fd);
3626 return false;
3627 }
3628
3629 rc = read(fd, buf, fe->tag_length);
3630 if (rc != fe->tag_length)
3631 {
3632 logf("read error #12");
3633 close(fd);
3634 return false;
3635 }
3636
3637 /* Check if the entry has already been removed */
3638 if (idx->flag & FLAG_DELETED)
3639 continue;
3640
3641 /* This flag must not be used yet. */
3642 if (idx->flag & FLAG_DIRCACHE)
3643 {
3644 logf("internal error!");
3645 close(fd);
3646 return false;
3647 }
3648
3649 if (idx->tag_seek[tag] != pos)
3650 {
3651 logf("corrupt data structures!");
3652 close(fd);
3653 return false;
3654 }
3655
3656 # ifdef HAVE_DIRCACHE
3657 if (dircache_is_enabled())
3658 {
3659 dc = dircache_get_entry_ptr(buf);
3660 if (dc == NULL)
3661 {
3662 logf("Entry no longer valid.");
3663 logf("-> %s", buf);
3664 delete_entry(fe->idx_id);
3665 continue ;
3666 }
3667
3668 idx->flag |= FLAG_DIRCACHE;
3669 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3670 idx->tag_seek[tag_filename] = (long)dc;
3671 }
3672 else
3673 # endif
3674 {
3675
3676 # if 0 /* Maybe we could enable this for flash players. Too slow otherwise. */
3677 /* Check if entry has been removed. */
3678 if (global_settings.tagcache_autoupdate)
3679 {
3680 int testfd;
3681
3682 testfd = open(buf, O_RDONLY);
3683 if (testfd < 0)
3684 {
3685 logf("Entry no longer valid.");
3686 logf("-> %s", buf);
3687 delete_entry(fe->idx_id);
3688 continue;
3689 }
3690 close(testfd);
3691 }
3692 # endif
3693 }
3694
3695 continue ;
3696 }
3697
3698 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3699 if (bytesleft < 0)
3700 {
3701 logf("too big tagcache #2");
3702 logf("tl: %d", fe->tag_length);
3703 logf("bl: %ld", bytesleft);
3704 close(fd);
3705 return false;
3706 }
3707
3708 p = fe->tag_data;
3709 rc = read(fd, fe->tag_data, fe->tag_length);
3710 p += rc;
3711
3712 if (rc != fe->tag_length)
3713 {
3714 logf("read error #13");
3715 logf("rc=0x%04x", rc); // 0x431
3716 logf("len=0x%04x", fe->tag_length); // 0x4000
3717 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3718 logf("tag=0x%02x", tag); // 0x00
3719 close(fd);
3720 return false;
3721 }
3722 }
3723 close(fd);
3724 }
3725
3726 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
3727 logf("tagcache loaded into ram!");
3728
3729 return true;
3730 }
3731 #endif /* HAVE_TC_RAMCACHE */
3732
3733 static bool check_deleted_files(void)
3734 {
3735 int fd, testfd;
3736 char buf[TAG_MAXLEN+32];
3737 struct tagfile_entry tfe;
3738
3739 logf("reverse scan...");
3740 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
3741 fd = open(buf, O_RDONLY);
3742
3743 if (fd < 0)
3744 {
3745 logf("%s open fail", buf);
3746 return false;
3747 }
3748
3749 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
3750 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3751 == sizeof(struct tagfile_entry)
3752 #ifndef __PCTOOL__
3753 && !check_event_queue()
3754 #endif
3755 )
3756 {
3757 if (tfe.tag_length >= (long)sizeof(buf)-1)
3758 {
3759 logf("too long tag");
3760 close(fd);
3761 return false;
3762 }
3763
3764 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3765 {
3766 logf("read error #14");
3767 close(fd);
3768 return false;
3769 }
3770
3771 /* Check if the file has already deleted from the db. */
3772 if (*buf == '\0')
3773 continue;
3774
3775 /* Now check if the file exists. */
3776 testfd = open(buf, O_RDONLY);
3777 if (testfd < 0)
3778 {
3779 logf("Entry no longer valid.");
3780 logf("-> %s / %d", buf, tfe.tag_length);
3781 delete_entry(tfe.idx_id);
3782 }
3783 close(testfd);
3784 }
3785
3786 close(fd);
3787
3788 logf("done");
3789
3790 return true;
3791 }
3792
3793 static bool check_dir(const char *dirname)
3794 {
3795 DIR *dir;
3796 int len;
3797 int success = false;
3798
3799 dir = opendir(dirname);
3800 if (!dir)
3801 {
3802 logf("tagcache: opendir() failed");
3803 return false;
3804 }
3805
3806 /* Recursively scan the dir. */
3807 #ifdef __PCTOOL__
3808 while (1)
3809 #else
3810 while (!check_event_queue())
3811 #endif
3812 {
3813 struct dirent *entry;
3814
3815 entry = readdir(dir);
3816
3817 if (entry == NULL)
3818 {
3819 success = true;
3820 break ;
3821 }
3822
3823 if (!strcmp((char *)entry->d_name, ".") ||
3824 !strcmp((char *)entry->d_name, ".."))
3825 continue;
3826
3827 yield();
3828
3829 len = strlen(curpath);
3830 snprintf(&curpath[len], curpath_size - len, "/%s",
3831 entry->d_name);
3832
3833 processed_dir_count++;
3834 if (entry->attribute & ATTR_DIRECTORY)
3835 check_dir(curpath);
3836 else
3837 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
3838 add_tagcache(curpath, dir->internal_entry);
3839 #else
3840 add_tagcache(curpath);
3841 #endif
3842
3843 curpath[len] = '\0';
3844 }
3845
3846 closedir(dir);
3847
3848 return success;
3849 }
3850
3851 void build_tagcache(const char *path)
3852 {
3853 struct tagcache_header header;
3854 bool ret;
3855
3856 curpath[0] = '\0';
3857 data_size = 0;
3858 total_entry_count = 0;
3859 processed_dir_count = 0;
3860
3861 #ifdef HAVE_DIRCACHE
3862 while (dircache_is_initializing())
3863 sleep(1);
3864 #endif
3865
3866 logf("updating tagcache");
3867
3868 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
3869 if (cachefd >= 0)
3870 {
3871 logf("skipping, cache already waiting for commit");
3872 close(cachefd);
3873 return ;
3874 }
3875
3876 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
3877 if (cachefd < 0)
3878 {
3879 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
3880 return ;
3881 }
3882
3883 filenametag_fd = open_tag_fd(&header, tag_filename, false);
3884
3885 cpu_boost(true);
3886
3887 logf("Scanning files...");
3888 /* Scan for new files. */
3889 memset(&header, 0, sizeof(struct tagcache_header));
3890 write(cachefd, &header, sizeof(struct tagcache_header));
3891
3892 if (strcmp("/", path) != 0)
3893 strcpy(curpath, path);
3894 ret = check_dir(path);
3895
3896 /* Write the header. */
3897 header.magic = TAGCACHE_MAGIC;
3898 header.datasize = data_size;
3899 header.entry_count = total_entry_count;
3900 lseek(cachefd, 0, SEEK_SET);
3901 write(cachefd, &header, sizeof(struct tagcache_header));
3902 close(cachefd);
3903
3904 if (filenametag_fd >= 0)
3905 {
3906 close(filenametag_fd);
3907 filenametag_fd = -1;
3908 }
3909
3910 if (!ret)
3911 {
3912 logf("Aborted.");
3913 cpu_boost(false);
3914 return ;
3915 }
3916
3917 /* Commit changes to the database. */
3918 #ifdef __PCTOOL__
3919 allocate_tempbuf();
3920 #endif
3921 if (commit())
3922 {
3923 remove(TAGCACHE_FILE_TEMP);
3924 logf("tagcache built!");
3925 }
3926 #ifdef __PCTOOL__
3927 free_tempbuf();
3928 #endif
3929
3930 #ifdef HAVE_TC_RAMCACHE
3931 if (hdr)
3932 {
3933 /* Import runtime statistics if we just initialized the db. */
3934 if (hdr->h.serial == 0)
3935 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
3936 }
3937 #endif
3938
3939 cpu_boost(false);
3940 }
3941
3942 #ifdef HAVE_TC_RAMCACHE
3943 static void load_ramcache(void)
3944 {
3945 if (!hdr)
3946 return ;
3947
3948 cpu_boost(true);
3949
3950 /* At first we should load the cache (if exists). */
3951 tc_stat.ramcache = load_tagcache();
3952
3953 if (!tc_stat.ramcache)
3954 {
3955 /* If loading failed, it must indicate some problem with the db
3956 * so disable it entirely to prevent further issues. */
3957 tc_stat.ready = false;
3958 hdr = NULL;
3959 }
3960
3961 cpu_boost(false);
3962 }
3963
3964 void tagcache_unload_ramcache(void)
3965 {
3966 tc_stat.ramcache = false;
3967 /* Just to make sure there is no statefile present. */
3968 // remove(TAGCACHE_STATEFILE);
3969 }
3970 #endif
3971
3972 #ifndef __PCTOOL__
3973 static void tagcache_thread(void)
3974 {
3975 struct event ev;
3976 bool check_done = false;
3977
3978 /* If the previous cache build/update was interrupted, commit
3979 * the changes first in foreground. */
3980 cpu_boost(true);
3981 allocate_tempbuf();
3982 commit();
3983 free_tempbuf();
3984
3985 #ifdef HAVE_TC_RAMCACHE
3986 # ifdef HAVE_EEPROM_SETTINGS
3987 if (firmware_settings.initialized && firmware_settings.disk_clean)
3988 check_done = tagcache_dumpload();
3989
3990 remove(TAGCACHE_STATEFILE);
3991 # endif
3992
3993 /* Allocate space for the tagcache if found on disk. */
3994 if (global_settings.tagcache_ram && !tc_stat.ramcache)
3995 allocate_tagcache();
3996 #endif
3997
3998 cpu_boost(false);
3999 tc_stat.initialized = true;
4000
4001 /* Don't delay bootup with the header check but do it on background. */
4002 sleep(HZ);
4003 tc_stat.ready = check_all_headers();
4004 tc_stat.readyvalid = true;
4005
4006 while (1)
4007 {
4008 run_command_queue(false);
4009
4010 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4011
4012 switch (ev.id)
4013 {
4014 case Q_IMPORT_CHANGELOG:
4015 tagcache_import_changelog();
4016 break;
4017
4018 case Q_REBUILD:
4019 remove_files();
4020 build_tagcache("/");
4021 break;
4022
4023 case Q_UPDATE:
4024 build_tagcache("/");
4025 #ifdef HAVE_TC_RAMCACHE
4026 load_ramcache();
4027 #endif
4028 check_deleted_files();
4029 break ;
4030
4031 case Q_START_SCAN:
4032 check_done = false;
4033 case SYS_TIMEOUT:
4034 if (check_done || !tc_stat.ready)
4035 break ;
4036
4037 #ifdef HAVE_TC_RAMCACHE
4038 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4039 {
4040 load_ramcache();
4041 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4042 build_tagcache("/");
4043 }
4044 else
4045 #endif
4046 if (global_settings.tagcache_autoupdate)
4047 {
4048 build_tagcache("/");
4049 /* Don't do auto removal without dircache (very slow). */
4050 #ifdef HAVE_DIRCACHE
4051 if (dircache_is_enabled())
4052 check_deleted_files();
4053 #endif
4054 }
4055
4056 logf("tagcache check done");
4057
4058 check_done = true;
4059 break ;
4060
4061 case Q_STOP_SCAN:
4062 break ;
4063
4064 case SYS_POWEROFF:
4065 break ;
4066
4067 #ifndef SIMULATOR
4068 case SYS_USB_CONNECTED:
4069 logf("USB: TagCache");
4070 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4071 usb_wait_for_disconnect(&tagcache_queue);
4072 break ;
4073 #endif
4074 }
4075 }
4076 }
4077
4078 bool tagcache_prepare_shutdown(void)
4079 {
4080 if (tagcache_get_commit_step() > 0)
4081 return false;
4082
4083 tagcache_stop_scan();
4084 while (read_lock || write_lock)
4085 sleep(1);
4086
4087 return true;
4088 }
4089
4090 void tagcache_shutdown(void)
4091 {
4092 /* Flush the command queue. */
4093 run_command_queue(true);
4094
4095 #ifdef HAVE_EEPROM_SETTINGS
4096 if (tc_stat.ramcache)
4097 tagcache_dumpsave();
4098 #endif
4099 }
4100
4101 static int get_progress(void)
4102 {
4103 int total_count = -1;
4104
4105 #ifdef HAVE_DIRCACHE
4106 if (dircache_is_enabled())
4107 {
4108 total_count = dircache_get_entry_count();
4109 }
4110 else
4111 #endif
4112 #ifdef HAVE_TC_RAMCACHE
4113 {
4114 if (hdr && tc_stat.ramcache)
4115 total_count = hdr->h.tch.entry_count;
4116 }
4117 #endif
4118
4119 if (total_count < 0)
4120 return -1;
4121
4122 return processed_dir_count * 100 / total_count;
4123 }
4124
4125 struct tagcache_stat* tagcache_get_stat(void)
4126 {
4127 tc_stat.progress = get_progress();
4128 tc_stat.processed_entries = processed_dir_count;
4129
4130 return &tc_stat;
4131 }
4132
4133 void tagcache_start_scan(void)
4134 {
4135 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4136 }
4137
4138 bool tagcache_update(void)
4139 {
4140 if (!tc_stat.ready)
4141 return false;
4142
4143 queue_post(&tagcache_queue, Q_UPDATE, 0);
4144 return false;
4145 }
4146
4147 bool tagcache_rebuild()
4148 {
4149 queue_post(&tagcache_queue, Q_REBUILD, 0);
4150 return false;
4151 }
4152
4153 void tagcache_stop_scan(void)
4154 {
4155 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4156 }
4157
4158 #ifdef HAVE_TC_RAMCACHE
4159 bool tagcache_is_ramcache(void)
4160 {
4161 return tc_stat.ramcache;
4162 }
4163 #endif
4164
4165 #endif /* !__PCTOOL__ */
4166
4167
4168 void tagcache_init(void)
4169 {
4170 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4171 memset(&current_tcmh, 0, sizeof(struct master_header));
4172 filenametag_fd = -1;
4173 write_lock = read_lock = 0;
4174
4175 #ifndef __PCTOOL__
4176 mutex_init(&command_queue_mutex);
4177 queue_init(&tagcache_queue, true);
4178 create_thread(tagcache_thread, tagcache_stack,
4179 sizeof(tagcache_stack), tagcache_thread_name
4180 IF_PRIO(, PRIORITY_BACKGROUND)
4181 IF_COP(, CPU, false));
4182 #else
4183 tc_stat.initialized = true;
4184 allocate_tempbuf();
4185 commit();
4186 free_tempbuf();
4187 tc_stat.ready = check_all_headers();
4188 #endif
4189 }
4190
4191 #ifdef __PCTOOL__
4192 void tagcache_reverse_scan(void)
4193 {
4194 logf("Checking for deleted files");
4195 check_deleted_files();
4196 }
4197 #endif
4198
4199 bool tagcache_is_initialized(void)
4200 {
4201 return tc_stat.initialized;
4202 }
4203 bool tagcache_is_usable(void)
4204 {
4205 return tc_stat.initialized && tc_stat.ready;
4206 }
4207 int tagcache_get_commit_step(void)
4208 {
4209 return tc_stat.commit_step;
4210 }
4211 int tagcache_get_max_commit_step(void)
4212 {
4213 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4214 }
4215
Open Source Jukebox Firmware