?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Fix red
[Rockbox.git] / apps / tagcache.c
bloba53bdc7f8406ceafaebee3171fc65150b547c821
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "thread.h"
61 #include "kernel.h"
62 #include "system.h"
63 #include "logf.h"
64 #include "string.h"
65 #include "usb.h"
66 #include "metadata.h"
67 #include "id3.h"
68 #include "tagcache.h"
69 #include "buffer.h"
70 #include "crc32.h"
71 #include "misc.h"
72 #include "settings.h"
73 #include "dir.h"
74 #include "structec.h"
75 #ifndef __PCTOOL__
76 #include "atoi.h"
77 #include "splash.h"
78 #include "lang.h"
79 #include "eeprom_settings.h"
80 #endif
81
82 #ifdef __PCTOOL__
83 #define yield() do { } while(0)
84 #define sim_sleep(timeout) do { } while(0)
85 #define do_timed_yield() do { } while(0)
86 #endif
87
88 #ifndef __PCTOOL__
89 /* Tag Cache thread. */
90 static struct event_queue tagcache_queue;
91 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
92 static const char tagcache_thread_name[] = "tagcache";
93 #endif
94
95 #define UNTAGGED "<Untagged>"
96
97 /* Previous path when scanning directory tree recursively. */
98 static char curpath[TAG_MAXLEN+32];
99 static long curpath_size = sizeof(curpath);
100
101 /* Used when removing duplicates. */
102 static char *tempbuf; /* Allocated when needed. */
103 static long tempbufidx; /* Current location in buffer. */
104 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
105 static long tempbuf_left; /* Buffer space left. */
106 static long tempbuf_pos;
107
108 /* Tags we want to get sorted (loaded to the tempbuf). */
109 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
110 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
111
112 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
113 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
114 tag_composer, tag_comment, tag_albumartist, tag_grouping };
115
116 /* Numeric tags (we can use these tags with conditional clauses). */
117 static const int numeric_tags[] = { tag_year, tag_discnumber,
118 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
119 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
120 tag_virt_length_min, tag_virt_length_sec,
121 tag_virt_playtime_min, tag_virt_playtime_sec,
122 tag_virt_entryage, tag_virt_autoscore };
123
124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
125 static const char *tags_str[] = { "artist", "album", "genre", "title",
126 "filename", "composer", "comment", "albumartist", "grouping", "year",
127 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
128 "playtime", "lastplayed", "commitid", "mtime" };
129
130 /* Status information of the tagcache. */
131 static struct tagcache_stat tc_stat;
132
133 /* Queue commands. */
134 enum tagcache_queue {
135 Q_STOP_SCAN = 0,
136 Q_START_SCAN,
137 Q_IMPORT_CHANGELOG,
138 Q_UPDATE,
139 Q_REBUILD,
140
141 /* Internal tagcache command queue. */
142 CMD_UPDATE_MASTER_HEADER,
143 CMD_UPDATE_NUMERIC,
144 };
145
146 struct tagcache_command_entry {
147 long command;
148 long idx_id;
149 long tag;
150 long data;
151 };
152
153 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
154 static volatile int command_queue_widx = 0;
155 static volatile int command_queue_ridx = 0;
156 static struct mutex command_queue_mutex;
157 /* Timestamp of the last added event, so we can wait a bit before committing the
158 * whole queue at once. */
159 static long command_queue_timestamp = 0;
160
161 /* Tag database structures. */
162
163 /* Variable-length tag entry in tag files. */
164 struct tagfile_entry {
165 short tag_length; /* Length of the data in bytes including '\0' */
166 short idx_id; /* Corresponding entry location in index file of not unique tags */
167 char tag_data[0]; /* Begin of the tag data */
168 };
169
170 /* Fixed-size tag entry in master db index. */
171 struct index_entry {
172 long tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
173 long flag; /* Status flags */
174 };
175
176 /* Header is the same in every file. */
177 struct tagcache_header {
178 long magic; /* Header version number */
179 long datasize; /* Data size in bytes */
180 long entry_count; /* Number of entries in this file */
181 };
182
183 struct master_header {
184 struct tagcache_header tch;
185 long serial; /* Increasing counting number */
186 long commitid; /* Number of commits so far */
187 long dirty;
188 };
189
190 /* For the endianess correction */
191 static const char *tagfile_entry_ec = "ss";
192 static const char *index_entry_ec = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
193 static const char *tagcache_header_ec = "lll";
194 static const char *master_header_ec = "llllll";
195
196 static struct master_header current_tcmh;
197
198 #ifdef HAVE_TC_RAMCACHE
199 /* Header is created when loading database to ram. */
200 struct ramcache_header {
201 struct master_header h; /* Header from the master index */
202 struct index_entry *indices; /* Master index file content */
203 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
204 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
205 };
206
207 # ifdef HAVE_EEPROM_SETTINGS
208 struct statefile_header {
209 struct ramcache_header *hdr;
210 struct tagcache_stat tc_stat;
211 };
212 # endif
213
214 /* Pointer to allocated ramcache_header */
215 static struct ramcache_header *hdr;
216 #endif
217
218 /**
219 * Full tag entries stored in a temporary file waiting
220 * for commit to the cache. */
221 struct temp_file_entry {
222 long tag_offset[TAG_COUNT];
223 short tag_length[TAG_COUNT];
224 long flag;
225
226 long data_length;
227 };
228
229 struct tempbuf_id_list {
230 long id;
231 struct tempbuf_id_list *next;
232 };
233
234 struct tempbuf_searchidx {
235 long idx_id;
236 char *str;
237 int seek;
238 struct tempbuf_id_list idlist;
239 };
240
241 /* Lookup buffer for fixing messed up index while after sorting. */
242 static long commit_entry_count;
243 static long lookup_buffer_depth;
244 static struct tempbuf_searchidx **lookup;
245
246 /* Used when building the temporary file. */
247 static int cachefd = -1, filenametag_fd;
248 static int total_entry_count = 0;
249 static int data_size = 0;
250 static int processed_dir_count;
251
252 /* Thread safe locking */
253 static volatile int write_lock;
254 static volatile int read_lock;
255
256 static bool delete_entry(long idx_id);
257
258 const char* tagcache_tag_to_str(int tag)
259 {
260 return tags_str[tag];
261 }
262
263 bool tagcache_is_numeric_tag(int type)
264 {
265 int i;
266
267 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
268 {
269 if (type == numeric_tags[i])
270 return true;
271 }
272
273 return false;
274 }
275
276 bool tagcache_is_unique_tag(int type)
277 {
278 int i;
279
280 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
281 {
282 if (type == unique_tags[i])
283 return true;
284 }
285
286 return false;
287 }
288
289 bool tagcache_is_sorted_tag(int type)
290 {
291 int i;
292
293 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
294 {
295 if (type == sorted_tags[i])
296 return true;
297 }
298
299 return false;
300 }
301
302 #ifdef HAVE_DIRCACHE
303 /**
304 * Returns true if specified flag is still present, i.e., dircache
305 * has not been reloaded.
306 */
307 static bool is_dircache_intact(void)
308 {
309 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
310 }
311 #endif
312
313 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
314 {
315 int fd;
316 char buf[MAX_PATH];
317 int rc;
318
319 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
320 return -1;
321
322 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
323
324 fd = open(buf, write ? O_RDWR : O_RDONLY);
325 if (fd < 0)
326 {
327 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
328 tc_stat.ready = false;
329 return fd;
330 }
331
332 /* Check the header. */
333 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
334 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
335 {
336 logf("header error");
337 tc_stat.ready = false;
338 close(fd);
339 return -2;
340 }
341
342 return fd;
343 }
344
345 static int open_master_fd(struct master_header *hdr, bool write)
346 {
347 int fd;
348 int rc;
349
350 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
351 if (fd < 0)
352 {
353 logf("master file open failed for R/W");
354 tc_stat.ready = false;
355 return fd;
356 }
357
358 tc_stat.econ = false;
359
360 /* Check the header. */
361 rc = read(fd, hdr, sizeof(struct master_header));
362 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
363 {
364 /* Success. */
365 return fd;
366 }
367
368 /* Trying to read again, this time with endianess correction enabled. */
369 lseek(fd, 0, SEEK_SET);
370
371 rc = ecread(fd, hdr, 1, master_header_ec, true);
372 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
373 {
374 logf("header error");
375 tc_stat.ready = false;
376 close(fd);
377 return -2;
378 }
379
380 tc_stat.econ = true;
381
382 return fd;
383 }
384
385 #ifndef __PCTOOL__
386 static bool do_timed_yield(void)
387 {
388 /* Sorting can lock up for quite a while, so yield occasionally */
389 static long wakeup_tick = 0;
390 if (current_tick >= wakeup_tick)
391 {
392 wakeup_tick = current_tick + (HZ/4);
393 yield();
394 return true;
395 }
396 return false;
397 }
398 #endif
399
400 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
401 static long find_entry_ram(const char *filename,
402 const struct dirent *dc)
403 {
404 static long last_pos = 0;
405 int i;
406
407 /* Check if we tagcache is loaded into ram. */
408 if (!tc_stat.ramcache)
409 return -1;
410
411 if (dc == NULL)
412 dc = dircache_get_entry_ptr(filename);
413
414 if (dc == NULL)
415 {
416 logf("tagcache: file not found.");
417 return -1;
418 }
419
420 try_again:
421
422 if (last_pos > 0)
423 i = last_pos;
424 else
425 i = 0;
426
427 for (; i < hdr->h.tch.entry_count; i++)
428 {
429 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
430 {
431 last_pos = MAX(0, i - 3);
432 return i;
433 }
434
435 do_timed_yield();
436 }
437
438 if (last_pos > 0)
439 {
440 last_pos = 0;
441 goto try_again;
442 }
443
444 return -1;
445 }
446 #endif
447
448 static long find_entry_disk(const char *filename)
449 {
450 struct tagcache_header tch;
451 static long last_pos = -1;
452 long pos_history[POS_HISTORY_COUNT];
453 long pos_history_idx = 0;
454 bool found = false;
455 struct tagfile_entry tfe;
456 int fd;
457 char buf[TAG_MAXLEN+32];
458 int i;
459 int pos = -1;
460
461 if (!tc_stat.ready)
462 return -2;
463
464 fd = filenametag_fd;
465 if (fd < 0)
466 {
467 last_pos = -1;
468 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
469 return -1;
470 }
471
472 check_again:
473
474 if (last_pos > 0)
475 lseek(fd, last_pos, SEEK_SET);
476 else
477 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
478
479 while (true)
480 {
481 pos = lseek(fd, 0, SEEK_CUR);
482 for (i = pos_history_idx-1; i >= 0; i--)
483 pos_history[i+1] = pos_history[i];
484 pos_history[0] = pos;
485
486 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
487 != sizeof(struct tagfile_entry))
488 {
489 break ;
490 }
491
492 if (tfe.tag_length >= (long)sizeof(buf))
493 {
494 logf("too long tag #1");
495 close(fd);
496 last_pos = -1;
497 return -2;
498 }
499
500 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
501 {
502 logf("read error #2");
503 close(fd);
504 last_pos = -1;
505 return -3;
506 }
507
508 if (!strcasecmp(filename, buf))
509 {
510 last_pos = pos_history[pos_history_idx];
511 found = true;
512 break ;
513 }
514
515 if (pos_history_idx < POS_HISTORY_COUNT - 1)
516 pos_history_idx++;
517 }
518
519 /* Not found? */
520 if (!found)
521 {
522 if (last_pos > 0)
523 {
524 last_pos = -1;
525 logf("seek again");
526 goto check_again;
527 }
528
529 if (fd != filenametag_fd)
530 close(fd);
531 return -4;
532 }
533
534 if (fd != filenametag_fd)
535 close(fd);
536
537 return tfe.idx_id;
538 }
539
540 static int find_index(const char *filename)
541 {
542 long idx_id = -1;
543
544 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
545 if (tc_stat.ramcache && is_dircache_intact())
546 idx_id = find_entry_ram(filename, NULL);
547 #endif
548
549 if (idx_id < 0)
550 idx_id = find_entry_disk(filename);
551
552 return idx_id;
553 }
554
555 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
556 {
557 int idx_id;
558
559 if (!tc_stat.ready)
560 return false;
561
562 idx_id = find_index(filename);
563 if (idx_id < 0)
564 return false;
565
566 if (!tagcache_search(tcs, tag_filename))
567 return false;
568
569 tcs->entry_count = 0;
570 tcs->idx_id = idx_id;
571
572 return true;
573 }
574
575 static bool get_index(int masterfd, int idxid,
576 struct index_entry *idx, bool use_ram)
577 {
578 bool localfd = false;
579
580 if (idxid < 0)
581 {
582 logf("Incorrect idxid: %d", idxid);
583 return false;
584 }
585
586 #ifdef HAVE_TC_RAMCACHE
587 if (tc_stat.ramcache && use_ram)
588 {
589 if (hdr->indices[idxid].flag & FLAG_DELETED)
590 return false;
591
592 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
593 return true;
594 }
595 #else
596 (void)use_ram;
597 #endif
598
599 if (masterfd < 0)
600 {
601 struct master_header tcmh;
602
603 localfd = true;
604 masterfd = open_master_fd(&tcmh, false);
605 if (masterfd < 0)
606 return false;
607 }
608
609 lseek(masterfd, idxid * sizeof(struct index_entry)
610 + sizeof(struct master_header), SEEK_SET);
611 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
612 != sizeof(struct index_entry))
613 {
614 logf("read error #3");
615 if (localfd)
616 close(masterfd);
617
618 return false;
619 }
620
621 if (localfd)
622 close(masterfd);
623
624 if (idx->flag & FLAG_DELETED)
625 return false;
626
627 return true;
628 }
629
630 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
631 {
632 /* We need to exclude all memory only flags & tags when writing to disk. */
633 if (idx->flag & FLAG_DIRCACHE)
634 {
635 logf("memory only flags!");
636 return false;
637 }
638
639 #ifdef HAVE_TC_RAMCACHE
640 /* Only update numeric data. Writing the whole index to RAM by memcpy
641 * destroys dircache pointers!
642 */
643 if (tc_stat.ramcache)
644 {
645 int tag;
646 struct index_entry *idx_ram = &hdr->indices[idxid];
647
648 for (tag = 0; tag < TAG_COUNT; tag++)
649 {
650 if (tagcache_is_numeric_tag(tag))
651 {
652 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
653 }
654 }
655
656 /* Don't touch the dircache flag. */
657 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
658 }
659 #endif
660
661 lseek(masterfd, idxid * sizeof(struct index_entry)
662 + sizeof(struct master_header), SEEK_SET);
663 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
664 != sizeof(struct index_entry))
665 {
666 logf("write error #3");
667 logf("idxid: %d", idxid);
668 return false;
669 }
670
671 return true;
672 }
673
674 static bool open_files(struct tagcache_search *tcs, int tag)
675 {
676 if (tcs->idxfd[tag] < 0)
677 {
678 char fn[MAX_PATH];
679
680 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
681 tcs->idxfd[tag] = open(fn, O_RDONLY);
682 }
683
684 if (tcs->idxfd[tag] < 0)
685 {
686 logf("File not open!");
687 return false;
688 }
689
690 return true;
691 }
692
693 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
694 int tag, char *buf, long size)
695 {
696 struct tagfile_entry tfe;
697 long seek;
698
699 *buf = '\0';
700
701 if (tagcache_is_numeric_tag(tag))
702 return false;
703
704 seek = idx->tag_seek[tag];
705 if (seek < 0)
706 {
707 logf("Retrieve failed");
708 return false;
709 }
710
711 #ifdef HAVE_TC_RAMCACHE
712 if (tcs->ramsearch)
713 {
714 struct tagfile_entry *ep;
715
716 # ifdef HAVE_DIRCACHE
717 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
718 && is_dircache_intact())
719 {
720 dircache_copy_path((struct dirent *)seek,
721 buf, size);
722 return true;
723 }
724 else
725 # endif
726 if (tag != tag_filename)
727 {
728 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
729 strncpy(buf, ep->tag_data, size-1);
730
731 return true;
732 }
733 }
734 #endif
735
736 if (!open_files(tcs, tag))
737 return false;
738
739 lseek(tcs->idxfd[tag], seek, SEEK_SET);
740 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
741 != sizeof(struct tagfile_entry))
742 {
743 logf("read error #5");
744 return false;
745 }
746
747 if (tfe.tag_length >= size)
748 {
749 logf("too small buffer");
750 return false;
751 }
752
753 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
754 tfe.tag_length)
755 {
756 logf("read error #6");
757 return false;
758 }
759
760 buf[tfe.tag_length] = '\0';
761
762 return true;
763 }
764
765 static long check_virtual_tags(int tag, const struct index_entry *idx)
766 {
767 long data = 0;
768
769 switch (tag)
770 {
771 case tag_virt_length_sec:
772 data = (idx->tag_seek[tag_length]/1000) % 60;
773 break;
774
775 case tag_virt_length_min:
776 data = (idx->tag_seek[tag_length]/1000) / 60;
777 break;
778
779 case tag_virt_playtime_sec:
780 data = (idx->tag_seek[tag_playtime]/1000) % 60;
781 break;
782
783 case tag_virt_playtime_min:
784 data = (idx->tag_seek[tag_playtime]/1000) / 60;
785 break;
786
787 case tag_virt_autoscore:
788 if (idx->tag_seek[tag_length] == 0
789 || idx->tag_seek[tag_playcount] == 0)
790 {
791 data = 0;
792 }
793 else
794 {
795 data = 100 * idx->tag_seek[tag_playtime]
796 / idx->tag_seek[tag_length]
797 / idx->tag_seek[tag_playcount];
798 }
799 break;
800
801 /* How many commits before the file has been added to the DB. */
802 case tag_virt_entryage:
803 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
804 break;
805
806 default:
807 data = idx->tag_seek[tag];
808 }
809
810 return data;
811 }
812
813 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
814 {
815 struct index_entry idx;
816
817 if (!tc_stat.ready)
818 return false;
819
820 if (!tagcache_is_numeric_tag(tag))
821 return -1;
822
823 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
824 return -2;
825
826 return check_virtual_tags(tag, &idx);
827 }
828
829 inline static bool str_ends_with(const char *str1, const char *str2)
830 {
831 int str_len = strlen(str1);
832 int clause_len = strlen(str2);
833
834 if (clause_len > str_len)
835 return false;
836
837 return !strcasecmp(&str1[str_len - clause_len], str2);
838 }
839
840 inline static bool str_oneof(const char *str, const char *list)
841 {
842 const char *sep;
843 int l, len = strlen(str);
844
845 while (*list)
846 {
847 sep = strchr(list, '|');
848 l = sep ? (long)sep - (long)list : (int)strlen(list);
849 if ((l==len) && !strncasecmp(str, list, len))
850 return true;
851 list += sep ? l + 1 : l;
852 }
853
854 return false;
855 }
856
857 static bool check_against_clause(long numeric, const char *str,
858 const struct tagcache_search_clause *clause)
859 {
860 if (clause->numeric)
861 {
862 switch (clause->type)
863 {
864 case clause_is:
865 return numeric == clause->numeric_data;
866 case clause_is_not:
867 return numeric != clause->numeric_data;
868 case clause_gt:
869 return numeric > clause->numeric_data;
870 case clause_gteq:
871 return numeric >= clause->numeric_data;
872 case clause_lt:
873 return numeric < clause->numeric_data;
874 case clause_lteq:
875 return numeric <= clause->numeric_data;
876 default:
877 logf("Incorrect numeric tag: %d", clause->type);
878 }
879 }
880 else
881 {
882 switch (clause->type)
883 {
884 case clause_is:
885 return !strcasecmp(clause->str, str);
886 case clause_is_not:
887 return strcasecmp(clause->str, str);
888 case clause_gt:
889 return 0>strcasecmp(clause->str, str);
890 case clause_gteq:
891 return 0>=strcasecmp(clause->str, str);
892 case clause_lt:
893 return 0<strcasecmp(clause->str, str);
894 case clause_lteq:
895 return 0<=strcasecmp(clause->str, str);
896 case clause_contains:
897 return (strcasestr(str, clause->str) != NULL);
898 case clause_not_contains:
899 return (strcasestr(str, clause->str) == NULL);
900 case clause_begins_with:
901 return (strcasestr(str, clause->str) == str);
902 case clause_not_begins_with:
903 return (strcasestr(str, clause->str) != str);
904 case clause_ends_with:
905 return str_ends_with(str, clause->str);
906 case clause_not_ends_with:
907 return !str_ends_with(str, clause->str);
908 case clause_oneof:
909 return str_oneof(str, clause->str);
910
911 default:
912 logf("Incorrect tag: %d", clause->type);
913 }
914 }
915
916 return false;
917 }
918
919 static bool check_clauses(struct tagcache_search *tcs,
920 struct index_entry *idx,
921 struct tagcache_search_clause **clause, int count)
922 {
923 int i;
924
925 #ifdef HAVE_TC_RAMCACHE
926 if (tcs->ramsearch)
927 {
928 /* Go through all conditional clauses. */
929 for (i = 0; i < count; i++)
930 {
931 struct tagfile_entry *tfe;
932 int seek;
933 char buf[256];
934 char *str = NULL;
935
936 seek = check_virtual_tags(clause[i]->tag, idx);
937
938 if (!tagcache_is_numeric_tag(clause[i]->tag))
939 {
940 if (clause[i]->tag == tag_filename)
941 {
942 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
943 str = buf;
944 }
945 else
946 {
947 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
948 str = tfe->tag_data;
949 }
950 }
951
952 if (!check_against_clause(seek, str, clause[i]))
953 return false;
954 }
955 }
956 else
957 #endif
958 {
959 /* Check for conditions. */
960 for (i = 0; i < count; i++)
961 {
962 struct tagfile_entry tfe;
963 int seek;
964 char str[256];
965
966 seek = check_virtual_tags(clause[i]->tag, idx);
967
968 memset(str, 0, sizeof str);
969 if (!tagcache_is_numeric_tag(clause[i]->tag))
970 {
971 int fd = tcs->idxfd[clause[i]->tag];
972 lseek(fd, seek, SEEK_SET);
973 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
974 if (tfe.tag_length >= (int)sizeof(str))
975 {
976 logf("Too long tag read!");
977 break ;
978 }
979
980 read(fd, str, tfe.tag_length);
981
982 /* Check if entry has been deleted. */
983 if (str[0] == '\0')
984 break;
985 }
986
987 if (!check_against_clause(seek, str, clause[i]))
988 return false;
989 }
990 }
991
992 return true;
993 }
994
995 bool tagcache_check_clauses(struct tagcache_search *tcs,
996 struct tagcache_search_clause **clause, int count)
997 {
998 struct index_entry idx;
999
1000 if (count == 0)
1001 return true;
1002
1003 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1004 return false;
1005
1006 return check_clauses(tcs, &idx, clause, count);
1007 }
1008
1009 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1010 {
1011 int i;
1012
1013 /* If uniq buffer is not defined we must return true for search to work. */
1014 if (tcs->unique_list == NULL
1015 || (!tagcache_is_unique_tag(tcs->type)
1016 && !tagcache_is_numeric_tag(tcs->type)))
1017 {
1018 return true;
1019 }
1020
1021 for (i = 0; i < tcs->unique_list_count; i++)
1022 {
1023 /* Return false if entry is found. */
1024 if (tcs->unique_list[i] == id)
1025 return false;
1026 }
1027
1028 if (tcs->unique_list_count < tcs->unique_list_capacity)
1029 {
1030 tcs->unique_list[i] = id;
1031 tcs->unique_list_count++;
1032 }
1033
1034 return true;
1035 }
1036
1037 static bool build_lookup_list(struct tagcache_search *tcs)
1038 {
1039 struct index_entry entry;
1040 int i;
1041
1042 tcs->seek_list_count = 0;
1043
1044 #ifdef HAVE_TC_RAMCACHE
1045 if (tcs->ramsearch)
1046 {
1047 int j;
1048
1049 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1050 {
1051 struct index_entry *idx = &hdr->indices[i];
1052 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1053 break ;
1054
1055 /* Skip deleted files. */
1056 if (idx->flag & FLAG_DELETED)
1057 continue;
1058
1059 /* Go through all filters.. */
1060 for (j = 0; j < tcs->filter_count; j++)
1061 {
1062 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1063 {
1064 break ;
1065 }
1066 }
1067
1068 if (j < tcs->filter_count)
1069 continue ;
1070
1071 /* Check for conditions. */
1072 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1073 continue;
1074
1075 /* Add to the seek list if not already in uniq buffer. */
1076 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1077 continue;
1078
1079 /* Lets add it. */
1080 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1081 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1082 tcs->seek_list_count++;
1083 }
1084
1085 tcs->seek_pos = i;
1086
1087 return tcs->seek_list_count > 0;
1088 }
1089 #endif
1090
1091 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1092 sizeof(struct master_header), SEEK_SET);
1093
1094 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1095 == sizeof(struct index_entry))
1096 {
1097 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1098 break ;
1099
1100 tcs->seek_pos++;
1101
1102 /* Check if entry has been deleted. */
1103 if (entry.flag & FLAG_DELETED)
1104 continue;
1105
1106 /* Go through all filters.. */
1107 for (i = 0; i < tcs->filter_count; i++)
1108 {
1109 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1110 break ;
1111 }
1112
1113 if (i < tcs->filter_count)
1114 continue ;
1115
1116 /* Check for conditions. */
1117 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1118 continue;
1119
1120 /* Add to the seek list if not already in uniq buffer. */
1121 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1122 continue;
1123
1124 /* Lets add it. */
1125 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1126 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1127 tcs->seek_list_count++;
1128
1129 yield();
1130 }
1131
1132 return tcs->seek_list_count > 0;
1133 }
1134
1135
1136 static void remove_files(void)
1137 {
1138 int i;
1139 char buf[MAX_PATH];
1140
1141 tc_stat.ready = false;
1142 tc_stat.ramcache = false;
1143 tc_stat.econ = false;
1144 remove(TAGCACHE_FILE_MASTER);
1145 for (i = 0; i < TAG_COUNT; i++)
1146 {
1147 if (tagcache_is_numeric_tag(i))
1148 continue;
1149
1150 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1151 remove(buf);
1152 }
1153 }
1154
1155
1156 static bool check_all_headers(void)
1157 {
1158 struct master_header myhdr;
1159 struct tagcache_header tch;
1160 int tag;
1161 int fd;
1162
1163 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1164 return false;
1165
1166 close(fd);
1167 if (myhdr.dirty)
1168 {
1169 logf("tagcache is dirty!");
1170 return false;
1171 }
1172
1173 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1174
1175 for (tag = 0; tag < TAG_COUNT; tag++)
1176 {
1177 if (tagcache_is_numeric_tag(tag))
1178 continue;
1179
1180 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1181 return false;
1182
1183 close(fd);
1184 }
1185
1186 return true;
1187 }
1188
1189 bool tagcache_search(struct tagcache_search *tcs, int tag)
1190 {
1191 struct tagcache_header tag_hdr;
1192 struct master_header master_hdr;
1193 int i;
1194
1195 if (tcs->initialized)
1196 tagcache_search_finish(tcs);
1197
1198 while (read_lock)
1199 sleep(1);
1200
1201 memset(tcs, 0, sizeof(struct tagcache_search));
1202 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1203 return false;
1204
1205 tcs->position = sizeof(struct tagcache_header);
1206 tcs->type = tag;
1207 tcs->seek_pos = 0;
1208 tcs->seek_list_count = 0;
1209 tcs->filter_count = 0;
1210 tcs->masterfd = -1;
1211
1212 for (i = 0; i < TAG_COUNT; i++)
1213 tcs->idxfd[i] = -1;
1214
1215 #ifndef HAVE_TC_RAMCACHE
1216 tcs->ramsearch = false;
1217 #else
1218 tcs->ramsearch = tc_stat.ramcache;
1219 if (tcs->ramsearch)
1220 {
1221 tcs->entry_count = hdr->entry_count[tcs->type];
1222 }
1223 else
1224 #endif
1225 {
1226 if (!tagcache_is_numeric_tag(tcs->type))
1227 {
1228 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1229 if (tcs->idxfd[tcs->type] < 0)
1230 return false;
1231 }
1232
1233 /* Always open as R/W so we can pass tcs to functions that modify data also
1234 * without failing. */
1235 tcs->masterfd = open_master_fd(&master_hdr, true);
1236
1237 if (tcs->masterfd < 0)
1238 return false;
1239 }
1240
1241 tcs->valid = true;
1242 tcs->initialized = true;
1243 write_lock++;
1244
1245 return true;
1246 }
1247
1248 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1249 void *buffer, long length)
1250 {
1251 tcs->unique_list = (unsigned long *)buffer;
1252 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1253 tcs->unique_list_count = 0;
1254 }
1255
1256 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1257 int tag, int seek)
1258 {
1259 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1260 return false;
1261
1262 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1263 return false;
1264
1265 tcs->filter_tag[tcs->filter_count] = tag;
1266 tcs->filter_seek[tcs->filter_count] = seek;
1267 tcs->filter_count++;
1268
1269 return true;
1270 }
1271
1272 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1273 struct tagcache_search_clause *clause)
1274 {
1275 int i;
1276
1277 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1278 {
1279 logf("Too many clauses");
1280 return false;
1281 }
1282
1283 /* Check if there is already a similar filter in present (filters are
1284 * much faster than clauses).
1285 */
1286 for (i = 0; i < tcs->filter_count; i++)
1287 {
1288 if (tcs->filter_tag[i] == clause->tag)
1289 return true;
1290 }
1291
1292 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1293 {
1294 char buf[MAX_PATH];
1295
1296 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1297 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1298 }
1299
1300 tcs->clause[tcs->clause_count] = clause;
1301 tcs->clause_count++;
1302
1303 return true;
1304 }
1305
1306 /* TODO: Remove this mess. */
1307 #ifdef HAVE_DIRCACHE
1308 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1309 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1310 #else
1311 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1312 #endif
1313
1314 static bool get_next(struct tagcache_search *tcs)
1315 {
1316 static char buf[TAG_MAXLEN+32];
1317 struct tagfile_entry entry;
1318 long flag = 0;
1319
1320 if (!tcs->valid || !tc_stat.ready)
1321 return false;
1322
1323 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1324 #ifdef HAVE_TC_RAMCACHE
1325 && !tcs->ramsearch
1326 #endif
1327 )
1328 return false;
1329
1330 /* Relative fetch. */
1331 if (tcs->filter_count > 0 || tcs->clause_count > 0
1332 || tagcache_is_numeric_tag(tcs->type))
1333 {
1334 /* Check for end of list. */
1335 if (tcs->seek_list_count == 0)
1336 {
1337 /* Try to fetch more. */
1338 if (!build_lookup_list(tcs))
1339 {
1340 tcs->valid = false;
1341 return false;
1342 }
1343 }
1344
1345 tcs->seek_list_count--;
1346 flag = tcs->seek_flags[tcs->seek_list_count];
1347
1348 /* Seek stream to the correct position and continue to direct fetch. */
1349 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1350 && !tagcache_is_numeric_tag(tcs->type))
1351 {
1352 if (!open_files(tcs, tcs->type))
1353 return false;
1354
1355 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1356 }
1357 else
1358 tcs->position = tcs->seek_list[tcs->seek_list_count];
1359 }
1360
1361 if (tagcache_is_numeric_tag(tcs->type))
1362 {
1363 snprintf(buf, sizeof(buf), "%d", tcs->position);
1364 tcs->result_seek = tcs->position;
1365 tcs->result = buf;
1366 tcs->result_len = strlen(buf) + 1;
1367 return true;
1368 }
1369
1370 /* Direct fetch. */
1371 #ifdef HAVE_TC_RAMCACHE
1372 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1373 {
1374 struct tagfile_entry *ep;
1375
1376 if (tcs->entry_count == 0)
1377 {
1378 tcs->valid = false;
1379 return false;
1380 }
1381 tcs->entry_count--;
1382
1383 tcs->result_seek = tcs->position;
1384
1385 # ifdef HAVE_DIRCACHE
1386 if (tcs->type == tag_filename)
1387 {
1388 dircache_copy_path((struct dirent *)tcs->position,
1389 buf, sizeof buf);
1390 tcs->result = buf;
1391 tcs->result_len = strlen(buf) + 1;
1392 tcs->idx_id = FLAG_GET_ATTR(flag);
1393 tcs->ramresult = false;
1394
1395 return true;
1396 }
1397 # endif
1398
1399 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1400 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1401 tcs->result = ep->tag_data;
1402 tcs->result_len = strlen(tcs->result) + 1;
1403 tcs->idx_id = ep->idx_id;
1404 tcs->ramresult = true;
1405
1406 return true;
1407 }
1408 else
1409 #endif
1410 {
1411 if (!open_files(tcs, tcs->type))
1412 return false;
1413
1414 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1415 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1416 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1417 {
1418 /* End of data. */
1419 tcs->valid = false;
1420 return false;
1421 }
1422 }
1423
1424 if (entry.tag_length > (long)sizeof(buf))
1425 {
1426 tcs->valid = false;
1427 logf("too long tag #2");
1428 return false;
1429 }
1430
1431 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1432 {
1433 tcs->valid = false;
1434 logf("read error #4");
1435 return false;
1436 }
1437
1438 tcs->result = buf;
1439 tcs->result_len = strlen(tcs->result) + 1;
1440 tcs->idx_id = entry.idx_id;
1441 tcs->ramresult = false;
1442
1443 return true;
1444 }
1445
1446 bool tagcache_get_next(struct tagcache_search *tcs)
1447 {
1448 while (get_next(tcs))
1449 {
1450 if (tcs->result_len > 1)
1451 return true;
1452 }
1453
1454 return false;
1455 }
1456
1457 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1458 int tag, char *buf, long size)
1459 {
1460 struct index_entry idx;
1461
1462 *buf = '\0';
1463 if (!get_index(tcs->masterfd, idxid, &idx, true))
1464 return false;
1465
1466 return retrieve(tcs, &idx, tag, buf, size);
1467 }
1468
1469 static bool update_master_header(void)
1470 {
1471 struct master_header myhdr;
1472 int fd;
1473
1474 if (!tc_stat.ready)
1475 return false;
1476
1477 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1478 return false;
1479
1480 myhdr.serial = current_tcmh.serial;
1481 myhdr.commitid = current_tcmh.commitid;
1482
1483 /* Write it back */
1484 lseek(fd, 0, SEEK_SET);
1485 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1486 close(fd);
1487
1488 #ifdef HAVE_TC_RAMCACHE
1489 if (hdr)
1490 {
1491 hdr->h.serial = current_tcmh.serial;
1492 hdr->h.commitid = current_tcmh.commitid;
1493 }
1494 #endif
1495
1496 return true;
1497 }
1498
1499 #if 0
1500
1501 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1502 {
1503 struct tagentry *entry;
1504
1505 if (tcs->type != tag_title)
1506 return ;
1507
1508 /* We will need reserve buffer for this. */
1509 if (tcs->ramcache)
1510 {
1511 struct tagfile_entry *ep;
1512
1513 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1514 tcs->seek_list[tcs->seek_list_count];
1515 }
1516
1517 entry = find_entry_ram();
1518
1519 }
1520 #endif
1521
1522 void tagcache_search_finish(struct tagcache_search *tcs)
1523 {
1524 int i;
1525
1526 if (!tcs->initialized)
1527 return;
1528
1529 if (tcs->masterfd >= 0)
1530 {
1531 close(tcs->masterfd);
1532 tcs->masterfd = -1;
1533 }
1534
1535 for (i = 0; i < TAG_COUNT; i++)
1536 {
1537 if (tcs->idxfd[i] >= 0)
1538 {
1539 close(tcs->idxfd[i]);
1540 tcs->idxfd[i] = -1;
1541 }
1542 }
1543
1544 tcs->ramsearch = false;
1545 tcs->valid = false;
1546 tcs->initialized = 0;
1547 if (write_lock > 0)
1548 write_lock--;
1549 }
1550
1551 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1552 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1553 {
1554 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1555 }
1556
1557 static long get_tag_numeric(const struct index_entry *entry, int tag)
1558 {
1559 return check_virtual_tags(tag, entry);
1560 }
1561
1562 static char* get_tag_string(const struct index_entry *entry, int tag)
1563 {
1564 char* s = get_tag(entry, tag)->tag_data;
1565 return strcmp(s, UNTAGGED) ? s : NULL;
1566 }
1567
1568 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1569 {
1570 struct index_entry *entry;
1571 int idx_id;
1572
1573 if (!tc_stat.ready)
1574 return false;
1575
1576 /* Find the corresponding entry in tagcache. */
1577 idx_id = find_entry_ram(filename, NULL);
1578 if (idx_id < 0 || !tc_stat.ramcache)
1579 return false;
1580
1581 entry = &hdr->indices[idx_id];
1582
1583 id3->title = get_tag_string(entry, tag_title);
1584 id3->artist = get_tag_string(entry, tag_artist);
1585 id3->album = get_tag_string(entry, tag_album);
1586 id3->genre_string = get_tag_string(entry, tag_genre);
1587 id3->composer = get_tag_string(entry, tag_composer);
1588 id3->comment = get_tag_string(entry, tag_comment);
1589 id3->albumartist = get_tag_string(entry, tag_albumartist);
1590 id3->grouping = get_tag_string(entry, tag_grouping);
1591
1592 id3->playcount = get_tag_numeric(entry, tag_playcount);
1593 id3->rating = get_tag_numeric(entry, tag_rating);
1594 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1595 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1596 id3->year = get_tag_numeric(entry, tag_year);
1597
1598 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1599 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1600 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1601 if (id3->bitrate == 0)
1602 id3->bitrate = 1;
1603
1604 return true;
1605 }
1606 #endif
1607
1608 static inline void write_item(const char *item)
1609 {
1610 int len = strlen(item) + 1;
1611
1612 data_size += len;
1613 write(cachefd, item, len);
1614 }
1615
1616 static int check_if_empty(char **tag)
1617 {
1618 int length;
1619
1620 if (*tag == NULL || **tag == '\0')
1621 {
1622 *tag = UNTAGGED;
1623 return sizeof(UNTAGGED); /* Tag length */
1624 }
1625
1626 length = strlen(*tag);
1627 if (length > TAG_MAXLEN)
1628 {
1629 logf("over length tag: %s", *tag);
1630 length = TAG_MAXLEN;
1631 (*tag)[length] = '\0';
1632 }
1633
1634 return length + 1;
1635 }
1636
1637 #define ADD_TAG(entry,tag,data) \
1638 /* Adding tag */ \
1639 entry.tag_offset[tag] = offset; \
1640 entry.tag_length[tag] = check_if_empty(data); \
1641 offset += entry.tag_length[tag]
1642
1643 static void add_tagcache(char *path, unsigned long mtime
1644 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1645 ,const struct dirent *dc
1646 #endif
1647 )
1648 {
1649 struct mp3entry id3;
1650 struct temp_file_entry entry;
1651 bool ret;
1652 int fd;
1653 int idx_id = -1;
1654 char tracknumfix[3];
1655 int offset = 0;
1656 int path_length = strlen(path);
1657 bool has_albumartist;
1658 bool has_grouping;
1659
1660 if (cachefd < 0)
1661 return ;
1662
1663 /* Check for overlength file path. */
1664 if (path_length > TAG_MAXLEN)
1665 {
1666 /* Path can't be shortened. */
1667 logf("Too long path: %s", path);
1668 return ;
1669 }
1670
1671 /* Check if the file is supported. */
1672 if (probe_file_format(path) == AFMT_UNKNOWN)
1673 return ;
1674
1675 /* Check if the file is already cached. */
1676 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1677 if (tc_stat.ramcache && is_dircache_intact())
1678 {
1679 idx_id = find_entry_ram(path, dc);
1680 }
1681 else
1682 #endif
1683 {
1684 if (filenametag_fd >= 0)
1685 {
1686 idx_id = find_entry_disk(path);
1687 }
1688 }
1689
1690 /* Check if file has been modified. */
1691 if (idx_id >= 0)
1692 {
1693 struct index_entry idx;
1694
1695 if (!get_index(-1, idx_id, &idx, true))
1696 {
1697 logf("failed to retrieve index entry");
1698 return ;
1699 }
1700
1701 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1702 {
1703 /* No changes to file. */
1704 return ;
1705 }
1706
1707 /* Metadata might have been changed. Delete the entry. */
1708 logf("Re-adding: %s", path);
1709 if (!delete_entry(idx_id))
1710 {
1711 logf("delete_entry failed: %d", idx_id);
1712 return ;
1713 }
1714 }
1715
1716 fd = open(path, O_RDONLY);
1717 if (fd < 0)
1718 {
1719 logf("open fail: %s", path);
1720 return ;
1721 }
1722
1723 memset(&id3, 0, sizeof(struct mp3entry));
1724 memset(&entry, 0, sizeof(struct temp_file_entry));
1725 memset(&tracknumfix, 0, sizeof(tracknumfix));
1726 ret = get_metadata(&id3, fd, path);
1727 close(fd);
1728
1729 if (!ret)
1730 return ;
1731
1732 logf("-> %s", path);
1733
1734 /* Generate track number if missing. */
1735 if (id3.tracknum <= 0)
1736 {
1737 const char *p = strrchr(path, '.');
1738
1739 if (p == NULL)
1740 p = &path[strlen(path)-1];
1741
1742 while (*p != '/')
1743 {
1744 if (isdigit(*p) && isdigit(*(p-1)))
1745 {
1746 tracknumfix[1] = *p--;
1747 tracknumfix[0] = *p;
1748 break;
1749 }
1750 p--;
1751 }
1752
1753 if (tracknumfix[0] != '\0')
1754 {
1755 id3.tracknum = atoi(tracknumfix);
1756 /* Set a flag to indicate track number has been generated. */
1757 entry.flag |= FLAG_TRKNUMGEN;
1758 }
1759 else
1760 {
1761 /* Unable to generate track number. */
1762 id3.tracknum = -1;
1763 }
1764 }
1765
1766 /* Numeric tags */
1767 entry.tag_offset[tag_year] = id3.year;
1768 entry.tag_offset[tag_discnumber] = id3.discnum;
1769 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1770 entry.tag_offset[tag_length] = id3.length;
1771 entry.tag_offset[tag_bitrate] = id3.bitrate;
1772 entry.tag_offset[tag_mtime] = mtime;
1773
1774 /* String tags. */
1775 has_albumartist = id3.albumartist != NULL
1776 && strlen(id3.albumartist) > 0;
1777 has_grouping = id3.grouping != NULL
1778 && strlen(id3.grouping) > 0;
1779
1780 ADD_TAG(entry, tag_filename, &path);
1781 ADD_TAG(entry, tag_title, &id3.title);
1782 ADD_TAG(entry, tag_artist, &id3.artist);
1783 ADD_TAG(entry, tag_album, &id3.album);
1784 ADD_TAG(entry, tag_genre, &id3.genre_string);
1785 ADD_TAG(entry, tag_composer, &id3.composer);
1786 ADD_TAG(entry, tag_comment, &id3.comment);
1787 if (has_albumartist)
1788 {
1789 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1790 }
1791 else
1792 {
1793 ADD_TAG(entry, tag_albumartist, &id3.artist);
1794 }
1795 if (has_grouping)
1796 {
1797 ADD_TAG(entry, tag_grouping, &id3.grouping);
1798 }
1799 else
1800 {
1801 ADD_TAG(entry, tag_grouping, &id3.title);
1802 }
1803 entry.data_length = offset;
1804
1805 /* Write the header */
1806 write(cachefd, &entry, sizeof(struct temp_file_entry));
1807
1808 /* And tags also... Correct order is critical */
1809 write_item(path);
1810 write_item(id3.title);
1811 write_item(id3.artist);
1812 write_item(id3.album);
1813 write_item(id3.genre_string);
1814 write_item(id3.composer);
1815 write_item(id3.comment);
1816 if (has_albumartist)
1817 {
1818 write_item(id3.albumartist);
1819 }
1820 else
1821 {
1822 write_item(id3.artist);
1823 }
1824 if (has_grouping)
1825 {
1826 write_item(id3.grouping);
1827 }
1828 else
1829 {
1830 write_item(id3.title);
1831 }
1832 total_entry_count++;
1833 }
1834
1835 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1836 {
1837 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1838 int len = strlen(str)+1;
1839 int i;
1840 unsigned crc32;
1841 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1842 char buf[TAG_MAXLEN+32];
1843
1844 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1845 buf[i] = tolower(str[i]);
1846 buf[i] = '\0';
1847
1848 crc32 = crc_32(buf, i, 0xffffffff);
1849
1850 if (unique)
1851 {
1852 /* Check if the crc does not exist -> entry does not exist for sure. */
1853 for (i = 0; i < tempbufidx; i++)
1854 {
1855 if (crcbuf[-i] != crc32)
1856 continue;
1857
1858 if (!strcasecmp(str, index[i].str))
1859 {
1860 if (id < 0 || id >= lookup_buffer_depth)
1861 {
1862 logf("lookup buf overf.: %d", id);
1863 return false;
1864 }
1865
1866 lookup[id] = &index[i];
1867 return true;
1868 }
1869 }
1870 }
1871
1872 /* Insert to CRC buffer. */
1873 crcbuf[-tempbufidx] = crc32;
1874 tempbuf_left -= 4;
1875
1876 /* Insert it to the buffer. */
1877 tempbuf_left -= len;
1878 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1879 return false;
1880
1881 if (id >= lookup_buffer_depth)
1882 {
1883 logf("lookup buf overf. #2: %d", id);
1884 return false;
1885 }
1886
1887 if (id >= 0)
1888 {
1889 lookup[id] = &index[tempbufidx];
1890 index[tempbufidx].idlist.id = id;
1891 }
1892 else
1893 index[tempbufidx].idlist.id = -1;
1894
1895 index[tempbufidx].idlist.next = NULL;
1896 index[tempbufidx].idx_id = idx_id;
1897 index[tempbufidx].seek = -1;
1898 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1899 memcpy(index[tempbufidx].str, str, len);
1900 tempbuf_pos += len;
1901 tempbufidx++;
1902
1903 return true;
1904 }
1905
1906 static int compare(const void *p1, const void *p2)
1907 {
1908 do_timed_yield();
1909
1910 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1911 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1912
1913 if (strcmp(e1->str, UNTAGGED) == 0)
1914 {
1915 if (strcmp(e2->str, UNTAGGED) == 0)
1916 return 0;
1917 return -1;
1918 }
1919 else if (strcmp(e2->str, UNTAGGED) == 0)
1920 return 1;
1921
1922 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1923 }
1924
1925 static int tempbuf_sort(int fd)
1926 {
1927 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1928 struct tagfile_entry fe;
1929 int i;
1930 int length;
1931
1932 /* Generate reverse lookup entries. */
1933 for (i = 0; i < lookup_buffer_depth; i++)
1934 {
1935 struct tempbuf_id_list *idlist;
1936
1937 if (!lookup[i])
1938 continue;
1939
1940 if (lookup[i]->idlist.id == i)
1941 continue;
1942
1943 idlist = &lookup[i]->idlist;
1944 while (idlist->next != NULL)
1945 idlist = idlist->next;
1946
1947 tempbuf_left -= sizeof(struct tempbuf_id_list);
1948 if (tempbuf_left - 4 < 0)
1949 return -1;
1950
1951 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1952 if (tempbuf_pos & 0x03)
1953 {
1954 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1955 tempbuf_left -= 3;
1956 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1957 }
1958 tempbuf_pos += sizeof(struct tempbuf_id_list);
1959
1960 idlist = idlist->next;
1961 idlist->id = i;
1962 idlist->next = NULL;
1963
1964 do_timed_yield();
1965 }
1966
1967 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1968 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1969
1970 for (i = 0; i < tempbufidx; i++)
1971 {
1972 struct tempbuf_id_list *idlist = &index[i].idlist;
1973
1974 /* Fix the lookup list. */
1975 while (idlist != NULL)
1976 {
1977 if (idlist->id >= 0)
1978 lookup[idlist->id] = &index[i];
1979 idlist = idlist->next;
1980 }
1981
1982 index[i].seek = lseek(fd, 0, SEEK_CUR);
1983 length = strlen(index[i].str) + 1;
1984 fe.tag_length = length;
1985 fe.idx_id = index[i].idx_id;
1986
1987 /* Check the chunk alignment. */
1988 if ((fe.tag_length + sizeof(struct tagfile_entry))
1989 % TAGFILE_ENTRY_CHUNK_LENGTH)
1990 {
1991 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1992 ((fe.tag_length + sizeof(struct tagfile_entry))
1993 % TAGFILE_ENTRY_CHUNK_LENGTH);
1994 }
1995
1996 #ifdef TAGCACHE_STRICT_ALIGN
1997 /* Make sure the entry is long aligned. */
1998 if (index[i].seek & 0x03)
1999 {
2000 logf("tempbuf_sort: alignment error!");
2001 return -3;
2002 }
2003 #endif
2004
2005 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2006 sizeof(struct tagfile_entry))
2007 {
2008 logf("tempbuf_sort: write error #1");
2009 return -1;
2010 }
2011
2012 if (write(fd, index[i].str, length) != length)
2013 {
2014 logf("tempbuf_sort: write error #2");
2015 return -2;
2016 }
2017
2018 /* Write some padding. */
2019 if (fe.tag_length - length > 0)
2020 write(fd, "XXXXXXXX", fe.tag_length - length);
2021 }
2022
2023 return i;
2024 }
2025
2026 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2027 {
2028 if (id < 0 || id >= lookup_buffer_depth)
2029 return NULL;
2030
2031 return lookup[id];
2032 }
2033
2034
2035 inline static int tempbuf_find_location(int id)
2036 {
2037 struct tempbuf_searchidx *entry;
2038
2039 entry = tempbuf_locate(id);
2040 if (entry == NULL)
2041 return -1;
2042
2043 return entry->seek;
2044 }
2045
2046 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2047 {
2048 struct master_header tcmh;
2049 struct index_entry idx;
2050 int masterfd;
2051 int masterfd_pos;
2052 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2053 int max_entries;
2054 int entries_processed = 0;
2055 int i, j;
2056 char buf[TAG_MAXLEN];
2057
2058 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2059
2060 logf("Building numeric indices...");
2061 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2062
2063 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2064 return false;
2065
2066 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2067 SEEK_CUR);
2068 if (masterfd_pos == filesize(masterfd))
2069 {
2070 logf("we can't append!");
2071 close(masterfd);
2072 return false;
2073 }
2074
2075 while (entries_processed < h->entry_count)
2076 {
2077 int count = MIN(h->entry_count - entries_processed, max_entries);
2078
2079 /* Read in as many entries as possible. */
2080 for (i = 0; i < count; i++)
2081 {
2082 struct temp_file_entry *tfe = &entrybuf[i];
2083 int datastart;
2084
2085 /* Read in numeric data. */
2086 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2087 sizeof(struct temp_file_entry))
2088 {
2089 logf("read fail #1");
2090 close(masterfd);
2091 return false;
2092 }
2093
2094 datastart = lseek(tmpfd, 0, SEEK_CUR);
2095
2096 /**
2097 * Read string data from the following tags:
2098 * - tag_filename
2099 * - tag_artist
2100 * - tag_album
2101 * - tag_title
2102 *
2103 * A crc32 hash is calculated from the read data
2104 * and stored back to the data offset field kept in memory.
2105 */
2106 #define tmpdb_read_string_tag(tag) \
2107 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2108 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2109 { \
2110 logf("read fail: buffer overflow"); \
2111 close(masterfd); \
2112 return false; \
2113 } \
2114 \
2115 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2116 tfe->tag_length[tag]) \
2117 { \
2118 logf("read fail #2"); \
2119 close(masterfd); \
2120 return false; \
2121 } \
2122 \
2123 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2124 lseek(tmpfd, datastart, SEEK_SET)
2125
2126 tmpdb_read_string_tag(tag_filename);
2127 tmpdb_read_string_tag(tag_artist);
2128 tmpdb_read_string_tag(tag_album);
2129 tmpdb_read_string_tag(tag_title);
2130
2131 /* Seek to the end of the string data. */
2132 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2133 }
2134
2135 /* Backup the master index position. */
2136 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2137 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2138
2139 /* Check if we can resurrect some deleted runtime statistics data. */
2140 for (i = 0; i < tcmh.tch.entry_count; i++)
2141 {
2142 /* Read the index entry. */
2143 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2144 != sizeof(struct index_entry))
2145 {
2146 logf("read fail #3");
2147 close(masterfd);
2148 return false;
2149 }
2150
2151 /**
2152 * Skip unless the entry is marked as being deleted
2153 * or the data has already been resurrected.
2154 */
2155 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2156 continue;
2157
2158 /* Now try to match the entry. */
2159 /**
2160 * To succesfully match a song, the following conditions
2161 * must apply:
2162 *
2163 * For numeric fields: tag_length
2164 * - Full identical match is required
2165 *
2166 * If tag_filename matches, no further checking necessary.
2167 *
2168 * For string hashes: tag_artist, tag_album, tag_title
2169 * - Two of these must match
2170 */
2171 for (j = 0; j < count; j++)
2172 {
2173 struct temp_file_entry *tfe = &entrybuf[j];
2174
2175 /* Try to match numeric fields first. */
2176 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2177 continue;
2178
2179 /* Now it's time to do the hash matching. */
2180 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2181 {
2182 int match_count = 0;
2183
2184 /* No filename match, check if we can match two other tags. */
2185 #define tmpdb_match(tag) \
2186 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2187 match_count++
2188
2189 tmpdb_match(tag_artist);
2190 tmpdb_match(tag_album);
2191 tmpdb_match(tag_title);
2192
2193 if (match_count < 2)
2194 {
2195 /* Still no match found, give up. */
2196 continue;
2197 }
2198 }
2199
2200 /* A match found, now copy & resurrect the statistical data. */
2201 #define tmpdb_copy_tag(tag) \
2202 tfe->tag_offset[tag] = idx.tag_seek[tag]
2203
2204 tmpdb_copy_tag(tag_playcount);
2205 tmpdb_copy_tag(tag_rating);
2206 tmpdb_copy_tag(tag_playtime);
2207 tmpdb_copy_tag(tag_lastplayed);
2208 tmpdb_copy_tag(tag_commitid);
2209
2210 /* Avoid processing this entry again. */
2211 idx.flag |= FLAG_RESURRECTED;
2212
2213 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2214 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2215 != sizeof(struct index_entry))
2216 {
2217 logf("masterfd writeback fail #1");
2218 close(masterfd);
2219 return false;
2220 }
2221
2222 logf("Entry resurrected");
2223 }
2224 }
2225
2226
2227 /* Restore the master index position. */
2228 lseek(masterfd, masterfd_pos, SEEK_SET);
2229
2230 /* Commit the data to the index. */
2231 for (i = 0; i < count; i++)
2232 {
2233 int loc = lseek(masterfd, 0, SEEK_CUR);
2234
2235 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2236 != sizeof(struct index_entry))
2237 {
2238 logf("read fail #3");
2239 close(masterfd);
2240 return false;
2241 }
2242
2243 for (j = 0; j < TAG_COUNT; j++)
2244 {
2245 if (!tagcache_is_numeric_tag(j))
2246 continue;
2247
2248 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2249 }
2250 idx.flag = entrybuf[i].flag;
2251
2252 if (idx.tag_seek[tag_commitid])
2253 {
2254 /* Data has been resurrected. */
2255 idx.flag |= FLAG_DIRTYNUM;
2256 }
2257 else if (tc_stat.ready && current_tcmh.commitid > 0)
2258 {
2259 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2260 idx.flag |= FLAG_DIRTYNUM;
2261 }
2262
2263 /* Write back the updated index. */
2264 lseek(masterfd, loc, SEEK_SET);
2265 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2266 != sizeof(struct index_entry))
2267 {
2268 logf("write fail");
2269 close(masterfd);
2270 return false;
2271 }
2272 }
2273
2274 entries_processed += count;
2275 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2276 }
2277
2278 close(masterfd);
2279
2280 return true;
2281 }
2282
2283 /**
2284 * Return values:
2285 * > 0 success
2286 * == 0 temporary failure
2287 * < 0 fatal error
2288 */
2289 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2290 {
2291 int i;
2292 struct tagcache_header tch;
2293 struct master_header tcmh;
2294 struct index_entry idxbuf[IDX_BUF_DEPTH];
2295 int idxbuf_pos;
2296 char buf[TAG_MAXLEN+32];
2297 int fd = -1, masterfd;
2298 bool error = false;
2299 int init;
2300 int masterfd_pos;
2301
2302 logf("Building index: %d", index_type);
2303
2304 /* Check the number of entries we need to allocate ram for. */
2305 commit_entry_count = h->entry_count + 1;
2306
2307 masterfd = open_master_fd(&tcmh, false);
2308 if (masterfd >= 0)
2309 {
2310 commit_entry_count += tcmh.tch.entry_count;
2311 close(masterfd);
2312 }
2313 else
2314 remove_files(); /* Just to be sure we are clean. */
2315
2316 /* Open the index file, which contains the tag names. */
2317 fd = open_tag_fd(&tch, index_type, true);
2318 if (fd >= 0)
2319 {
2320 logf("tch.datasize=%ld", tch.datasize);
2321 lookup_buffer_depth = 1 +
2322 /* First part */ commit_entry_count +
2323 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2324 }
2325 else
2326 {
2327 lookup_buffer_depth = 1 +
2328 /* First part */ commit_entry_count +
2329 /* Second part */ 0;
2330 }
2331
2332 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2333 logf("commit_entry_count=%ld", commit_entry_count);
2334
2335 /* Allocate buffer for all index entries from both old and new
2336 * tag files. */
2337 tempbufidx = 0;
2338 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2339
2340 /* Allocate lookup buffer. The first portion of commit_entry_count
2341 * contains the new tags in the temporary file and the second
2342 * part for locating entries already in the db.
2343 *
2344 * New tags Old tags
2345 * +---------+---------------------------+
2346 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2347 * +---------+---------------------------+
2348 *
2349 * Old tags are inserted to a temporary buffer with position:
2350 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2351 * And new tags with index:
2352 * tempbuf_insert(idx, ...);
2353 *
2354 * The buffer is sorted and written into tag file:
2355 * tempbuf_sort(...);
2356 * leaving master index locations messed up.
2357 *
2358 * That is fixed using the lookup buffer for old tags:
2359 * new_seek = tempbuf_find_location(old_seek, ...);
2360 * and for new tags:
2361 * new_seek = tempbuf_find_location(idx);
2362 */
2363 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2364 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2365 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2366
2367 /* And calculate the remaining data space used mainly for storing
2368 * tag data (strings). */
2369 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2370 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2371 {
2372 logf("Buffer way too small!");
2373 return 0;
2374 }
2375
2376 if (fd >= 0)
2377 {
2378 /**
2379 * If tag file contains unique tags (sorted index), we will load
2380 * it entirely into memory so we can resort it later for use with
2381 * chunked browsing.
2382 */
2383 if (tagcache_is_sorted_tag(index_type))
2384 {
2385 logf("loading tags...");
2386 for (i = 0; i < tch.entry_count; i++)
2387 {
2388 struct tagfile_entry entry;
2389 int loc = lseek(fd, 0, SEEK_CUR);
2390 bool ret;
2391
2392 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2393 != sizeof(struct tagfile_entry))
2394 {
2395 logf("read error #7");
2396 close(fd);
2397 return -2;
2398 }
2399
2400 if (entry.tag_length >= (int)sizeof(buf))
2401 {
2402 logf("too long tag #3");
2403 close(fd);
2404 return -2;
2405 }
2406
2407 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2408 {
2409 logf("read error #8");
2410 close(fd);
2411 return -2;
2412 }
2413
2414 /* Skip deleted entries. */
2415 if (buf[0] == '\0')
2416 continue;
2417
2418 /**
2419 * Save the tag and tag id in the memory buffer. Tag id
2420 * is saved so we can later reindex the master lookup
2421 * table when the index gets resorted.
2422 */
2423 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2424 + commit_entry_count, entry.idx_id,
2425 tagcache_is_unique_tag(index_type));
2426 if (!ret)
2427 {
2428 close(fd);
2429 return -3;
2430 }
2431 do_timed_yield();
2432 }
2433 logf("done");
2434 }
2435 else
2436 tempbufidx = tch.entry_count;
2437 }
2438 else
2439 {
2440 /**
2441 * Creating new index file to store the tags. No need to preload
2442 * anything whether the index type is sorted or not.
2443 */
2444 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2445 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2446 if (fd < 0)
2447 {
2448 logf("%s open fail", buf);
2449 return -2;
2450 }
2451
2452 tch.magic = TAGCACHE_MAGIC;
2453 tch.entry_count = 0;
2454 tch.datasize = 0;
2455
2456 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2457 != sizeof(struct tagcache_header))
2458 {
2459 logf("header write failed");
2460 close(fd);
2461 return -2;
2462 }
2463 }
2464
2465 /* Loading the tag lookup file as "master file". */
2466 logf("Loading index file");
2467 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2468
2469 if (masterfd < 0)
2470 {
2471 logf("Creating new DB");
2472 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2473
2474 if (masterfd < 0)
2475 {
2476 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2477 close(fd);
2478 return -2;
2479 }
2480
2481 /* Write the header (write real values later). */
2482 memset(&tcmh, 0, sizeof(struct master_header));
2483 tcmh.tch = *h;
2484 tcmh.tch.entry_count = 0;
2485 tcmh.tch.datasize = 0;
2486 tcmh.dirty = true;
2487 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2488 init = true;
2489 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2490 }
2491 else
2492 {
2493 /**
2494 * Master file already exists so we need to process the current
2495 * file first.
2496 */
2497 init = false;
2498
2499 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2500 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2501 {
2502 logf("header error");
2503 close(fd);
2504 close(masterfd);
2505 return -2;
2506 }
2507
2508 /**
2509 * If we reach end of the master file, we need to expand it to
2510 * hold new tags. If the current index is not sorted, we can
2511 * simply append new data to end of the file.
2512 * However, if the index is sorted, we need to update all tag
2513 * pointers in the master file for the current index.
2514 */
2515 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2516 SEEK_CUR);
2517 if (masterfd_pos == filesize(masterfd))
2518 {
2519 logf("appending...");
2520 init = true;
2521 }
2522 }
2523
2524 /**
2525 * Load new unique tags in memory to be sorted later and added
2526 * to the master lookup file.
2527 */
2528 if (tagcache_is_sorted_tag(index_type))
2529 {
2530 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2531 /* h is the header of the temporary file containing new tags. */
2532 logf("inserting new tags...");
2533 for (i = 0; i < h->entry_count; i++)
2534 {
2535 struct temp_file_entry entry;
2536
2537 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2538 sizeof(struct temp_file_entry))
2539 {
2540 logf("read fail #3");
2541 error = true;
2542 goto error_exit;
2543 }
2544
2545 /* Read data. */
2546 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2547 {
2548 logf("too long entry!");
2549 error = true;
2550 goto error_exit;
2551 }
2552
2553 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2554 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2555 entry.tag_length[index_type])
2556 {
2557 logf("read fail #4");
2558 error = true;
2559 goto error_exit;
2560 }
2561
2562 if (tagcache_is_unique_tag(index_type))
2563 error = !tempbuf_insert(buf, i, -1, true);
2564 else
2565 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2566
2567 if (error)
2568 {
2569 logf("insert error");
2570 goto error_exit;
2571 }
2572
2573 /* Skip to next. */
2574 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2575 entry.tag_length[index_type], SEEK_CUR);
2576 do_timed_yield();
2577 }
2578 logf("done");
2579
2580 /* Sort the buffer data and write it to the index file. */
2581 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2582 i = tempbuf_sort(fd);
2583 if (i < 0)
2584 goto error_exit;
2585 logf("sorted %d tags", i);
2586
2587 /**
2588 * Now update all indexes in the master lookup file.
2589 */
2590 logf("updating indices...");
2591 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2592 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2593 {
2594 int j;
2595 int loc = lseek(masterfd, 0, SEEK_CUR);
2596
2597 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2598
2599 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2600 != (int)sizeof(struct index_entry)*idxbuf_pos)
2601 {
2602 logf("read fail #5");
2603 error = true;
2604 goto error_exit ;
2605 }
2606 lseek(masterfd, loc, SEEK_SET);
2607
2608 for (j = 0; j < idxbuf_pos; j++)
2609 {
2610 if (idxbuf[j].flag & FLAG_DELETED)
2611 {
2612 /* We can just ignore deleted entries. */
2613 // idxbuf[j].tag_seek[index_type] = 0;
2614 continue;
2615 }
2616
2617 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2618 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2619 + commit_entry_count);
2620
2621 if (idxbuf[j].tag_seek[index_type] < 0)
2622 {
2623 logf("update error: %d/%d/%ld",
2624 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2625 error = true;
2626 goto error_exit;
2627 }
2628
2629 do_timed_yield();
2630 }
2631
2632 /* Write back the updated index. */
2633 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2634 index_entry_ec, tc_stat.econ) !=
2635 (int)sizeof(struct index_entry)*idxbuf_pos)
2636 {
2637 logf("write fail");
2638 error = true;
2639 goto error_exit;
2640 }
2641 }
2642 logf("done");
2643 }
2644
2645 /**
2646 * Walk through the temporary file containing the new tags.
2647 */
2648 // build_normal_index(h, tmpfd, masterfd, idx);
2649 logf("updating new indices...");
2650 lseek(masterfd, masterfd_pos, SEEK_SET);
2651 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2652 lseek(fd, 0, SEEK_END);
2653 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2654 {
2655 int j;
2656
2657 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2658 if (init)
2659 {
2660 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2661 }
2662 else
2663 {
2664 int loc = lseek(masterfd, 0, SEEK_CUR);
2665
2666 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2667 != (int)sizeof(struct index_entry)*idxbuf_pos)
2668 {
2669 logf("read fail #6");
2670 error = true;
2671 break ;
2672 }
2673 lseek(masterfd, loc, SEEK_SET);
2674 }
2675
2676 /* Read entry headers. */
2677 for (j = 0; j < idxbuf_pos; j++)
2678 {
2679 if (!tagcache_is_sorted_tag(index_type))
2680 {
2681 struct temp_file_entry entry;
2682 struct tagfile_entry fe;
2683
2684 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2685 sizeof(struct temp_file_entry))
2686 {
2687 logf("read fail #7");
2688 error = true;
2689 break ;
2690 }
2691
2692 /* Read data. */
2693 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2694 {
2695 logf("too long entry!");
2696 logf("length=%d", entry.tag_length[index_type]);
2697 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2698 error = true;
2699 break ;
2700 }
2701
2702 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2703 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2704 entry.tag_length[index_type])
2705 {
2706 logf("read fail #8");
2707 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2708 logf("length=0x%02x", entry.tag_length[index_type]);
2709 error = true;
2710 break ;
2711 }
2712
2713 /* Write to index file. */
2714 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2715 fe.tag_length = entry.tag_length[index_type];
2716 fe.idx_id = tcmh.tch.entry_count + i + j;
2717 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2718 write(fd, buf, fe.tag_length);
2719 tempbufidx++;
2720
2721 /* Skip to next. */
2722 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2723 entry.tag_length[index_type], SEEK_CUR);
2724 }
2725 else
2726 {
2727 /* Locate the correct entry from the sorted array. */
2728 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2729 if (idxbuf[j].tag_seek[index_type] < 0)
2730 {
2731 logf("entry not found (%d)", j);
2732 error = true;
2733 break ;
2734 }
2735 }
2736 }
2737
2738 /* Write index. */
2739 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2740 index_entry_ec, tc_stat.econ) !=
2741 (int)sizeof(struct index_entry)*idxbuf_pos)
2742 {
2743 logf("tagcache: write fail #4");
2744 error = true;
2745 break ;
2746 }
2747
2748 do_timed_yield();
2749 }
2750 logf("done");
2751
2752 /* Finally write the header. */
2753 tch.magic = TAGCACHE_MAGIC;
2754 tch.entry_count = tempbufidx;
2755 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2756 lseek(fd, 0, SEEK_SET);
2757 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2758
2759 if (index_type != tag_filename)
2760 h->datasize += tch.datasize;
2761 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2762 error_exit:
2763
2764 close(fd);
2765 close(masterfd);
2766
2767 if (error)
2768 return -2;
2769
2770 return 1;
2771 }
2772
2773 static bool commit(void)
2774 {
2775 struct tagcache_header tch;
2776 struct master_header tcmh;
2777 int i, len, rc;
2778 int tmpfd;
2779 int masterfd;
2780 #ifdef HAVE_DIRCACHE
2781 bool dircache_buffer_stolen = false;
2782 #endif
2783 bool local_allocation = false;
2784
2785 logf("committing tagcache");
2786
2787 while (write_lock)
2788 sleep(1);
2789
2790 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2791 if (tmpfd < 0)
2792 {
2793 logf("nothing to commit");
2794 return true;
2795 }
2796
2797
2798 /* Load the header. */
2799 len = sizeof(struct tagcache_header);
2800 rc = read(tmpfd, &tch, len);
2801
2802 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2803 {
2804 logf("incorrect header");
2805 close(tmpfd);
2806 remove(TAGCACHE_FILE_TEMP);
2807 return false;
2808 }
2809
2810 if (tch.entry_count == 0)
2811 {
2812 logf("nothing to commit");
2813 close(tmpfd);
2814 remove(TAGCACHE_FILE_TEMP);
2815 return true;
2816 }
2817
2818 #ifdef HAVE_EEPROM_SETTINGS
2819 remove(TAGCACHE_STATEFILE);
2820 #endif
2821
2822 /* At first be sure to unload the ramcache! */
2823 #ifdef HAVE_TC_RAMCACHE
2824 tc_stat.ramcache = false;
2825 #endif
2826
2827 read_lock++;
2828
2829 /* Try to steal every buffer we can :) */
2830 if (tempbuf_size == 0)
2831 local_allocation = true;
2832
2833 #ifdef HAVE_DIRCACHE
2834 if (tempbuf_size == 0)
2835 {
2836 /* Try to steal the dircache buffer. */
2837 tempbuf = dircache_steal_buffer(&tempbuf_size);
2838 tempbuf_size &= ~0x03;
2839
2840 if (tempbuf_size > 0)
2841 {
2842 dircache_buffer_stolen = true;
2843 }
2844 }
2845 #endif
2846
2847 #ifdef HAVE_TC_RAMCACHE
2848 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2849 {
2850 tempbuf = (char *)(hdr + 1);
2851 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2852 tempbuf_size &= ~0x03;
2853 }
2854 #endif
2855
2856 /* And finally fail if there are no buffers available. */
2857 if (tempbuf_size == 0)
2858 {
2859 logf("delaying commit until next boot");
2860 tc_stat.commit_delayed = true;
2861 close(tmpfd);
2862 read_lock--;
2863 return false;
2864 }
2865
2866 logf("commit %ld entries...", tch.entry_count);
2867
2868 /* Mark DB dirty so it will stay disabled if commit fails. */
2869 current_tcmh.dirty = true;
2870 update_master_header();
2871
2872 /* Now create the index files. */
2873 tc_stat.commit_step = 0;
2874 tch.datasize = 0;
2875 tc_stat.commit_delayed = false;
2876
2877 for (i = 0; i < TAG_COUNT; i++)
2878 {
2879 int ret;
2880
2881 if (tagcache_is_numeric_tag(i))
2882 continue;
2883
2884 tc_stat.commit_step++;
2885 ret = build_index(i, &tch, tmpfd);
2886 if (ret <= 0)
2887 {
2888 close(tmpfd);
2889 logf("tagcache failed init");
2890 if (ret < 0)
2891 remove_files();
2892 else
2893 tc_stat.commit_delayed = true;
2894 tc_stat.commit_step = 0;
2895 read_lock--;
2896 return false;
2897 }
2898 }
2899
2900 if (!build_numeric_indices(&tch, tmpfd))
2901 {
2902 logf("Failure to commit numeric indices");
2903 close(tmpfd);
2904 remove_files();
2905 tc_stat.commit_step = 0;
2906 read_lock--;
2907 return false;
2908 }
2909
2910 close(tmpfd);
2911 tc_stat.commit_step = 0;
2912
2913 /* Update the master index headers. */
2914 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2915 {
2916 read_lock--;
2917 return false;
2918 }
2919
2920 tcmh.tch.entry_count += tch.entry_count;
2921 tcmh.tch.datasize = sizeof(struct master_header)
2922 + sizeof(struct index_entry) * tcmh.tch.entry_count
2923 + tch.datasize;
2924 tcmh.dirty = false;
2925 tcmh.commitid++;
2926
2927 lseek(masterfd, 0, SEEK_SET);
2928 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2929 close(masterfd);
2930
2931 logf("tagcache committed");
2932 remove(TAGCACHE_FILE_TEMP);
2933 tc_stat.ready = check_all_headers();
2934 tc_stat.readyvalid = true;
2935
2936 if (local_allocation)
2937 {
2938 tempbuf = NULL;
2939 tempbuf_size = 0;
2940 }
2941
2942 #ifdef HAVE_DIRCACHE
2943 /* Rebuild the dircache, if we stole the buffer. */
2944 if (dircache_buffer_stolen)
2945 dircache_build(0);
2946 #endif
2947
2948 #ifdef HAVE_TC_RAMCACHE
2949 /* Reload tagcache. */
2950 if (tc_stat.ramcache_allocated > 0)
2951 tagcache_start_scan();
2952 #endif
2953
2954 read_lock--;
2955
2956 return true;
2957 }
2958
2959 static void allocate_tempbuf(void)
2960 {
2961 /* Yeah, malloc would be really nice now :) */
2962 #ifdef __PCTOOL__
2963 tempbuf_size = 32*1024*1024;
2964 tempbuf = malloc(tempbuf_size);
2965 #else
2966 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2967 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2968 audiobuf += tempbuf_size;
2969 #endif
2970 }
2971
2972 static void free_tempbuf(void)
2973 {
2974 if (tempbuf_size == 0)
2975 return ;
2976
2977 #ifdef __PCTOOL__
2978 free(tempbuf);
2979 #else
2980 audiobuf -= tempbuf_size;
2981 #endif
2982 tempbuf = NULL;
2983 tempbuf_size = 0;
2984 }
2985
2986 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2987 {
2988 struct index_entry idx;
2989
2990 if (!tc_stat.ready)
2991 return false;
2992
2993 if (!tagcache_is_numeric_tag(tag))
2994 return false;
2995
2996 if (!get_index(masterfd, idx_id, &idx, false))
2997 return false;
2998
2999 idx.tag_seek[tag] = data;
3000 idx.flag |= FLAG_DIRTYNUM;
3001
3002 return write_index(masterfd, idx_id, &idx);
3003 }
3004
3005 #if 0
3006 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3007 int tag, long data)
3008 {
3009 struct master_header myhdr;
3010
3011 if (tcs->masterfd < 0)
3012 {
3013 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3014 return false;
3015 }
3016
3017 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3018 }
3019 #endif
3020
3021 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3022
3023 static bool command_queue_is_full(void)
3024 {
3025 int next;
3026
3027 next = command_queue_widx + 1;
3028 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3029 next = 0;
3030
3031 return (next == command_queue_ridx);
3032 }
3033
3034 void run_command_queue(bool force)
3035 {
3036 struct master_header myhdr;
3037 int masterfd;
3038
3039 if (COMMAND_QUEUE_IS_EMPTY)
3040 return;
3041
3042 if (!force && !command_queue_is_full()
3043 && current_tick - TAGCACHE_COMMAND_QUEUE_COMMIT_DELAY
3044 < command_queue_timestamp)
3045 {
3046 return;
3047 }
3048
3049 mutex_lock(&command_queue_mutex);
3050
3051 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3052 return;
3053
3054 while (command_queue_ridx != command_queue_widx)
3055 {
3056 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3057
3058 switch (ce->command)
3059 {
3060 case CMD_UPDATE_MASTER_HEADER:
3061 {
3062 close(masterfd);
3063 update_master_header();
3064
3065 /* Re-open the masterfd. */
3066 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3067 return;
3068
3069 break;
3070 }
3071 case CMD_UPDATE_NUMERIC:
3072 {
3073 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3074 break;
3075 }
3076 }
3077
3078 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3079 command_queue_ridx = 0;
3080 }
3081
3082 close(masterfd);
3083
3084 mutex_unlock(&command_queue_mutex);
3085 }
3086
3087 static void queue_command(int cmd, long idx_id, int tag, long data)
3088 {
3089 while (1)
3090 {
3091 int next;
3092
3093 mutex_lock(&command_queue_mutex);
3094 next = command_queue_widx + 1;
3095 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3096 next = 0;
3097
3098 /* Make sure queue is not full. */
3099 if (next != command_queue_ridx)
3100 {
3101 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3102
3103 ce->command = cmd;
3104 ce->idx_id = idx_id;
3105 ce->tag = tag;
3106 ce->data = data;
3107
3108 command_queue_widx = next;
3109 command_queue_timestamp = current_tick;
3110 mutex_unlock(&command_queue_mutex);
3111 break;
3112 }
3113
3114 /* Queue is full, try again later... */
3115 mutex_unlock(&command_queue_mutex);
3116 sleep(1);
3117 }
3118 }
3119
3120 long tagcache_increase_serial(void)
3121 {
3122 long old;
3123
3124 if (!tc_stat.ready)
3125 return -2;
3126
3127 while (read_lock)
3128 sleep(1);
3129
3130 old = current_tcmh.serial++;
3131 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3132
3133 return old;
3134 }
3135
3136 void tagcache_update_numeric(int idx_id, int tag, long data)
3137 {
3138 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3139 }
3140
3141 long tagcache_get_serial(void)
3142 {
3143 return current_tcmh.serial;
3144 }
3145
3146 long tagcache_get_commitid(void)
3147 {
3148 return current_tcmh.commitid;
3149 }
3150
3151 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3152 {
3153 char buf[512];
3154 int i;
3155
3156 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3157 for (i = strlen(buf); i < (long)sizeof(buf)-3; i++)
3158 {
3159 if (*datastr == '\0')
3160 break;
3161
3162 if (*datastr == '"')
3163 {
3164 buf[i] = '\\';
3165 datastr++;
3166 continue;
3167 }
3168
3169 buf[i] = *(datastr++);
3170 }
3171
3172 strcpy(&buf[i], "\" ");
3173
3174 write(fd, buf, i + 2);
3175
3176 return true;
3177 }
3178
3179 static bool read_tag(char *dest, long size,
3180 const char *src, const char *tagstr)
3181 {
3182 int pos;
3183 char current_tag[32];
3184
3185 while (*src != '\0')
3186 {
3187 /* Skip all whitespace */
3188 while (*src == ' ')
3189 src++;
3190
3191 if (*src == '\0')
3192 break;
3193
3194 pos = 0;
3195 /* Read in tag name */
3196 while (*src != '=' && *src != ' ')
3197 {
3198 current_tag[pos] = *src;
3199 src++;
3200 pos++;
3201
3202 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3203 return false;
3204 }
3205 current_tag[pos] = '\0';
3206
3207 /* Read in tag data */
3208
3209 /* Find the start. */
3210 while (*src != '"' && *src != '\0')
3211 src++;
3212
3213 if (*src == '\0' || *(++src) == '\0')
3214 return false;
3215
3216 /* Read the data. */
3217 for (pos = 0; pos < size; pos++)
3218 {
3219 if (*src == '\0')
3220 break;
3221
3222 if (*src == '\\' && *(src+1) == '"')
3223 {
3224 dest[pos] = '"';
3225 src += 2;
3226 continue;
3227 }
3228
3229 dest[pos] = *src;
3230
3231 if (*src == '"')
3232 {
3233 src++;
3234 break;
3235 }
3236
3237 if (*src == '\0')
3238 break;
3239
3240 src++;
3241 }
3242 dest[pos] = '\0';
3243
3244 if (!strcasecmp(tagstr, current_tag))
3245 return true;
3246 }
3247
3248 return false;
3249 }
3250
3251 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3252 {
3253 struct index_entry idx;
3254 char tag_data[TAG_MAXLEN+32];
3255 int idx_id;
3256 long masterfd = (long)parameters;
3257 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3258 tag_commitid };
3259 int i;
3260 (void)line_n;
3261
3262 if (*buf == '#')
3263 return 0;
3264
3265 logf("%d/%s", line_n, buf);
3266 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3267 {
3268 logf("filename missing");
3269 logf("-> %s", buf);
3270 return 0;
3271 }
3272
3273 idx_id = find_index(tag_data);
3274 if (idx_id < 0)
3275 {
3276 logf("entry not found");
3277 return 0;
3278 }
3279
3280 if (!get_index(masterfd, idx_id, &idx, false))
3281 {
3282 logf("failed to retrieve index entry");
3283 return 0;
3284 }
3285
3286 /* Stop if tag has already been modified. */
3287 if (idx.flag & FLAG_DIRTYNUM)
3288 return 0;
3289
3290 logf("import: %s", tag_data);
3291
3292 idx.flag |= FLAG_DIRTYNUM;
3293 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3294 {
3295 int data;
3296
3297 if (!read_tag(tag_data, sizeof tag_data, buf,
3298 tagcache_tag_to_str(import_tags[i])))
3299 {
3300 continue;
3301 }
3302
3303 data = atoi(tag_data);
3304 if (data < 0)
3305 continue;
3306
3307 idx.tag_seek[import_tags[i]] = data;
3308
3309 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3310 current_tcmh.serial = data;
3311 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3312 current_tcmh.commitid = data + 1;
3313 }
3314
3315 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3316 }
3317
3318 #ifndef __PCTOOL__
3319 bool tagcache_import_changelog(void)
3320 {
3321 struct master_header myhdr;
3322 struct tagcache_header tch;
3323 int clfd;
3324 long masterfd;
3325 char buf[2048];
3326
3327 if (!tc_stat.ready)
3328 return false;
3329
3330 while (read_lock)
3331 sleep(1);
3332
3333 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3334 if (clfd < 0)
3335 {
3336 logf("failure to open changelog");
3337 return false;
3338 }
3339
3340 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3341 {
3342 close(clfd);
3343 return false;
3344 }
3345
3346 write_lock++;
3347
3348 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3349
3350 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3351 parse_changelog_line);
3352
3353 close(clfd);
3354 close(masterfd);
3355
3356 if (filenametag_fd >= 0)
3357 close(filenametag_fd);
3358
3359 write_lock--;
3360
3361 update_master_header();
3362
3363 return true;
3364 }
3365 #endif
3366
3367 bool tagcache_create_changelog(struct tagcache_search *tcs)
3368 {
3369 struct master_header myhdr;
3370 struct index_entry idx;
3371 char buf[TAG_MAXLEN+32];
3372 char temp[32];
3373 int clfd;
3374 int i, j;
3375
3376 if (!tc_stat.ready)
3377 return false;
3378
3379 if (!tagcache_search(tcs, tag_filename))
3380 return false;
3381
3382 /* Initialize the changelog */
3383 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3384 if (clfd < 0)
3385 {
3386 logf("failure to open changelog");
3387 return false;
3388 }
3389
3390 if (tcs->masterfd < 0)
3391 {
3392 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3393 return false;
3394 }
3395 else
3396 {
3397 lseek(tcs->masterfd, 0, SEEK_SET);
3398 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3399 }
3400
3401 write(clfd, "## Changelog version 1\n", 23);
3402
3403 for (i = 0; i < myhdr.tch.entry_count; i++)
3404 {
3405 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3406 != sizeof(struct index_entry))
3407 {
3408 logf("read error #9");
3409 tagcache_search_finish(tcs);
3410 close(clfd);
3411 return false;
3412 }
3413
3414 /* Skip until the entry found has been modified. */
3415 if (! (idx.flag & FLAG_DIRTYNUM) )
3416 continue;
3417
3418 /* Skip deleted entries too. */
3419 if (idx.flag & FLAG_DELETED)
3420 continue;
3421
3422 /* Now retrieve all tags. */
3423 for (j = 0; j < TAG_COUNT; j++)
3424 {
3425 if (tagcache_is_numeric_tag(j))
3426 {
3427 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3428 write_tag(clfd, tagcache_tag_to_str(j), temp);
3429 continue;
3430 }
3431
3432 tcs->type = j;
3433 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3434 write_tag(clfd, tagcache_tag_to_str(j), buf);
3435 }
3436
3437 write(clfd, "\n", 1);
3438 do_timed_yield();
3439 }
3440
3441 close(clfd);
3442
3443 tagcache_search_finish(tcs);
3444
3445 return true;
3446 }
3447
3448 static bool delete_entry(long idx_id)
3449 {
3450 int fd = -1;
3451 int masterfd = -1;
3452 int tag, i;
3453 struct index_entry idx, myidx;
3454 struct master_header myhdr;
3455 char buf[TAG_MAXLEN+32];
3456 int in_use[TAG_COUNT];
3457
3458 logf("delete_entry(): %ld", idx_id);
3459
3460 #ifdef HAVE_TC_RAMCACHE
3461 /* At first mark the entry removed from ram cache. */
3462 if (tc_stat.ramcache)
3463 hdr->indices[idx_id].flag |= FLAG_DELETED;
3464 #endif
3465
3466 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3467 return false;
3468
3469 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3470 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3471 != sizeof(struct index_entry))
3472 {
3473 logf("delete_entry(): read error");
3474 goto cleanup;
3475 }
3476
3477 if (myidx.flag & FLAG_DELETED)
3478 {
3479 logf("delete_entry(): already deleted!");
3480 goto cleanup;
3481 }
3482
3483 myidx.flag |= FLAG_DELETED;
3484 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3485 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3486 != sizeof(struct index_entry))
3487 {
3488 logf("delete_entry(): write_error #1");
3489 goto cleanup;
3490 }
3491
3492 /* Now check which tags are no longer in use (if any) */
3493 for (tag = 0; tag < TAG_COUNT; tag++)
3494 in_use[tag] = 0;
3495
3496 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3497 for (i = 0; i < myhdr.tch.entry_count; i++)
3498 {
3499 struct index_entry *idxp;
3500
3501 #ifdef HAVE_TC_RAMCACHE
3502 /* Use RAM DB if available for greater speed */
3503 if (tc_stat.ramcache)
3504 idxp = &hdr->indices[i];
3505 else
3506 #endif
3507 {
3508 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3509 != sizeof(struct index_entry))
3510 {
3511 logf("delete_entry(): read error #2");
3512 goto cleanup;
3513 }
3514 idxp = &idx;
3515 }
3516
3517 if (idxp->flag & FLAG_DELETED)
3518 continue;
3519
3520 for (tag = 0; tag < TAG_COUNT; tag++)
3521 {
3522 if (tagcache_is_numeric_tag(tag))
3523 continue;
3524
3525 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3526 in_use[tag]++;
3527 }
3528 }
3529
3530 /* Now delete all tags no longer in use. */
3531 for (tag = 0; tag < TAG_COUNT; tag++)
3532 {
3533 struct tagcache_header tch;
3534 int oldseek = myidx.tag_seek[tag];
3535
3536 if (tagcache_is_numeric_tag(tag))
3537 continue;
3538
3539 /**
3540 * Replace tag seek with a hash value of the field string data.
3541 * That way runtime statistics of moved or altered files can be
3542 * resurrected.
3543 */
3544 #ifdef HAVE_TC_RAMCACHE
3545 if (tc_stat.ramcache && tag != tag_filename)
3546 {
3547 struct tagfile_entry *tfe;
3548 long *seek = &hdr->indices[idx_id].tag_seek[tag];
3549
3550 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3551 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3552 myidx.tag_seek[tag] = *seek;
3553 }
3554 else
3555 #endif
3556 {
3557 struct tagfile_entry tfe;
3558
3559 /* Open the index file, which contains the tag names. */
3560 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3561 goto cleanup;
3562
3563 /* Skip the header block */
3564 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3565 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3566 != sizeof(struct tagfile_entry))
3567 {
3568 logf("delete_entry(): read error #3");
3569 goto cleanup;
3570 }
3571
3572 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3573 {
3574 logf("delete_entry(): read error #4");
3575 goto cleanup;
3576 }
3577
3578 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3579 }
3580
3581 if (in_use[tag])
3582 {
3583 logf("in use: %d/%d", tag, in_use[tag]);
3584 if (fd >= 0)
3585 {
3586 close(fd);
3587 fd = -1;
3588 }
3589 continue;
3590 }
3591
3592 #ifdef HAVE_TC_RAMCACHE
3593 /* Delete from ram. */
3594 if (tc_stat.ramcache && tag != tag_filename)
3595 {
3596 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3597 tagentry->tag_data[0] = '\0';
3598 }
3599 #endif
3600
3601 /* Open the index file, which contains the tag names. */
3602 if (fd < 0)
3603 {
3604 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3605 goto cleanup;
3606 }
3607
3608 /* Skip the header block */
3609 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3610
3611 /* Debug, print 10 first characters of the tag
3612 read(fd, buf, 10);
3613 buf[10]='\0';
3614 logf("TAG:%s", buf);
3615 lseek(fd, -10, SEEK_CUR);
3616 */
3617
3618 /* Write first data byte in tag as \0 */
3619 write(fd, "", 1);
3620
3621 /* Now tag data has been removed */
3622 close(fd);
3623 fd = -1;
3624 }
3625
3626 /* Write index entry back into master index. */
3627 lseek(masterfd, sizeof(struct master_header) +
3628 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3629 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3630 != sizeof(struct index_entry))
3631 {
3632 logf("delete_entry(): write_error #2");
3633 goto cleanup;
3634 }
3635
3636 close(masterfd);
3637
3638 return true;
3639
3640 cleanup:
3641 if (fd >= 0)
3642 close(fd);
3643 if (masterfd >= 0)
3644 close(masterfd);
3645
3646 return false;
3647 }
3648
3649 #ifndef __PCTOOL__
3650 /**
3651 * Returns true if there is an event waiting in the queue
3652 * that requires the current operation to be aborted.
3653 */
3654 static bool check_event_queue(void)
3655 {
3656 struct queue_event ev;
3657
3658 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3659 switch (ev.id)
3660 {
3661 case Q_STOP_SCAN:
3662 case SYS_POWEROFF:
3663 case SYS_USB_CONNECTED:
3664 /* Put the event back into the queue. */
3665 queue_post(&tagcache_queue, ev.id, ev.data);
3666 return true;
3667 }
3668
3669 return false;
3670 }
3671 #endif
3672
3673 #ifdef HAVE_TC_RAMCACHE
3674 static bool allocate_tagcache(void)
3675 {
3676 struct master_header tcmh;
3677 int fd;
3678
3679 /* Load the header. */
3680 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3681 {
3682 hdr = NULL;
3683 return false;
3684 }
3685
3686 close(fd);
3687
3688 /**
3689 * Now calculate the required cache size plus
3690 * some extra space for alignment fixes.
3691 */
3692 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3693 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3694 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3695 memset(hdr, 0, sizeof(struct ramcache_header));
3696 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3697 hdr->indices = (struct index_entry *)(hdr + 1);
3698 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3699
3700 return true;
3701 }
3702
3703 # ifdef HAVE_EEPROM_SETTINGS
3704 static bool tagcache_dumpload(void)
3705 {
3706 struct statefile_header shdr;
3707 int fd, rc;
3708 long offpos;
3709 int i;
3710
3711 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3712 if (fd < 0)
3713 {
3714 logf("no tagcache statedump");
3715 return false;
3716 }
3717
3718 /* Check the statefile memory placement */
3719 hdr = buffer_alloc(0);
3720 rc = read(fd, &shdr, sizeof(struct statefile_header));
3721 if (rc != sizeof(struct statefile_header)
3722 /* || (long)hdr != (long)shdr.hdr */)
3723 {
3724 logf("incorrect statefile");
3725 hdr = NULL;
3726 close(fd);
3727 return false;
3728 }
3729
3730 offpos = (long)hdr - (long)shdr.hdr;
3731
3732 /* Lets allocate real memory and load it */
3733 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3734 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3735 close(fd);
3736
3737 if (rc != shdr.tc_stat.ramcache_allocated)
3738 {
3739 logf("read failure!");
3740 hdr = NULL;
3741 return false;
3742 }
3743
3744 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3745
3746 /* Now fix the pointers */
3747 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3748 for (i = 0; i < TAG_COUNT; i++)
3749 hdr->tags[i] += offpos;
3750
3751 return true;
3752 }
3753
3754 static bool tagcache_dumpsave(void)
3755 {
3756 struct statefile_header shdr;
3757 int fd;
3758
3759 if (!tc_stat.ramcache)
3760 return false;
3761
3762 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3763 if (fd < 0)
3764 {
3765 logf("failed to create a statedump");
3766 return false;
3767 }
3768
3769 /* Create the header */
3770 shdr.hdr = hdr;
3771 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3772 write(fd, &shdr, sizeof(struct statefile_header));
3773
3774 /* And dump the data too */
3775 write(fd, hdr, tc_stat.ramcache_allocated);
3776 close(fd);
3777
3778 return true;
3779 }
3780 # endif
3781
3782 static bool load_tagcache(void)
3783 {
3784 struct tagcache_header *tch;
3785 long bytesleft = tc_stat.ramcache_allocated;
3786 struct index_entry *idx;
3787 int rc, fd;
3788 char *p;
3789 int i, tag;
3790
3791 # ifdef HAVE_DIRCACHE
3792 while (dircache_is_initializing())
3793 sleep(1);
3794 # endif
3795
3796 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3797
3798 logf("loading tagcache to ram...");
3799
3800 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3801 if (fd < 0)
3802 {
3803 logf("tagcache open failed");
3804 return false;
3805 }
3806
3807 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3808 != sizeof(struct master_header)
3809 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3810 {
3811 logf("incorrect header");
3812 return false;
3813 }
3814
3815 idx = hdr->indices;
3816
3817 /* Load the master index table. */
3818 for (i = 0; i < hdr->h.tch.entry_count; i++)
3819 {
3820 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3821 if (rc != sizeof(struct index_entry))
3822 {
3823 logf("read error #10");
3824 close(fd);
3825 return false;
3826 }
3827
3828 bytesleft -= sizeof(struct index_entry);
3829 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3830 {
3831 logf("too big tagcache.");
3832 close(fd);
3833 return false;
3834 }
3835
3836 idx++;
3837 }
3838
3839 close(fd);
3840
3841 /* Load the tags. */
3842 p = (char *)idx;
3843 for (tag = 0; tag < TAG_COUNT; tag++)
3844 {
3845 struct tagfile_entry *fe;
3846 char buf[TAG_MAXLEN+32];
3847
3848 if (tagcache_is_numeric_tag(tag))
3849 continue ;
3850
3851 //p = ((void *)p+1);
3852 p = (char *)((long)p & ~0x03) + 0x04;
3853 hdr->tags[tag] = p;
3854
3855 /* Check the header. */
3856 tch = (struct tagcache_header *)p;
3857 p += sizeof(struct tagcache_header);
3858
3859 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3860 return false;
3861
3862 for (hdr->entry_count[tag] = 0;
3863 hdr->entry_count[tag] < tch->entry_count;
3864 hdr->entry_count[tag]++)
3865 {
3866 long pos;
3867
3868 if (do_timed_yield())
3869 {
3870 /* Abort if we got a critical event in queue */
3871 if (check_event_queue())
3872 return false;
3873 }
3874
3875 fe = (struct tagfile_entry *)p;
3876 pos = lseek(fd, 0, SEEK_CUR);
3877 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3878 if (rc != sizeof(struct tagfile_entry))
3879 {
3880 /* End of lookup table. */
3881 logf("read error #11");
3882 close(fd);
3883 return false;
3884 }
3885
3886 /* We have a special handling for the filename tags. */
3887 if (tag == tag_filename)
3888 {
3889 # ifdef HAVE_DIRCACHE
3890 const struct dirent *dc;
3891 # endif
3892
3893 // FIXME: This is wrong!
3894 // idx = &hdr->indices[hdr->entry_count[i]];
3895 idx = &hdr->indices[fe->idx_id];
3896
3897 if (fe->tag_length >= (long)sizeof(buf)-1)
3898 {
3899 read(fd, buf, 10);
3900 buf[10] = '\0';
3901 logf("TAG:%s", buf);
3902 logf("too long filename");
3903 close(fd);
3904 return false;
3905 }
3906
3907 rc = read(fd, buf, fe->tag_length);
3908 if (rc != fe->tag_length)
3909 {
3910 logf("read error #12");
3911 close(fd);
3912 return false;
3913 }
3914
3915 /* Check if the entry has already been removed */
3916 if (idx->flag & FLAG_DELETED)
3917 continue;
3918
3919 /* This flag must not be used yet. */
3920 if (idx->flag & FLAG_DIRCACHE)
3921 {
3922 logf("internal error!");
3923 close(fd);
3924 return false;
3925 }
3926
3927 if (idx->tag_seek[tag] != pos)
3928 {
3929 logf("corrupt data structures!");
3930 close(fd);
3931 return false;
3932 }
3933
3934 # ifdef HAVE_DIRCACHE
3935 if (dircache_is_enabled())
3936 {
3937 dc = dircache_get_entry_ptr(buf);
3938 if (dc == NULL)
3939 {
3940 logf("Entry no longer valid.");
3941 logf("-> %s", buf);
3942 delete_entry(fe->idx_id);
3943 continue ;
3944 }
3945
3946 idx->flag |= FLAG_DIRCACHE;
3947 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3948 idx->tag_seek[tag_filename] = (long)dc;
3949 }
3950 else
3951 # endif
3952 {
3953 /* This will be very slow unless dircache is enabled
3954 or target is flash based, but do it anyway for
3955 consistency. */
3956 /* Check if entry has been removed. */
3957 if (global_settings.tagcache_autoupdate)
3958 {
3959 if (!file_exists(buf))
3960 {
3961 logf("Entry no longer valid.");
3962 logf("-> %s", buf);
3963 delete_entry(fe->idx_id);
3964 continue;
3965 }
3966 }
3967 }
3968
3969 continue ;
3970 }
3971
3972 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3973 if (bytesleft < 0)
3974 {
3975 logf("too big tagcache #2");
3976 logf("tl: %d", fe->tag_length);
3977 logf("bl: %ld", bytesleft);
3978 close(fd);
3979 return false;
3980 }
3981
3982 p = fe->tag_data;
3983 rc = read(fd, fe->tag_data, fe->tag_length);
3984 p += rc;
3985
3986 if (rc != fe->tag_length)
3987 {
3988 logf("read error #13");
3989 logf("rc=0x%04x", rc); // 0x431
3990 logf("len=0x%04x", fe->tag_length); // 0x4000
3991 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3992 logf("tag=0x%02x", tag); // 0x00
3993 close(fd);
3994 return false;
3995 }
3996 }
3997 close(fd);
3998 }
3999
4000 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4001 logf("tagcache loaded into ram!");
4002
4003 return true;
4004 }
4005 #endif /* HAVE_TC_RAMCACHE */
4006
4007 static bool check_deleted_files(void)
4008 {
4009 int fd;
4010 char buf[TAG_MAXLEN+32];
4011 struct tagfile_entry tfe;
4012
4013 logf("reverse scan...");
4014 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4015 fd = open(buf, O_RDONLY);
4016
4017 if (fd < 0)
4018 {
4019 logf("%s open fail", buf);
4020 return false;
4021 }
4022
4023 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4024 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4025 == sizeof(struct tagfile_entry)
4026 #ifndef __PCTOOL__
4027 && !check_event_queue()
4028 #endif
4029 )
4030 {
4031 if (tfe.tag_length >= (long)sizeof(buf)-1)
4032 {
4033 logf("too long tag");
4034 close(fd);
4035 return false;
4036 }
4037
4038 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4039 {
4040 logf("read error #14");
4041 close(fd);
4042 return false;
4043 }
4044
4045 /* Check if the file has already deleted from the db. */
4046 if (*buf == '\0')
4047 continue;
4048
4049 /* Now check if the file exists. */
4050 if (!file_exists(buf))
4051 {
4052 logf("Entry no longer valid.");
4053 logf("-> %s / %d", buf, tfe.tag_length);
4054 delete_entry(tfe.idx_id);
4055 }
4056 }
4057
4058 close(fd);
4059
4060 logf("done");
4061
4062 return true;
4063 }
4064
4065 static bool check_dir(const char *dirname, int add_files)
4066 {
4067 DIR *dir;
4068 int len;
4069 int success = false;
4070 int ignore, unignore;
4071 char newpath[MAX_PATH];
4072
4073 dir = opendir(dirname);
4074 if (!dir)
4075 {
4076 logf("tagcache: opendir() failed");
4077 return false;
4078 }
4079
4080 /* check for a database.ignore file */
4081 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4082 ignore = file_exists(newpath);
4083 /* check for a database.unignore file */
4084 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4085 unignore = file_exists(newpath);
4086
4087 /* don't do anything if both ignore and unignore are there */
4088 if (ignore != unignore)
4089 add_files = unignore;
4090
4091 /* Recursively scan the dir. */
4092 #ifdef __PCTOOL__
4093 while (1)
4094 #else
4095 while (!check_event_queue())
4096 #endif
4097 {
4098 struct dirent *entry;
4099
4100 entry = readdir(dir);
4101
4102 if (entry == NULL)
4103 {
4104 success = true;
4105 break ;
4106 }
4107
4108 if (!strcmp((char *)entry->d_name, ".") ||
4109 !strcmp((char *)entry->d_name, ".."))
4110 continue;
4111
4112 yield();
4113
4114 len = strlen(curpath);
4115 snprintf(&curpath[len], curpath_size - len, "/%s",
4116 entry->d_name);
4117
4118 processed_dir_count++;
4119 if (entry->attribute & ATTR_DIRECTORY)
4120 check_dir(curpath, add_files);
4121 else if (add_files)
4122 {
4123 tc_stat.curentry = curpath;
4124
4125 /* Add a new entry to the temporary db file. */
4126 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4127 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4128 , dir->internal_entry
4129 #endif
4130 );
4131
4132 /* Wait until current path for debug screen is read and unset. */
4133 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4134 yield();
4135
4136 tc_stat.curentry = NULL;
4137 }
4138
4139 curpath[len] = '\0';
4140 }
4141
4142 closedir(dir);
4143
4144 return success;
4145 }
4146
4147 void tagcache_screensync_event(void)
4148 {
4149 tc_stat.curentry = NULL;
4150 }
4151
4152 void tagcache_screensync_enable(bool state)
4153 {
4154 tc_stat.syncscreen = state;
4155 }
4156
4157 void build_tagcache(const char *path)
4158 {
4159 struct tagcache_header header;
4160 bool ret;
4161
4162 curpath[0] = '\0';
4163 data_size = 0;
4164 total_entry_count = 0;
4165 processed_dir_count = 0;
4166
4167 #ifdef HAVE_DIRCACHE
4168 while (dircache_is_initializing())
4169 sleep(1);
4170 #endif
4171
4172 logf("updating tagcache");
4173
4174 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4175 if (cachefd >= 0)
4176 {
4177 logf("skipping, cache already waiting for commit");
4178 close(cachefd);
4179 return ;
4180 }
4181
4182 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4183 if (cachefd < 0)
4184 {
4185 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4186 return ;
4187 }
4188
4189 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4190
4191 cpu_boost(true);
4192
4193 logf("Scanning files...");
4194 /* Scan for new files. */
4195 memset(&header, 0, sizeof(struct tagcache_header));
4196 write(cachefd, &header, sizeof(struct tagcache_header));
4197
4198 if (strcmp("/", path) != 0)
4199 strcpy(curpath, path);
4200 ret = check_dir(path, true);
4201
4202 /* Write the header. */
4203 header.magic = TAGCACHE_MAGIC;
4204 header.datasize = data_size;
4205 header.entry_count = total_entry_count;
4206 lseek(cachefd, 0, SEEK_SET);
4207 write(cachefd, &header, sizeof(struct tagcache_header));
4208 close(cachefd);
4209
4210 if (filenametag_fd >= 0)
4211 {
4212 close(filenametag_fd);
4213 filenametag_fd = -1;
4214 }
4215
4216 if (!ret)
4217 {
4218 logf("Aborted.");
4219 cpu_boost(false);
4220 return ;
4221 }
4222
4223 /* Commit changes to the database. */
4224 #ifdef __PCTOOL__
4225 allocate_tempbuf();
4226 #endif
4227 if (commit())
4228 {
4229 remove(TAGCACHE_FILE_TEMP);
4230 logf("tagcache built!");
4231 }
4232 #ifdef __PCTOOL__
4233 free_tempbuf();
4234 #endif
4235
4236 #ifdef HAVE_TC_RAMCACHE
4237 if (hdr)
4238 {
4239 /* Import runtime statistics if we just initialized the db. */
4240 if (hdr->h.serial == 0)
4241 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4242 }
4243 #endif
4244
4245 cpu_boost(false);
4246 }
4247
4248 #ifdef HAVE_TC_RAMCACHE
4249 static void load_ramcache(void)
4250 {
4251 if (!hdr)
4252 return ;
4253
4254 cpu_boost(true);
4255
4256 /* At first we should load the cache (if exists). */
4257 tc_stat.ramcache = load_tagcache();
4258
4259 if (!tc_stat.ramcache)
4260 {
4261 /* If loading failed, it must indicate some problem with the db
4262 * so disable it entirely to prevent further issues. */
4263 tc_stat.ready = false;
4264 hdr = NULL;
4265 }
4266
4267 cpu_boost(false);
4268 }
4269
4270 void tagcache_unload_ramcache(void)
4271 {
4272 tc_stat.ramcache = false;
4273 /* Just to make sure there is no statefile present. */
4274 // remove(TAGCACHE_STATEFILE);
4275 }
4276 #endif
4277
4278 #ifndef __PCTOOL__
4279 static void tagcache_thread(void)
4280 {
4281 struct queue_event ev;
4282 bool check_done = false;
4283
4284 /* If the previous cache build/update was interrupted, commit
4285 * the changes first in foreground. */
4286 cpu_boost(true);
4287 allocate_tempbuf();
4288 commit();
4289 free_tempbuf();
4290
4291 #ifdef HAVE_TC_RAMCACHE
4292 # ifdef HAVE_EEPROM_SETTINGS
4293 if (firmware_settings.initialized && firmware_settings.disk_clean)
4294 check_done = tagcache_dumpload();
4295
4296 remove(TAGCACHE_STATEFILE);
4297 # endif
4298
4299 /* Allocate space for the tagcache if found on disk. */
4300 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4301 allocate_tagcache();
4302 #endif
4303
4304 cpu_boost(false);
4305 tc_stat.initialized = true;
4306
4307 /* Don't delay bootup with the header check but do it on background. */
4308 sleep(HZ);
4309 tc_stat.ready = check_all_headers();
4310 tc_stat.readyvalid = true;
4311
4312 while (1)
4313 {
4314 run_command_queue(false);
4315
4316 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4317
4318 switch (ev.id)
4319 {
4320 case Q_IMPORT_CHANGELOG:
4321 tagcache_import_changelog();
4322 break;
4323
4324 case Q_REBUILD:
4325 remove_files();
4326 build_tagcache("/");
4327 break;
4328
4329 case Q_UPDATE:
4330 build_tagcache("/");
4331 #ifdef HAVE_TC_RAMCACHE
4332 load_ramcache();
4333 #endif
4334 check_deleted_files();
4335 break ;
4336
4337 case Q_START_SCAN:
4338 check_done = false;
4339 case SYS_TIMEOUT:
4340 if (check_done || !tc_stat.ready)
4341 break ;
4342
4343 #ifdef HAVE_TC_RAMCACHE
4344 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4345 {
4346 load_ramcache();
4347 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4348 build_tagcache("/");
4349 }
4350 else
4351 #endif
4352 if (global_settings.tagcache_autoupdate)
4353 {
4354 build_tagcache("/");
4355
4356 /* This will be very slow unless dircache is enabled
4357 or target is flash based, but do it anyway for
4358 consistency. */
4359 check_deleted_files();
4360 }
4361
4362 logf("tagcache check done");
4363
4364 check_done = true;
4365 break ;
4366
4367 case Q_STOP_SCAN:
4368 break ;
4369
4370 case SYS_POWEROFF:
4371 break ;
4372
4373 #ifndef SIMULATOR
4374 case SYS_USB_CONNECTED:
4375 logf("USB: TagCache");
4376 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4377 usb_wait_for_disconnect(&tagcache_queue);
4378 break ;
4379 #endif
4380 }
4381 }
4382 }
4383
4384 bool tagcache_prepare_shutdown(void)
4385 {
4386 if (tagcache_get_commit_step() > 0)
4387 return false;
4388
4389 tagcache_stop_scan();
4390 while (read_lock || write_lock)
4391 sleep(1);
4392
4393 return true;
4394 }
4395
4396 void tagcache_shutdown(void)
4397 {
4398 /* Flush the command queue. */
4399 run_command_queue(true);
4400
4401 #ifdef HAVE_EEPROM_SETTINGS
4402 if (tc_stat.ramcache)
4403 tagcache_dumpsave();
4404 #endif
4405 }
4406
4407 static int get_progress(void)
4408 {
4409 int total_count = -1;
4410
4411 #ifdef HAVE_DIRCACHE
4412 if (dircache_is_enabled())
4413 {
4414 total_count = dircache_get_entry_count();
4415 }
4416 else
4417 #endif
4418 #ifdef HAVE_TC_RAMCACHE
4419 {
4420 if (hdr && tc_stat.ramcache)
4421 total_count = hdr->h.tch.entry_count;
4422 }
4423 #endif
4424
4425 if (total_count < 0)
4426 return -1;
4427
4428 return processed_dir_count * 100 / total_count;
4429 }
4430
4431 struct tagcache_stat* tagcache_get_stat(void)
4432 {
4433 tc_stat.progress = get_progress();
4434 tc_stat.processed_entries = processed_dir_count;
4435
4436 return &tc_stat;
4437 }
4438
4439 void tagcache_start_scan(void)
4440 {
4441 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4442 }
4443
4444 bool tagcache_update(void)
4445 {
4446 if (!tc_stat.ready)
4447 return false;
4448
4449 queue_post(&tagcache_queue, Q_UPDATE, 0);
4450 return false;
4451 }
4452
4453 bool tagcache_rebuild()
4454 {
4455 queue_post(&tagcache_queue, Q_REBUILD, 0);
4456 return false;
4457 }
4458
4459 void tagcache_stop_scan(void)
4460 {
4461 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4462 }
4463
4464 #ifdef HAVE_TC_RAMCACHE
4465 bool tagcache_is_ramcache(void)
4466 {
4467 return tc_stat.ramcache;
4468 }
4469 #endif
4470
4471 #endif /* !__PCTOOL__ */
4472
4473
4474 void tagcache_init(void)
4475 {
4476 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4477 memset(&current_tcmh, 0, sizeof(struct master_header));
4478 filenametag_fd = -1;
4479 write_lock = read_lock = 0;
4480
4481 #ifndef __PCTOOL__
4482 mutex_init(&command_queue_mutex);
4483 queue_init(&tagcache_queue, true);
4484 create_thread(tagcache_thread, tagcache_stack,
4485 sizeof(tagcache_stack), 0, tagcache_thread_name
4486 IF_PRIO(, PRIORITY_BACKGROUND)
4487 IF_COP(, CPU));
4488 #else
4489 tc_stat.initialized = true;
4490 allocate_tempbuf();
4491 commit();
4492 free_tempbuf();
4493 tc_stat.ready = check_all_headers();
4494 #endif
4495 }
4496
4497 #ifdef __PCTOOL__
4498 void tagcache_reverse_scan(void)
4499 {
4500 logf("Checking for deleted files");
4501 check_deleted_files();
4502 }
4503 #endif
4504
4505 bool tagcache_is_initialized(void)
4506 {
4507 return tc_stat.initialized;
4508 }
4509 bool tagcache_is_usable(void)
4510 {
4511 return tc_stat.initialized && tc_stat.ready;
4512 }
4513 int tagcache_get_commit_step(void)
4514 {
4515 return tc_stat.commit_step;
4516 }
4517 int tagcache_get_max_commit_step(void)
4518 {
4519 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4520 }
4521
Open Source Jukebox Firmware