?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
E200: Restore the scroll wheel support in brickmania that got lost in the previous...
[Rockbox.git] / apps / tagcache.c
blob2b4f1a2181521538cadac64a1c5f8767e1e9b286
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "thread.h"
61 #include "kernel.h"
62 #include "system.h"
63 #include "logf.h"
64 #include "string.h"
65 #include "usb.h"
66 #include "metadata.h"
67 #include "id3.h"
68 #include "tagcache.h"
69 #include "buffer.h"
70 #include "crc32.h"
71 #include "misc.h"
72 #include "settings.h"
73 #include "dir.h"
74 #include "structec.h"
75 #ifndef __PCTOOL__
76 #include "atoi.h"
77 #include "splash.h"
78 #include "lang.h"
79 #include "eeprom_settings.h"
80 #endif
81
82 #ifdef __PCTOOL__
83 #define yield() do { } while(0)
84 #define sim_sleep(timeout) do { } while(0)
85 #define do_timed_yield() do { } while(0)
86 #endif
87
88 #ifndef __PCTOOL__
89 /* Tag Cache thread. */
90 static struct event_queue tagcache_queue;
91 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
92 static const char tagcache_thread_name[] = "tagcache";
93 #endif
94
95 #define UNTAGGED "<Untagged>"
96
97 /* Previous path when scanning directory tree recursively. */
98 static char curpath[TAG_MAXLEN+32];
99 static long curpath_size = sizeof(curpath);
100
101 /* Used when removing duplicates. */
102 static char *tempbuf; /* Allocated when needed. */
103 static long tempbufidx; /* Current location in buffer. */
104 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
105 static long tempbuf_left; /* Buffer space left. */
106 static long tempbuf_pos;
107
108 /* Tags we want to get sorted (loaded to the tempbuf). */
109 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
110 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
111
112 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
113 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
114 tag_composer, tag_comment, tag_albumartist, tag_grouping };
115
116 /* Numeric tags (we can use these tags with conditional clauses). */
117 static const int numeric_tags[] = { tag_year, tag_discnumber,
118 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
119 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
120 tag_virt_length_min, tag_virt_length_sec,
121 tag_virt_playtime_min, tag_virt_playtime_sec,
122 tag_virt_entryage, tag_virt_autoscore };
123
124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
125 static const char *tags_str[] = { "artist", "album", "genre", "title",
126 "filename", "composer", "comment", "albumartist", "grouping", "year",
127 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
128 "playtime", "lastplayed", "commitid", "mtime" };
129
130 /* Status information of the tagcache. */
131 static struct tagcache_stat tc_stat;
132
133 /* Queue commands. */
134 enum tagcache_queue {
135 Q_STOP_SCAN = 0,
136 Q_START_SCAN,
137 Q_IMPORT_CHANGELOG,
138 Q_UPDATE,
139 Q_REBUILD,
140
141 /* Internal tagcache command queue. */
142 CMD_UPDATE_MASTER_HEADER,
143 CMD_UPDATE_NUMERIC,
144 };
145
146 struct tagcache_command_entry {
147 long command;
148 long idx_id;
149 long tag;
150 long data;
151 };
152
153 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
154 static volatile int command_queue_widx = 0;
155 static volatile int command_queue_ridx = 0;
156 static struct mutex command_queue_mutex;
157 /* Timestamp of the last added event, so we can wait a bit before committing the
158 * whole queue at once. */
159 static long command_queue_timestamp = 0;
160
161 /* Tag database structures. */
162
163 /* Variable-length tag entry in tag files. */
164 struct tagfile_entry {
165 short tag_length; /* Length of the data in bytes including '\0' */
166 short idx_id; /* Corresponding entry location in index file of not unique tags */
167 char tag_data[0]; /* Begin of the tag data */
168 };
169
170 /* Fixed-size tag entry in master db index. */
171 struct index_entry {
172 long tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
173 long flag; /* Status flags */
174 };
175
176 /* Header is the same in every file. */
177 struct tagcache_header {
178 long magic; /* Header version number */
179 long datasize; /* Data size in bytes */
180 long entry_count; /* Number of entries in this file */
181 };
182
183 struct master_header {
184 struct tagcache_header tch;
185 long serial; /* Increasing counting number */
186 long commitid; /* Number of commits so far */
187 long dirty;
188 };
189
190 /* For the endianess correction */
191 static const char *tagfile_entry_ec = "ss";
192 static const char *index_entry_ec = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
193 static const char *tagcache_header_ec = "lll";
194 static const char *master_header_ec = "llllll";
195
196 static struct master_header current_tcmh;
197
198 #ifdef HAVE_TC_RAMCACHE
199 /* Header is created when loading database to ram. */
200 struct ramcache_header {
201 struct master_header h; /* Header from the master index */
202 struct index_entry *indices; /* Master index file content */
203 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
204 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
205 };
206
207 # ifdef HAVE_EEPROM_SETTINGS
208 struct statefile_header {
209 struct ramcache_header *hdr;
210 struct tagcache_stat tc_stat;
211 };
212 # endif
213
214 /* Pointer to allocated ramcache_header */
215 static struct ramcache_header *hdr;
216 #endif
217
218 /**
219 * Full tag entries stored in a temporary file waiting
220 * for commit to the cache. */
221 struct temp_file_entry {
222 long tag_offset[TAG_COUNT];
223 short tag_length[TAG_COUNT];
224 long flag;
225
226 long data_length;
227 };
228
229 struct tempbuf_id_list {
230 long id;
231 struct tempbuf_id_list *next;
232 };
233
234 struct tempbuf_searchidx {
235 long idx_id;
236 char *str;
237 int seek;
238 struct tempbuf_id_list idlist;
239 };
240
241 /* Lookup buffer for fixing messed up index while after sorting. */
242 static long commit_entry_count;
243 static long lookup_buffer_depth;
244 static struct tempbuf_searchidx **lookup;
245
246 /* Used when building the temporary file. */
247 static int cachefd = -1, filenametag_fd;
248 static int total_entry_count = 0;
249 static int data_size = 0;
250 static int processed_dir_count;
251
252 /* Thread safe locking */
253 static volatile int write_lock;
254 static volatile int read_lock;
255
256 static bool delete_entry(long idx_id);
257
258 const char* tagcache_tag_to_str(int tag)
259 {
260 return tags_str[tag];
261 }
262
263 bool tagcache_is_numeric_tag(int type)
264 {
265 int i;
266
267 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
268 {
269 if (type == numeric_tags[i])
270 return true;
271 }
272
273 return false;
274 }
275
276 bool tagcache_is_unique_tag(int type)
277 {
278 int i;
279
280 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
281 {
282 if (type == unique_tags[i])
283 return true;
284 }
285
286 return false;
287 }
288
289 bool tagcache_is_sorted_tag(int type)
290 {
291 int i;
292
293 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
294 {
295 if (type == sorted_tags[i])
296 return true;
297 }
298
299 return false;
300 }
301
302 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
303 {
304 int fd;
305 char buf[MAX_PATH];
306 int rc;
307
308 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
309 return -1;
310
311 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
312
313 fd = open(buf, write ? O_RDWR : O_RDONLY);
314 if (fd < 0)
315 {
316 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
317 tc_stat.ready = false;
318 return fd;
319 }
320
321 /* Check the header. */
322 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
323 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
324 {
325 logf("header error");
326 tc_stat.ready = false;
327 close(fd);
328 return -2;
329 }
330
331 return fd;
332 }
333
334 static int open_master_fd(struct master_header *hdr, bool write)
335 {
336 int fd;
337 int rc;
338
339 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
340 if (fd < 0)
341 {
342 logf("master file open failed for R/W");
343 tc_stat.ready = false;
344 return fd;
345 }
346
347 tc_stat.econ = false;
348
349 /* Check the header. */
350 rc = read(fd, hdr, sizeof(struct master_header));
351 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
352 {
353 /* Success. */
354 return fd;
355 }
356
357 /* Trying to read again, this time with endianess correction enabled. */
358 lseek(fd, 0, SEEK_SET);
359
360 rc = ecread(fd, hdr, 1, master_header_ec, true);
361 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
362 {
363 logf("header error");
364 tc_stat.ready = false;
365 close(fd);
366 return -2;
367 }
368
369 tc_stat.econ = true;
370
371 return fd;
372 }
373
374 #ifndef __PCTOOL__
375 static bool do_timed_yield(void)
376 {
377 /* Sorting can lock up for quite a while, so yield occasionally */
378 static long wakeup_tick = 0;
379 if (current_tick >= wakeup_tick)
380 {
381 wakeup_tick = current_tick + (HZ/4);
382 yield();
383 return true;
384 }
385 return false;
386 }
387 #endif
388
389 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
390 static long find_entry_ram(const char *filename,
391 const struct dirent *dc)
392 {
393 static long last_pos = 0;
394 int i;
395
396 /* Check if we tagcache is loaded into ram. */
397 if (!tc_stat.ramcache)
398 return -1;
399
400 if (dc == NULL)
401 dc = dircache_get_entry_ptr(filename);
402
403 if (dc == NULL)
404 {
405 logf("tagcache: file not found.");
406 return -1;
407 }
408
409 try_again:
410
411 if (last_pos > 0)
412 i = last_pos;
413 else
414 i = 0;
415
416 for (; i < hdr->h.tch.entry_count; i++)
417 {
418 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
419 {
420 last_pos = MAX(0, i - 3);
421 return i;
422 }
423
424 do_timed_yield();
425 }
426
427 if (last_pos > 0)
428 {
429 last_pos = 0;
430 goto try_again;
431 }
432
433 return -1;
434 }
435 #endif
436
437 static long find_entry_disk(const char *filename)
438 {
439 struct tagcache_header tch;
440 static long last_pos = -1;
441 long pos_history[POS_HISTORY_COUNT];
442 long pos_history_idx = 0;
443 bool found = false;
444 struct tagfile_entry tfe;
445 int fd;
446 char buf[TAG_MAXLEN+32];
447 int i;
448 int pos = -1;
449
450 if (!tc_stat.ready)
451 return -2;
452
453 fd = filenametag_fd;
454 if (fd < 0)
455 {
456 last_pos = -1;
457 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
458 return -1;
459 }
460
461 check_again:
462
463 if (last_pos > 0)
464 lseek(fd, last_pos, SEEK_SET);
465 else
466 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
467
468 while (true)
469 {
470 pos = lseek(fd, 0, SEEK_CUR);
471 for (i = pos_history_idx-1; i >= 0; i--)
472 pos_history[i+1] = pos_history[i];
473 pos_history[0] = pos;
474
475 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
476 != sizeof(struct tagfile_entry))
477 {
478 break ;
479 }
480
481 if (tfe.tag_length >= (long)sizeof(buf))
482 {
483 logf("too long tag #1");
484 close(fd);
485 last_pos = -1;
486 return -2;
487 }
488
489 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
490 {
491 logf("read error #2");
492 close(fd);
493 last_pos = -1;
494 return -3;
495 }
496
497 if (!strcasecmp(filename, buf))
498 {
499 last_pos = pos_history[pos_history_idx];
500 found = true;
501 break ;
502 }
503
504 if (pos_history_idx < POS_HISTORY_COUNT - 1)
505 pos_history_idx++;
506 }
507
508 /* Not found? */
509 if (!found)
510 {
511 if (last_pos > 0)
512 {
513 last_pos = -1;
514 logf("seek again");
515 goto check_again;
516 }
517
518 if (fd != filenametag_fd)
519 close(fd);
520 return -4;
521 }
522
523 if (fd != filenametag_fd)
524 close(fd);
525
526 return tfe.idx_id;
527 }
528
529 static int find_index(const char *filename)
530 {
531 long idx_id = -1;
532
533 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
534 if (tc_stat.ramcache && dircache_is_enabled())
535 idx_id = find_entry_ram(filename, NULL);
536 #endif
537
538 if (idx_id < 0)
539 idx_id = find_entry_disk(filename);
540
541 return idx_id;
542 }
543
544 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
545 {
546 int idx_id;
547
548 if (!tc_stat.ready)
549 return false;
550
551 idx_id = find_index(filename);
552 if (idx_id < 0)
553 return false;
554
555 if (!tagcache_search(tcs, tag_filename))
556 return false;
557
558 tcs->entry_count = 0;
559 tcs->idx_id = idx_id;
560
561 return true;
562 }
563
564 static bool get_index(int masterfd, int idxid,
565 struct index_entry *idx, bool use_ram)
566 {
567 bool localfd = false;
568
569 if (idxid < 0)
570 {
571 logf("Incorrect idxid: %d", idxid);
572 return false;
573 }
574
575 #ifdef HAVE_TC_RAMCACHE
576 if (tc_stat.ramcache && use_ram)
577 {
578 if (hdr->indices[idxid].flag & FLAG_DELETED)
579 return false;
580
581 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
582 return true;
583 }
584 #else
585 (void)use_ram;
586 #endif
587
588 if (masterfd < 0)
589 {
590 struct master_header tcmh;
591
592 localfd = true;
593 masterfd = open_master_fd(&tcmh, false);
594 if (masterfd < 0)
595 return false;
596 }
597
598 lseek(masterfd, idxid * sizeof(struct index_entry)
599 + sizeof(struct master_header), SEEK_SET);
600 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
601 != sizeof(struct index_entry))
602 {
603 logf("read error #3");
604 if (localfd)
605 close(masterfd);
606
607 return false;
608 }
609
610 if (localfd)
611 close(masterfd);
612
613 if (idx->flag & FLAG_DELETED)
614 return false;
615
616 return true;
617 }
618
619 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
620 {
621 /* We need to exclude all memory only flags & tags when writing to disk. */
622 if (idx->flag & FLAG_DIRCACHE)
623 {
624 logf("memory only flags!");
625 return false;
626 }
627
628 #ifdef HAVE_TC_RAMCACHE
629 /* Only update numeric data. Writing the whole index to RAM by memcpy
630 * destroys dircache pointers!
631 */
632 if (tc_stat.ramcache)
633 {
634 int tag;
635 struct index_entry *idx_ram = &hdr->indices[idxid];
636
637 for (tag = 0; tag < TAG_COUNT; tag++)
638 {
639 if (tagcache_is_numeric_tag(tag))
640 {
641 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
642 }
643 }
644
645 /* Don't touch the dircache flag. */
646 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
647 }
648 #endif
649
650 lseek(masterfd, idxid * sizeof(struct index_entry)
651 + sizeof(struct master_header), SEEK_SET);
652 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
653 != sizeof(struct index_entry))
654 {
655 logf("write error #3");
656 logf("idxid: %d", idxid);
657 return false;
658 }
659
660 return true;
661 }
662
663 static bool open_files(struct tagcache_search *tcs, int tag)
664 {
665 if (tcs->idxfd[tag] < 0)
666 {
667 char fn[MAX_PATH];
668
669 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
670 tcs->idxfd[tag] = open(fn, O_RDONLY);
671 }
672
673 if (tcs->idxfd[tag] < 0)
674 {
675 logf("File not open!");
676 return false;
677 }
678
679 return true;
680 }
681
682 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
683 int tag, char *buf, long size)
684 {
685 struct tagfile_entry tfe;
686 long seek;
687
688 *buf = '\0';
689
690 if (tagcache_is_numeric_tag(tag))
691 return false;
692
693 seek = idx->tag_seek[tag];
694 if (seek < 0)
695 {
696 logf("Retrieve failed");
697 return false;
698 }
699
700 #ifdef HAVE_TC_RAMCACHE
701 if (tcs->ramsearch)
702 {
703 struct tagfile_entry *ep;
704
705 # ifdef HAVE_DIRCACHE
706 if (tag == tag_filename && idx->flag & FLAG_DIRCACHE)
707 {
708 dircache_copy_path((struct dirent *)seek,
709 buf, size);
710 return true;
711 }
712 else
713 # endif
714 if (tag != tag_filename)
715 {
716 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
717 strncpy(buf, ep->tag_data, size-1);
718
719 return true;
720 }
721 }
722 #endif
723
724 if (!open_files(tcs, tag))
725 return false;
726
727 lseek(tcs->idxfd[tag], seek, SEEK_SET);
728 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
729 != sizeof(struct tagfile_entry))
730 {
731 logf("read error #5");
732 return false;
733 }
734
735 if (tfe.tag_length >= size)
736 {
737 logf("too small buffer");
738 return false;
739 }
740
741 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
742 tfe.tag_length)
743 {
744 logf("read error #6");
745 return false;
746 }
747
748 buf[tfe.tag_length] = '\0';
749
750 return true;
751 }
752
753 static long check_virtual_tags(int tag, const struct index_entry *idx)
754 {
755 long data = 0;
756
757 switch (tag)
758 {
759 case tag_virt_length_sec:
760 data = (idx->tag_seek[tag_length]/1000) % 60;
761 break;
762
763 case tag_virt_length_min:
764 data = (idx->tag_seek[tag_length]/1000) / 60;
765 break;
766
767 case tag_virt_playtime_sec:
768 data = (idx->tag_seek[tag_playtime]/1000) % 60;
769 break;
770
771 case tag_virt_playtime_min:
772 data = (idx->tag_seek[tag_playtime]/1000) / 60;
773 break;
774
775 case tag_virt_autoscore:
776 if (idx->tag_seek[tag_length] == 0
777 || idx->tag_seek[tag_playcount] == 0)
778 {
779 data = 0;
780 }
781 else
782 {
783 data = 100 * idx->tag_seek[tag_playtime]
784 / idx->tag_seek[tag_length]
785 / idx->tag_seek[tag_playcount];
786 }
787 break;
788
789 /* How many commits before the file has been added to the DB. */
790 case tag_virt_entryage:
791 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
792 break;
793
794 default:
795 data = idx->tag_seek[tag];
796 }
797
798 return data;
799 }
800
801 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
802 {
803 struct index_entry idx;
804
805 if (!tc_stat.ready)
806 return false;
807
808 if (!tagcache_is_numeric_tag(tag))
809 return -1;
810
811 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
812 return -2;
813
814 return check_virtual_tags(tag, &idx);
815 }
816
817 inline static bool str_ends_with(const char *str1, const char *str2)
818 {
819 int str_len = strlen(str1);
820 int clause_len = strlen(str2);
821
822 if (clause_len > str_len)
823 return false;
824
825 return !strcasecmp(&str1[str_len - clause_len], str2);
826 }
827
828 inline static bool str_oneof(const char *str, const char *list)
829 {
830 const char *sep;
831 int l, len = strlen(str);
832
833 while (*list)
834 {
835 sep = strchr(list, '|');
836 l = sep ? (long)sep - (long)list : (int)strlen(list);
837 if ((l==len) && !strncasecmp(str, list, len))
838 return true;
839 list += sep ? l + 1 : l;
840 }
841
842 return false;
843 }
844
845 static bool check_against_clause(long numeric, const char *str,
846 const struct tagcache_search_clause *clause)
847 {
848 if (clause->numeric)
849 {
850 switch (clause->type)
851 {
852 case clause_is:
853 return numeric == clause->numeric_data;
854 case clause_is_not:
855 return numeric != clause->numeric_data;
856 case clause_gt:
857 return numeric > clause->numeric_data;
858 case clause_gteq:
859 return numeric >= clause->numeric_data;
860 case clause_lt:
861 return numeric < clause->numeric_data;
862 case clause_lteq:
863 return numeric <= clause->numeric_data;
864 default:
865 logf("Incorrect numeric tag: %d", clause->type);
866 }
867 }
868 else
869 {
870 switch (clause->type)
871 {
872 case clause_is:
873 return !strcasecmp(clause->str, str);
874 case clause_is_not:
875 return strcasecmp(clause->str, str);
876 case clause_gt:
877 return 0>strcasecmp(clause->str, str);
878 case clause_gteq:
879 return 0>=strcasecmp(clause->str, str);
880 case clause_lt:
881 return 0<strcasecmp(clause->str, str);
882 case clause_lteq:
883 return 0<=strcasecmp(clause->str, str);
884 case clause_contains:
885 return (strcasestr(str, clause->str) != NULL);
886 case clause_not_contains:
887 return (strcasestr(str, clause->str) == NULL);
888 case clause_begins_with:
889 return (strcasestr(str, clause->str) == str);
890 case clause_not_begins_with:
891 return (strcasestr(str, clause->str) != str);
892 case clause_ends_with:
893 return str_ends_with(str, clause->str);
894 case clause_not_ends_with:
895 return !str_ends_with(str, clause->str);
896 case clause_oneof:
897 return str_oneof(str, clause->str);
898
899 default:
900 logf("Incorrect tag: %d", clause->type);
901 }
902 }
903
904 return false;
905 }
906
907 static bool check_clauses(struct tagcache_search *tcs,
908 struct index_entry *idx,
909 struct tagcache_search_clause **clause, int count)
910 {
911 int i;
912
913 #ifdef HAVE_TC_RAMCACHE
914 if (tcs->ramsearch)
915 {
916 /* Go through all conditional clauses. */
917 for (i = 0; i < count; i++)
918 {
919 struct tagfile_entry *tfe;
920 int seek;
921 char buf[256];
922 char *str = NULL;
923
924 seek = check_virtual_tags(clause[i]->tag, idx);
925
926 if (!tagcache_is_numeric_tag(clause[i]->tag))
927 {
928 if (clause[i]->tag == tag_filename)
929 {
930 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
931 str = buf;
932 }
933 else
934 {
935 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
936 str = tfe->tag_data;
937 }
938 }
939
940 if (!check_against_clause(seek, str, clause[i]))
941 return false;
942 }
943 }
944 else
945 #endif
946 {
947 /* Check for conditions. */
948 for (i = 0; i < count; i++)
949 {
950 struct tagfile_entry tfe;
951 int seek;
952 char str[256];
953
954 seek = check_virtual_tags(clause[i]->tag, idx);
955
956 memset(str, 0, sizeof str);
957 if (!tagcache_is_numeric_tag(clause[i]->tag))
958 {
959 int fd = tcs->idxfd[clause[i]->tag];
960 lseek(fd, seek, SEEK_SET);
961 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
962 if (tfe.tag_length >= (int)sizeof(str))
963 {
964 logf("Too long tag read!");
965 break ;
966 }
967
968 read(fd, str, tfe.tag_length);
969
970 /* Check if entry has been deleted. */
971 if (str[0] == '\0')
972 break;
973 }
974
975 if (!check_against_clause(seek, str, clause[i]))
976 return false;
977 }
978 }
979
980 return true;
981 }
982
983 bool tagcache_check_clauses(struct tagcache_search *tcs,
984 struct tagcache_search_clause **clause, int count)
985 {
986 struct index_entry idx;
987
988 if (count == 0)
989 return true;
990
991 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
992 return false;
993
994 return check_clauses(tcs, &idx, clause, count);
995 }
996
997 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
998 {
999 int i;
1000
1001 /* If uniq buffer is not defined we must return true for search to work. */
1002 if (tcs->unique_list == NULL
1003 || (!tagcache_is_unique_tag(tcs->type)
1004 && !tagcache_is_numeric_tag(tcs->type)))
1005 {
1006 return true;
1007 }
1008
1009 for (i = 0; i < tcs->unique_list_count; i++)
1010 {
1011 /* Return false if entry is found. */
1012 if (tcs->unique_list[i] == id)
1013 return false;
1014 }
1015
1016 if (tcs->unique_list_count < tcs->unique_list_capacity)
1017 {
1018 tcs->unique_list[i] = id;
1019 tcs->unique_list_count++;
1020 }
1021
1022 return true;
1023 }
1024
1025 static bool build_lookup_list(struct tagcache_search *tcs)
1026 {
1027 struct index_entry entry;
1028 int i;
1029
1030 tcs->seek_list_count = 0;
1031
1032 #ifdef HAVE_TC_RAMCACHE
1033 if (tcs->ramsearch)
1034 {
1035 int j;
1036
1037 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1038 {
1039 struct index_entry *idx = &hdr->indices[i];
1040 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1041 break ;
1042
1043 /* Skip deleted files. */
1044 if (idx->flag & FLAG_DELETED)
1045 continue;
1046
1047 /* Go through all filters.. */
1048 for (j = 0; j < tcs->filter_count; j++)
1049 {
1050 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1051 {
1052 break ;
1053 }
1054 }
1055
1056 if (j < tcs->filter_count)
1057 continue ;
1058
1059 /* Check for conditions. */
1060 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1061 continue;
1062
1063 /* Add to the seek list if not already in uniq buffer. */
1064 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1065 continue;
1066
1067 /* Lets add it. */
1068 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1069 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1070 tcs->seek_list_count++;
1071 }
1072
1073 tcs->seek_pos = i;
1074
1075 return tcs->seek_list_count > 0;
1076 }
1077 #endif
1078
1079 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1080 sizeof(struct master_header), SEEK_SET);
1081
1082 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1083 == sizeof(struct index_entry))
1084 {
1085 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1086 break ;
1087
1088 tcs->seek_pos++;
1089
1090 /* Check if entry has been deleted. */
1091 if (entry.flag & FLAG_DELETED)
1092 continue;
1093
1094 /* Go through all filters.. */
1095 for (i = 0; i < tcs->filter_count; i++)
1096 {
1097 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1098 break ;
1099 }
1100
1101 if (i < tcs->filter_count)
1102 continue ;
1103
1104 /* Check for conditions. */
1105 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1106 continue;
1107
1108 /* Add to the seek list if not already in uniq buffer. */
1109 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1110 continue;
1111
1112 /* Lets add it. */
1113 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1114 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1115 tcs->seek_list_count++;
1116
1117 yield();
1118 }
1119
1120 return tcs->seek_list_count > 0;
1121 }
1122
1123
1124 static void remove_files(void)
1125 {
1126 int i;
1127 char buf[MAX_PATH];
1128
1129 tc_stat.ready = false;
1130 tc_stat.ramcache = false;
1131 tc_stat.econ = false;
1132 remove(TAGCACHE_FILE_MASTER);
1133 for (i = 0; i < TAG_COUNT; i++)
1134 {
1135 if (tagcache_is_numeric_tag(i))
1136 continue;
1137
1138 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1139 remove(buf);
1140 }
1141 }
1142
1143
1144 static bool check_all_headers(void)
1145 {
1146 struct master_header myhdr;
1147 struct tagcache_header tch;
1148 int tag;
1149 int fd;
1150
1151 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1152 return false;
1153
1154 close(fd);
1155 if (myhdr.dirty)
1156 {
1157 logf("tagcache is dirty!");
1158 return false;
1159 }
1160
1161 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1162
1163 for (tag = 0; tag < TAG_COUNT; tag++)
1164 {
1165 if (tagcache_is_numeric_tag(tag))
1166 continue;
1167
1168 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1169 return false;
1170
1171 close(fd);
1172 }
1173
1174 return true;
1175 }
1176
1177 bool tagcache_search(struct tagcache_search *tcs, int tag)
1178 {
1179 struct tagcache_header tag_hdr;
1180 struct master_header master_hdr;
1181 int i;
1182
1183 if (tcs->initialized)
1184 tagcache_search_finish(tcs);
1185
1186 while (read_lock)
1187 sleep(1);
1188
1189 memset(tcs, 0, sizeof(struct tagcache_search));
1190 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1191 return false;
1192
1193 tcs->position = sizeof(struct tagcache_header);
1194 tcs->type = tag;
1195 tcs->seek_pos = 0;
1196 tcs->seek_list_count = 0;
1197 tcs->filter_count = 0;
1198 tcs->masterfd = -1;
1199
1200 for (i = 0; i < TAG_COUNT; i++)
1201 tcs->idxfd[i] = -1;
1202
1203 #ifndef HAVE_TC_RAMCACHE
1204 tcs->ramsearch = false;
1205 #else
1206 tcs->ramsearch = tc_stat.ramcache;
1207 if (tcs->ramsearch)
1208 {
1209 tcs->entry_count = hdr->entry_count[tcs->type];
1210 }
1211 else
1212 #endif
1213 {
1214 if (!tagcache_is_numeric_tag(tcs->type))
1215 {
1216 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1217 if (tcs->idxfd[tcs->type] < 0)
1218 return false;
1219 }
1220
1221 /* Always open as R/W so we can pass tcs to functions that modify data also
1222 * without failing. */
1223 tcs->masterfd = open_master_fd(&master_hdr, true);
1224
1225 if (tcs->masterfd < 0)
1226 return false;
1227 }
1228
1229 tcs->valid = true;
1230 tcs->initialized = true;
1231 write_lock++;
1232
1233 return true;
1234 }
1235
1236 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1237 void *buffer, long length)
1238 {
1239 tcs->unique_list = (unsigned long *)buffer;
1240 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1241 tcs->unique_list_count = 0;
1242 }
1243
1244 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1245 int tag, int seek)
1246 {
1247 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1248 return false;
1249
1250 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1251 return false;
1252
1253 tcs->filter_tag[tcs->filter_count] = tag;
1254 tcs->filter_seek[tcs->filter_count] = seek;
1255 tcs->filter_count++;
1256
1257 return true;
1258 }
1259
1260 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1261 struct tagcache_search_clause *clause)
1262 {
1263 int i;
1264
1265 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1266 {
1267 logf("Too many clauses");
1268 return false;
1269 }
1270
1271 /* Check if there is already a similar filter in present (filters are
1272 * much faster than clauses).
1273 */
1274 for (i = 0; i < tcs->filter_count; i++)
1275 {
1276 if (tcs->filter_tag[i] == clause->tag)
1277 return true;
1278 }
1279
1280 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1281 {
1282 char buf[MAX_PATH];
1283
1284 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1285 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1286 }
1287
1288 tcs->clause[tcs->clause_count] = clause;
1289 tcs->clause_count++;
1290
1291 return true;
1292 }
1293
1294 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1295 ? (flag & FLAG_DIRCACHE) : 1)
1296
1297 static bool get_next(struct tagcache_search *tcs)
1298 {
1299 static char buf[TAG_MAXLEN+32];
1300 struct tagfile_entry entry;
1301 long flag = 0;
1302
1303 if (!tcs->valid || !tc_stat.ready)
1304 return false;
1305
1306 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1307 #ifdef HAVE_TC_RAMCACHE
1308 && !tcs->ramsearch
1309 #endif
1310 )
1311 return false;
1312
1313 /* Relative fetch. */
1314 if (tcs->filter_count > 0 || tcs->clause_count > 0
1315 || tagcache_is_numeric_tag(tcs->type))
1316 {
1317 /* Check for end of list. */
1318 if (tcs->seek_list_count == 0)
1319 {
1320 /* Try to fetch more. */
1321 if (!build_lookup_list(tcs))
1322 {
1323 tcs->valid = false;
1324 return false;
1325 }
1326 }
1327
1328 tcs->seek_list_count--;
1329 flag = tcs->seek_flags[tcs->seek_list_count];
1330
1331 /* Seek stream to the correct position and continue to direct fetch. */
1332 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1333 && !tagcache_is_numeric_tag(tcs->type))
1334 {
1335 if (!open_files(tcs, tcs->type))
1336 return false;
1337
1338 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1339 }
1340 else
1341 tcs->position = tcs->seek_list[tcs->seek_list_count];
1342 }
1343
1344 if (tagcache_is_numeric_tag(tcs->type))
1345 {
1346 snprintf(buf, sizeof(buf), "%d", tcs->position);
1347 tcs->result_seek = tcs->position;
1348 tcs->result = buf;
1349 tcs->result_len = strlen(buf) + 1;
1350 return true;
1351 }
1352
1353 /* Direct fetch. */
1354 #ifdef HAVE_TC_RAMCACHE
1355 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1356 {
1357 struct tagfile_entry *ep;
1358
1359 if (tcs->entry_count == 0)
1360 {
1361 tcs->valid = false;
1362 return false;
1363 }
1364 tcs->entry_count--;
1365
1366 tcs->result_seek = tcs->position;
1367
1368 # ifdef HAVE_DIRCACHE
1369 if (tcs->type == tag_filename)
1370 {
1371 dircache_copy_path((struct dirent *)tcs->position,
1372 buf, sizeof buf);
1373 tcs->result = buf;
1374 tcs->result_len = strlen(buf) + 1;
1375 tcs->idx_id = FLAG_GET_ATTR(flag);
1376 tcs->ramresult = false;
1377
1378 return true;
1379 }
1380 # endif
1381
1382 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1383 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1384 tcs->result = ep->tag_data;
1385 tcs->result_len = strlen(tcs->result) + 1;
1386 tcs->idx_id = ep->idx_id;
1387 tcs->ramresult = true;
1388
1389 return true;
1390 }
1391 else
1392 #endif
1393 {
1394 if (!open_files(tcs, tcs->type))
1395 return false;
1396
1397 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1398 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1399 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1400 {
1401 /* End of data. */
1402 tcs->valid = false;
1403 return false;
1404 }
1405 }
1406
1407 if (entry.tag_length > (long)sizeof(buf))
1408 {
1409 tcs->valid = false;
1410 logf("too long tag #2");
1411 return false;
1412 }
1413
1414 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1415 {
1416 tcs->valid = false;
1417 logf("read error #4");
1418 return false;
1419 }
1420
1421 tcs->result = buf;
1422 tcs->result_len = strlen(tcs->result) + 1;
1423 tcs->idx_id = entry.idx_id;
1424 tcs->ramresult = false;
1425
1426 return true;
1427 }
1428
1429 bool tagcache_get_next(struct tagcache_search *tcs)
1430 {
1431 while (get_next(tcs))
1432 {
1433 if (tcs->result_len > 1)
1434 return true;
1435 }
1436
1437 return false;
1438 }
1439
1440 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1441 int tag, char *buf, long size)
1442 {
1443 struct index_entry idx;
1444
1445 *buf = '\0';
1446 if (!get_index(tcs->masterfd, idxid, &idx, true))
1447 return false;
1448
1449 return retrieve(tcs, &idx, tag, buf, size);
1450 }
1451
1452 static bool update_master_header(void)
1453 {
1454 struct master_header myhdr;
1455 int fd;
1456
1457 if (!tc_stat.ready)
1458 return false;
1459
1460 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1461 return false;
1462
1463 myhdr.serial = current_tcmh.serial;
1464 myhdr.commitid = current_tcmh.commitid;
1465
1466 /* Write it back */
1467 lseek(fd, 0, SEEK_SET);
1468 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1469 close(fd);
1470
1471 #ifdef HAVE_TC_RAMCACHE
1472 if (hdr)
1473 {
1474 hdr->h.serial = current_tcmh.serial;
1475 hdr->h.commitid = current_tcmh.commitid;
1476 }
1477 #endif
1478
1479 return true;
1480 }
1481
1482 #if 0
1483
1484 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1485 {
1486 struct tagentry *entry;
1487
1488 if (tcs->type != tag_title)
1489 return ;
1490
1491 /* We will need reserve buffer for this. */
1492 if (tcs->ramcache)
1493 {
1494 struct tagfile_entry *ep;
1495
1496 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1497 tcs->seek_list[tcs->seek_list_count];
1498 }
1499
1500 entry = find_entry_ram();
1501
1502 }
1503 #endif
1504
1505 void tagcache_search_finish(struct tagcache_search *tcs)
1506 {
1507 int i;
1508
1509 if (!tcs->initialized)
1510 return;
1511
1512 if (tcs->masterfd >= 0)
1513 {
1514 close(tcs->masterfd);
1515 tcs->masterfd = -1;
1516 }
1517
1518 for (i = 0; i < TAG_COUNT; i++)
1519 {
1520 if (tcs->idxfd[i] >= 0)
1521 {
1522 close(tcs->idxfd[i]);
1523 tcs->idxfd[i] = -1;
1524 }
1525 }
1526
1527 tcs->ramsearch = false;
1528 tcs->valid = false;
1529 tcs->initialized = 0;
1530 if (write_lock > 0)
1531 write_lock--;
1532 }
1533
1534 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1535 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1536 {
1537 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1538 }
1539
1540 static long get_tag_numeric(const struct index_entry *entry, int tag)
1541 {
1542 return check_virtual_tags(tag, entry);
1543 }
1544
1545 static char* get_tag_string(const struct index_entry *entry, int tag)
1546 {
1547 char* s = get_tag(entry, tag)->tag_data;
1548 return strcmp(s, UNTAGGED) ? s : NULL;
1549 }
1550
1551 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1552 {
1553 struct index_entry *entry;
1554 int idx_id;
1555
1556 if (!tc_stat.ready)
1557 return false;
1558
1559 /* Find the corresponding entry in tagcache. */
1560 idx_id = find_entry_ram(filename, NULL);
1561 if (idx_id < 0 || !tc_stat.ramcache)
1562 return false;
1563
1564 entry = &hdr->indices[idx_id];
1565
1566 id3->title = get_tag_string(entry, tag_title);
1567 id3->artist = get_tag_string(entry, tag_artist);
1568 id3->album = get_tag_string(entry, tag_album);
1569 id3->genre_string = get_tag_string(entry, tag_genre);
1570 id3->composer = get_tag_string(entry, tag_composer);
1571 id3->comment = get_tag_string(entry, tag_comment);
1572 id3->albumartist = get_tag_string(entry, tag_albumartist);
1573 id3->grouping = get_tag_string(entry, tag_grouping);
1574
1575 id3->playcount = get_tag_numeric(entry, tag_playcount);
1576 id3->rating = get_tag_numeric(entry, tag_rating);
1577 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1578 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1579 id3->year = get_tag_numeric(entry, tag_year);
1580
1581 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1582 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1583 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1584 if (id3->bitrate == 0)
1585 id3->bitrate = 1;
1586
1587 return true;
1588 }
1589 #endif
1590
1591 static inline void write_item(const char *item)
1592 {
1593 int len = strlen(item) + 1;
1594
1595 data_size += len;
1596 write(cachefd, item, len);
1597 }
1598
1599 static int check_if_empty(char **tag)
1600 {
1601 int length;
1602
1603 if (*tag == NULL || *tag[0] == '\0')
1604 {
1605 *tag = UNTAGGED;
1606 return sizeof(UNTAGGED); /* Tag length */
1607 }
1608
1609 length = strlen(*tag);
1610 if (length > TAG_MAXLEN)
1611 {
1612 logf("over length tag: %s", *tag);
1613 length = TAG_MAXLEN;
1614 *tag[length] = '\0';
1615 }
1616
1617 return length + 1;
1618 }
1619
1620 #define ADD_TAG(entry,tag,data) \
1621 /* Adding tag */ \
1622 entry.tag_offset[tag] = offset; \
1623 entry.tag_length[tag] = check_if_empty(data); \
1624 offset += entry.tag_length[tag]
1625
1626 static void add_tagcache(char *path, unsigned long mtime
1627 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1628 ,const struct dirent *dc
1629 #endif
1630 )
1631 {
1632 struct mp3entry id3;
1633 struct temp_file_entry entry;
1634 bool ret;
1635 int fd;
1636 int idx_id = -1;
1637 char tracknumfix[3];
1638 int offset = 0;
1639 int path_length = strlen(path);
1640 bool has_albumartist;
1641 bool has_grouping;
1642
1643 if (cachefd < 0)
1644 return ;
1645
1646 /* Check for overlength file path. */
1647 if (path_length > TAG_MAXLEN)
1648 {
1649 /* Path can't be shortened. */
1650 logf("Too long path: %s", path);
1651 return ;
1652 }
1653
1654 /* Check if the file is supported. */
1655 if (probe_file_format(path) == AFMT_UNKNOWN)
1656 return ;
1657
1658 /* Check if the file is already cached. */
1659 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1660 if (tc_stat.ramcache && dircache_is_enabled())
1661 {
1662 idx_id = find_entry_ram(path, dc);
1663 }
1664 else
1665 #endif
1666 {
1667 if (filenametag_fd >= 0)
1668 {
1669 idx_id = find_entry_disk(path);
1670 }
1671 }
1672
1673 /* Check if file has been modified. */
1674 if (idx_id >= 0)
1675 {
1676 struct index_entry idx;
1677
1678 if (!get_index(-1, idx_id, &idx, true))
1679 {
1680 logf("failed to retrieve index entry");
1681 return ;
1682 }
1683
1684 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1685 {
1686 /* No changes to file. */
1687 return ;
1688 }
1689
1690 /* Metadata might have been changed. Delete the entry. */
1691 logf("Re-adding: %s", path);
1692 if (!delete_entry(idx_id))
1693 {
1694 logf("delete_entry failed: %d", idx_id);
1695 return ;
1696 }
1697 }
1698
1699 fd = open(path, O_RDONLY);
1700 if (fd < 0)
1701 {
1702 logf("open fail: %s", path);
1703 return ;
1704 }
1705
1706 memset(&id3, 0, sizeof(struct mp3entry));
1707 memset(&entry, 0, sizeof(struct temp_file_entry));
1708 memset(&tracknumfix, 0, sizeof(tracknumfix));
1709 ret = get_metadata(&id3, fd, path);
1710 close(fd);
1711
1712 if (!ret)
1713 return ;
1714
1715 logf("-> %s", path);
1716
1717 /* Generate track number if missing. */
1718 if (id3.tracknum <= 0)
1719 {
1720 const char *p = strrchr(path, '.');
1721
1722 if (p == NULL)
1723 p = &path[strlen(path)-1];
1724
1725 while (*p != '/')
1726 {
1727 if (isdigit(*p) && isdigit(*(p-1)))
1728 {
1729 tracknumfix[1] = *p--;
1730 tracknumfix[0] = *p;
1731 break;
1732 }
1733 p--;
1734 }
1735
1736 if (tracknumfix[0] != '\0')
1737 {
1738 id3.tracknum = atoi(tracknumfix);
1739 /* Set a flag to indicate track number has been generated. */
1740 entry.flag |= FLAG_TRKNUMGEN;
1741 }
1742 else
1743 {
1744 /* Unable to generate track number. */
1745 id3.tracknum = -1;
1746 }
1747 }
1748
1749 /* Numeric tags */
1750 entry.tag_offset[tag_year] = id3.year;
1751 entry.tag_offset[tag_discnumber] = id3.discnum;
1752 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1753 entry.tag_offset[tag_length] = id3.length;
1754 entry.tag_offset[tag_bitrate] = id3.bitrate;
1755 entry.tag_offset[tag_mtime] = mtime;
1756
1757 /* String tags. */
1758 has_albumartist = id3.albumartist != NULL
1759 && strlen(id3.albumartist) > 0;
1760 has_grouping = id3.grouping != NULL
1761 && strlen(id3.grouping) > 0;
1762
1763 ADD_TAG(entry, tag_filename, &path);
1764 ADD_TAG(entry, tag_title, &id3.title);
1765 ADD_TAG(entry, tag_artist, &id3.artist);
1766 ADD_TAG(entry, tag_album, &id3.album);
1767 ADD_TAG(entry, tag_genre, &id3.genre_string);
1768 ADD_TAG(entry, tag_composer, &id3.composer);
1769 ADD_TAG(entry, tag_comment, &id3.comment);
1770 if (has_albumartist)
1771 {
1772 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1773 }
1774 else
1775 {
1776 ADD_TAG(entry, tag_albumartist, &id3.artist);
1777 }
1778 if (has_grouping)
1779 {
1780 ADD_TAG(entry, tag_grouping, &id3.grouping);
1781 }
1782 else
1783 {
1784 ADD_TAG(entry, tag_grouping, &id3.title);
1785 }
1786 entry.data_length = offset;
1787
1788 /* Write the header */
1789 write(cachefd, &entry, sizeof(struct temp_file_entry));
1790
1791 /* And tags also... Correct order is critical */
1792 write_item(path);
1793 write_item(id3.title);
1794 write_item(id3.artist);
1795 write_item(id3.album);
1796 write_item(id3.genre_string);
1797 write_item(id3.composer);
1798 write_item(id3.comment);
1799 if (has_albumartist)
1800 {
1801 write_item(id3.albumartist);
1802 }
1803 else
1804 {
1805 write_item(id3.artist);
1806 }
1807 if (has_grouping)
1808 {
1809 write_item(id3.grouping);
1810 }
1811 else
1812 {
1813 write_item(id3.title);
1814 }
1815 total_entry_count++;
1816 }
1817
1818 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1819 {
1820 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1821 int len = strlen(str)+1;
1822 int i;
1823 unsigned crc32;
1824 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1825 char buf[TAG_MAXLEN+32];
1826
1827 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1828 buf[i] = tolower(str[i]);
1829 buf[i] = '\0';
1830
1831 crc32 = crc_32(buf, i, 0xffffffff);
1832
1833 if (unique)
1834 {
1835 /* Check if the crc does not exist -> entry does not exist for sure. */
1836 for (i = 0; i < tempbufidx; i++)
1837 {
1838 if (crcbuf[-i] != crc32)
1839 continue;
1840
1841 if (!strcasecmp(str, index[i].str))
1842 {
1843 if (id < 0 || id >= lookup_buffer_depth)
1844 {
1845 logf("lookup buf overf.: %d", id);
1846 return false;
1847 }
1848
1849 lookup[id] = &index[i];
1850 return true;
1851 }
1852 }
1853 }
1854
1855 /* Insert to CRC buffer. */
1856 crcbuf[-tempbufidx] = crc32;
1857 tempbuf_left -= 4;
1858
1859 /* Insert it to the buffer. */
1860 tempbuf_left -= len;
1861 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1862 return false;
1863
1864 if (id >= lookup_buffer_depth)
1865 {
1866 logf("lookup buf overf. #2: %d", id);
1867 return false;
1868 }
1869
1870 if (id >= 0)
1871 {
1872 lookup[id] = &index[tempbufidx];
1873 index[tempbufidx].idlist.id = id;
1874 }
1875 else
1876 index[tempbufidx].idlist.id = -1;
1877
1878 index[tempbufidx].idlist.next = NULL;
1879 index[tempbufidx].idx_id = idx_id;
1880 index[tempbufidx].seek = -1;
1881 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1882 memcpy(index[tempbufidx].str, str, len);
1883 tempbuf_pos += len;
1884 tempbufidx++;
1885
1886 return true;
1887 }
1888
1889 static int compare(const void *p1, const void *p2)
1890 {
1891 do_timed_yield();
1892
1893 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1894 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1895
1896 if (strcmp(e1->str, UNTAGGED) == 0)
1897 {
1898 if (strcmp(e2->str, UNTAGGED) == 0)
1899 return 0;
1900 return -1;
1901 }
1902 else if (strcmp(e2->str, UNTAGGED) == 0)
1903 return 1;
1904
1905 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1906 }
1907
1908 static int tempbuf_sort(int fd)
1909 {
1910 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1911 struct tagfile_entry fe;
1912 int i;
1913 int length;
1914
1915 /* Generate reverse lookup entries. */
1916 for (i = 0; i < lookup_buffer_depth; i++)
1917 {
1918 struct tempbuf_id_list *idlist;
1919
1920 if (!lookup[i])
1921 continue;
1922
1923 if (lookup[i]->idlist.id == i)
1924 continue;
1925
1926 idlist = &lookup[i]->idlist;
1927 while (idlist->next != NULL)
1928 idlist = idlist->next;
1929
1930 tempbuf_left -= sizeof(struct tempbuf_id_list);
1931 if (tempbuf_left - 4 < 0)
1932 return -1;
1933
1934 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1935 if (tempbuf_pos & 0x03)
1936 {
1937 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1938 tempbuf_left -= 3;
1939 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1940 }
1941 tempbuf_pos += sizeof(struct tempbuf_id_list);
1942
1943 idlist = idlist->next;
1944 idlist->id = i;
1945 idlist->next = NULL;
1946
1947 do_timed_yield();
1948 }
1949
1950 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1951 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1952
1953 for (i = 0; i < tempbufidx; i++)
1954 {
1955 struct tempbuf_id_list *idlist = &index[i].idlist;
1956
1957 /* Fix the lookup list. */
1958 while (idlist != NULL)
1959 {
1960 if (idlist->id >= 0)
1961 lookup[idlist->id] = &index[i];
1962 idlist = idlist->next;
1963 }
1964
1965 index[i].seek = lseek(fd, 0, SEEK_CUR);
1966 length = strlen(index[i].str) + 1;
1967 fe.tag_length = length;
1968 fe.idx_id = index[i].idx_id;
1969
1970 /* Check the chunk alignment. */
1971 if ((fe.tag_length + sizeof(struct tagfile_entry))
1972 % TAGFILE_ENTRY_CHUNK_LENGTH)
1973 {
1974 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1975 ((fe.tag_length + sizeof(struct tagfile_entry))
1976 % TAGFILE_ENTRY_CHUNK_LENGTH);
1977 }
1978
1979 #ifdef TAGCACHE_STRICT_ALIGN
1980 /* Make sure the entry is long aligned. */
1981 if (index[i].seek & 0x03)
1982 {
1983 logf("tempbuf_sort: alignment error!");
1984 return -3;
1985 }
1986 #endif
1987
1988 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
1989 sizeof(struct tagfile_entry))
1990 {
1991 logf("tempbuf_sort: write error #1");
1992 return -1;
1993 }
1994
1995 if (write(fd, index[i].str, length) != length)
1996 {
1997 logf("tempbuf_sort: write error #2");
1998 return -2;
1999 }
2000
2001 /* Write some padding. */
2002 if (fe.tag_length - length > 0)
2003 write(fd, "XXXXXXXX", fe.tag_length - length);
2004 }
2005
2006 return i;
2007 }
2008
2009 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2010 {
2011 if (id < 0 || id >= lookup_buffer_depth)
2012 return NULL;
2013
2014 return lookup[id];
2015 }
2016
2017
2018 inline static int tempbuf_find_location(int id)
2019 {
2020 struct tempbuf_searchidx *entry;
2021
2022 entry = tempbuf_locate(id);
2023 if (entry == NULL)
2024 return -1;
2025
2026 return entry->seek;
2027 }
2028
2029 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2030 {
2031 struct master_header tcmh;
2032 struct index_entry idx;
2033 int masterfd;
2034 int masterfd_pos;
2035 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2036 int max_entries;
2037 int entries_processed = 0;
2038 int i, j;
2039 char buf[TAG_MAXLEN];
2040
2041 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2042
2043 logf("Building numeric indices...");
2044 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2045
2046 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2047 return false;
2048
2049 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2050 SEEK_CUR);
2051 if (masterfd_pos == filesize(masterfd))
2052 {
2053 logf("we can't append!");
2054 close(masterfd);
2055 return false;
2056 }
2057
2058 while (entries_processed < h->entry_count)
2059 {
2060 int count = MIN(h->entry_count - entries_processed, max_entries);
2061
2062 /* Read in as many entries as possible. */
2063 for (i = 0; i < count; i++)
2064 {
2065 struct temp_file_entry *tfe = &entrybuf[i];
2066 int datastart;
2067
2068 /* Read in numeric data. */
2069 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2070 sizeof(struct temp_file_entry))
2071 {
2072 logf("read fail #1");
2073 close(masterfd);
2074 return false;
2075 }
2076
2077 datastart = lseek(tmpfd, 0, SEEK_CUR);
2078
2079 /**
2080 * Read string data from the following tags:
2081 * - tag_filename
2082 * - tag_artist
2083 * - tag_album
2084 * - tag_title
2085 *
2086 * A crc32 hash is calculated from the read data
2087 * and stored back to the data offset field kept in memory.
2088 */
2089 #define tmpdb_read_string_tag(tag) \
2090 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2091 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2092 { \
2093 logf("read fail: buffer overflow"); \
2094 close(masterfd); \
2095 return false; \
2096 } \
2097 \
2098 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2099 tfe->tag_length[tag]) \
2100 { \
2101 logf("read fail #2"); \
2102 close(masterfd); \
2103 return false; \
2104 } \
2105 \
2106 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2107 lseek(tmpfd, datastart, SEEK_SET)
2108
2109 tmpdb_read_string_tag(tag_filename);
2110 tmpdb_read_string_tag(tag_artist);
2111 tmpdb_read_string_tag(tag_album);
2112 tmpdb_read_string_tag(tag_title);
2113
2114 /* Seek to the end of the string data. */
2115 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2116 }
2117
2118 /* Backup the master index position. */
2119 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2120 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2121
2122 /* Check if we can resurrect some deleted runtime statistics data. */
2123 for (i = 0; i < tcmh.tch.entry_count; i++)
2124 {
2125 /* Read the index entry. */
2126 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2127 != sizeof(struct index_entry))
2128 {
2129 logf("read fail #3");
2130 close(masterfd);
2131 return false;
2132 }
2133
2134 /**
2135 * Skip unless the entry is marked as being deleted
2136 * or the data has already been resurrected.
2137 */
2138 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2139 continue;
2140
2141 /* Now try to match the entry. */
2142 /**
2143 * To succesfully match a song, the following conditions
2144 * must apply:
2145 *
2146 * For numeric fields: tag_length
2147 * - Full identical match is required
2148 *
2149 * If tag_filename matches, no further checking necessary.
2150 *
2151 * For string hashes: tag_artist, tag_album, tag_title
2152 * - Two of these must match
2153 */
2154 for (j = 0; j < count; j++)
2155 {
2156 struct temp_file_entry *tfe = &entrybuf[j];
2157
2158 /* Try to match numeric fields first. */
2159 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2160 continue;
2161
2162 /* Now it's time to do the hash matching. */
2163 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2164 {
2165 int match_count = 0;
2166
2167 /* No filename match, check if we can match two other tags. */
2168 #define tmpdb_match(tag) \
2169 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2170 match_count++
2171
2172 tmpdb_match(tag_artist);
2173 tmpdb_match(tag_album);
2174 tmpdb_match(tag_title);
2175
2176 if (match_count < 2)
2177 {
2178 /* Still no match found, give up. */
2179 continue;
2180 }
2181 }
2182
2183 /* A match found, now copy & resurrect the statistical data. */
2184 #define tmpdb_copy_tag(tag) \
2185 tfe->tag_offset[tag] = idx.tag_seek[tag]
2186
2187 tmpdb_copy_tag(tag_playcount);
2188 tmpdb_copy_tag(tag_rating);
2189 tmpdb_copy_tag(tag_playtime);
2190 tmpdb_copy_tag(tag_lastplayed);
2191 tmpdb_copy_tag(tag_commitid);
2192
2193 /* Avoid processing this entry again. */
2194 idx.flag |= FLAG_RESURRECTED;
2195
2196 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2197 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2198 != sizeof(struct index_entry))
2199 {
2200 logf("masterfd writeback fail #1");
2201 close(masterfd);
2202 return false;
2203 }
2204
2205 logf("Entry resurrected");
2206 }
2207 }
2208
2209
2210 /* Restore the master index position. */
2211 lseek(masterfd, masterfd_pos, SEEK_SET);
2212
2213 /* Commit the data to the index. */
2214 for (i = 0; i < count; i++)
2215 {
2216 int loc = lseek(masterfd, 0, SEEK_CUR);
2217
2218 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2219 != sizeof(struct index_entry))
2220 {
2221 logf("read fail #3");
2222 close(masterfd);
2223 return false;
2224 }
2225
2226 for (j = 0; j < TAG_COUNT; j++)
2227 {
2228 if (!tagcache_is_numeric_tag(j))
2229 continue;
2230
2231 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2232 }
2233 idx.flag = entrybuf[i].flag;
2234
2235 if (idx.tag_seek[tag_commitid])
2236 {
2237 /* Data has been resurrected. */
2238 idx.flag |= FLAG_DIRTYNUM;
2239 }
2240 else if (tc_stat.ready && current_tcmh.commitid > 0)
2241 {
2242 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2243 idx.flag |= FLAG_DIRTYNUM;
2244 }
2245
2246 /* Write back the updated index. */
2247 lseek(masterfd, loc, SEEK_SET);
2248 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2249 != sizeof(struct index_entry))
2250 {
2251 logf("write fail");
2252 close(masterfd);
2253 return false;
2254 }
2255 }
2256
2257 entries_processed += count;
2258 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2259 }
2260
2261 close(masterfd);
2262
2263 return true;
2264 }
2265
2266 /**
2267 * Return values:
2268 * > 0 success
2269 * == 0 temporary failure
2270 * < 0 fatal error
2271 */
2272 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2273 {
2274 int i;
2275 struct tagcache_header tch;
2276 struct master_header tcmh;
2277 struct index_entry idxbuf[IDX_BUF_DEPTH];
2278 int idxbuf_pos;
2279 char buf[TAG_MAXLEN+32];
2280 int fd = -1, masterfd;
2281 bool error = false;
2282 int init;
2283 int masterfd_pos;
2284
2285 logf("Building index: %d", index_type);
2286
2287 /* Check the number of entries we need to allocate ram for. */
2288 commit_entry_count = h->entry_count + 1;
2289
2290 masterfd = open_master_fd(&tcmh, false);
2291 if (masterfd >= 0)
2292 {
2293 commit_entry_count += tcmh.tch.entry_count;
2294 close(masterfd);
2295 }
2296 else
2297 remove_files(); /* Just to be sure we are clean. */
2298
2299 /* Open the index file, which contains the tag names. */
2300 fd = open_tag_fd(&tch, index_type, true);
2301 if (fd >= 0)
2302 {
2303 logf("tch.datasize=%ld", tch.datasize);
2304 lookup_buffer_depth = 1 +
2305 /* First part */ commit_entry_count +
2306 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2307 }
2308 else
2309 {
2310 lookup_buffer_depth = 1 +
2311 /* First part */ commit_entry_count +
2312 /* Second part */ 0;
2313 }
2314
2315 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2316 logf("commit_entry_count=%ld", commit_entry_count);
2317
2318 /* Allocate buffer for all index entries from both old and new
2319 * tag files. */
2320 tempbufidx = 0;
2321 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2322
2323 /* Allocate lookup buffer. The first portion of commit_entry_count
2324 * contains the new tags in the temporary file and the second
2325 * part for locating entries already in the db.
2326 *
2327 * New tags Old tags
2328 * +---------+---------------------------+
2329 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2330 * +---------+---------------------------+
2331 *
2332 * Old tags are inserted to a temporary buffer with position:
2333 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2334 * And new tags with index:
2335 * tempbuf_insert(idx, ...);
2336 *
2337 * The buffer is sorted and written into tag file:
2338 * tempbuf_sort(...);
2339 * leaving master index locations messed up.
2340 *
2341 * That is fixed using the lookup buffer for old tags:
2342 * new_seek = tempbuf_find_location(old_seek, ...);
2343 * and for new tags:
2344 * new_seek = tempbuf_find_location(idx);
2345 */
2346 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2347 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2348 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2349
2350 /* And calculate the remaining data space used mainly for storing
2351 * tag data (strings). */
2352 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2353 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2354 {
2355 logf("Buffer way too small!");
2356 return 0;
2357 }
2358
2359 if (fd >= 0)
2360 {
2361 /**
2362 * If tag file contains unique tags (sorted index), we will load
2363 * it entirely into memory so we can resort it later for use with
2364 * chunked browsing.
2365 */
2366 if (tagcache_is_sorted_tag(index_type))
2367 {
2368 logf("loading tags...");
2369 for (i = 0; i < tch.entry_count; i++)
2370 {
2371 struct tagfile_entry entry;
2372 int loc = lseek(fd, 0, SEEK_CUR);
2373 bool ret;
2374
2375 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2376 != sizeof(struct tagfile_entry))
2377 {
2378 logf("read error #7");
2379 close(fd);
2380 return -2;
2381 }
2382
2383 if (entry.tag_length >= (int)sizeof(buf))
2384 {
2385 logf("too long tag #3");
2386 close(fd);
2387 return -2;
2388 }
2389
2390 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2391 {
2392 logf("read error #8");
2393 close(fd);
2394 return -2;
2395 }
2396
2397 /* Skip deleted entries. */
2398 if (buf[0] == '\0')
2399 continue;
2400
2401 /**
2402 * Save the tag and tag id in the memory buffer. Tag id
2403 * is saved so we can later reindex the master lookup
2404 * table when the index gets resorted.
2405 */
2406 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2407 + commit_entry_count, entry.idx_id,
2408 tagcache_is_unique_tag(index_type));
2409 if (!ret)
2410 {
2411 close(fd);
2412 return -3;
2413 }
2414 do_timed_yield();
2415 }
2416 logf("done");
2417 }
2418 else
2419 tempbufidx = tch.entry_count;
2420 }
2421 else
2422 {
2423 /**
2424 * Creating new index file to store the tags. No need to preload
2425 * anything whether the index type is sorted or not.
2426 */
2427 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2428 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2429 if (fd < 0)
2430 {
2431 logf("%s open fail", buf);
2432 return -2;
2433 }
2434
2435 tch.magic = TAGCACHE_MAGIC;
2436 tch.entry_count = 0;
2437 tch.datasize = 0;
2438
2439 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2440 != sizeof(struct tagcache_header))
2441 {
2442 logf("header write failed");
2443 close(fd);
2444 return -2;
2445 }
2446 }
2447
2448 /* Loading the tag lookup file as "master file". */
2449 logf("Loading index file");
2450 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2451
2452 if (masterfd < 0)
2453 {
2454 logf("Creating new DB");
2455 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2456
2457 if (masterfd < 0)
2458 {
2459 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2460 close(fd);
2461 return -2;
2462 }
2463
2464 /* Write the header (write real values later). */
2465 memset(&tcmh, 0, sizeof(struct master_header));
2466 tcmh.tch = *h;
2467 tcmh.tch.entry_count = 0;
2468 tcmh.tch.datasize = 0;
2469 tcmh.dirty = true;
2470 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2471 init = true;
2472 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2473 }
2474 else
2475 {
2476 /**
2477 * Master file already exists so we need to process the current
2478 * file first.
2479 */
2480 init = false;
2481
2482 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2483 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2484 {
2485 logf("header error");
2486 close(fd);
2487 close(masterfd);
2488 return -2;
2489 }
2490
2491 /**
2492 * If we reach end of the master file, we need to expand it to
2493 * hold new tags. If the current index is not sorted, we can
2494 * simply append new data to end of the file.
2495 * However, if the index is sorted, we need to update all tag
2496 * pointers in the master file for the current index.
2497 */
2498 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2499 SEEK_CUR);
2500 if (masterfd_pos == filesize(masterfd))
2501 {
2502 logf("appending...");
2503 init = true;
2504 }
2505 }
2506
2507 /**
2508 * Load new unique tags in memory to be sorted later and added
2509 * to the master lookup file.
2510 */
2511 if (tagcache_is_sorted_tag(index_type))
2512 {
2513 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2514 /* h is the header of the temporary file containing new tags. */
2515 logf("inserting new tags...");
2516 for (i = 0; i < h->entry_count; i++)
2517 {
2518 struct temp_file_entry entry;
2519
2520 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2521 sizeof(struct temp_file_entry))
2522 {
2523 logf("read fail #3");
2524 error = true;
2525 goto error_exit;
2526 }
2527
2528 /* Read data. */
2529 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2530 {
2531 logf("too long entry!");
2532 error = true;
2533 goto error_exit;
2534 }
2535
2536 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2537 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2538 entry.tag_length[index_type])
2539 {
2540 logf("read fail #4");
2541 error = true;
2542 goto error_exit;
2543 }
2544
2545 if (tagcache_is_unique_tag(index_type))
2546 error = !tempbuf_insert(buf, i, -1, true);
2547 else
2548 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2549
2550 if (error)
2551 {
2552 logf("insert error");
2553 goto error_exit;
2554 }
2555
2556 /* Skip to next. */
2557 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2558 entry.tag_length[index_type], SEEK_CUR);
2559 do_timed_yield();
2560 }
2561 logf("done");
2562
2563 /* Sort the buffer data and write it to the index file. */
2564 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2565 i = tempbuf_sort(fd);
2566 if (i < 0)
2567 goto error_exit;
2568 logf("sorted %d tags", i);
2569
2570 /**
2571 * Now update all indexes in the master lookup file.
2572 */
2573 logf("updating indices...");
2574 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2575 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2576 {
2577 int j;
2578 int loc = lseek(masterfd, 0, SEEK_CUR);
2579
2580 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2581
2582 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2583 != (int)sizeof(struct index_entry)*idxbuf_pos)
2584 {
2585 logf("read fail #5");
2586 error = true;
2587 goto error_exit ;
2588 }
2589 lseek(masterfd, loc, SEEK_SET);
2590
2591 for (j = 0; j < idxbuf_pos; j++)
2592 {
2593 if (idxbuf[j].flag & FLAG_DELETED)
2594 {
2595 /* We can just ignore deleted entries. */
2596 // idxbuf[j].tag_seek[index_type] = 0;
2597 continue;
2598 }
2599
2600 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2601 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2602 + commit_entry_count);
2603
2604 if (idxbuf[j].tag_seek[index_type] < 0)
2605 {
2606 logf("update error: %d/%d/%ld",
2607 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2608 error = true;
2609 goto error_exit;
2610 }
2611
2612 do_timed_yield();
2613 }
2614
2615 /* Write back the updated index. */
2616 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2617 index_entry_ec, tc_stat.econ) !=
2618 (int)sizeof(struct index_entry)*idxbuf_pos)
2619 {
2620 logf("write fail");
2621 error = true;
2622 goto error_exit;
2623 }
2624 }
2625 logf("done");
2626 }
2627
2628 /**
2629 * Walk through the temporary file containing the new tags.
2630 */
2631 // build_normal_index(h, tmpfd, masterfd, idx);
2632 logf("updating new indices...");
2633 lseek(masterfd, masterfd_pos, SEEK_SET);
2634 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2635 lseek(fd, 0, SEEK_END);
2636 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2637 {
2638 int j;
2639
2640 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2641 if (init)
2642 {
2643 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2644 }
2645 else
2646 {
2647 int loc = lseek(masterfd, 0, SEEK_CUR);
2648
2649 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2650 != (int)sizeof(struct index_entry)*idxbuf_pos)
2651 {
2652 logf("read fail #6");
2653 error = true;
2654 break ;
2655 }
2656 lseek(masterfd, loc, SEEK_SET);
2657 }
2658
2659 /* Read entry headers. */
2660 for (j = 0; j < idxbuf_pos; j++)
2661 {
2662 if (!tagcache_is_sorted_tag(index_type))
2663 {
2664 struct temp_file_entry entry;
2665 struct tagfile_entry fe;
2666
2667 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2668 sizeof(struct temp_file_entry))
2669 {
2670 logf("read fail #7");
2671 error = true;
2672 break ;
2673 }
2674
2675 /* Read data. */
2676 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2677 {
2678 logf("too long entry!");
2679 logf("length=%d", entry.tag_length[index_type]);
2680 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2681 error = true;
2682 break ;
2683 }
2684
2685 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2686 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2687 entry.tag_length[index_type])
2688 {
2689 logf("read fail #8");
2690 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2691 logf("length=0x%02x", entry.tag_length[index_type]);
2692 error = true;
2693 break ;
2694 }
2695
2696 /* Write to index file. */
2697 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2698 fe.tag_length = entry.tag_length[index_type];
2699 fe.idx_id = tcmh.tch.entry_count + i + j;
2700 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2701 write(fd, buf, fe.tag_length);
2702 tempbufidx++;
2703
2704 /* Skip to next. */
2705 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2706 entry.tag_length[index_type], SEEK_CUR);
2707 }
2708 else
2709 {
2710 /* Locate the correct entry from the sorted array. */
2711 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2712 if (idxbuf[j].tag_seek[index_type] < 0)
2713 {
2714 logf("entry not found (%d)", j);
2715 error = true;
2716 break ;
2717 }
2718 }
2719 }
2720
2721 /* Write index. */
2722 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2723 index_entry_ec, tc_stat.econ) !=
2724 (int)sizeof(struct index_entry)*idxbuf_pos)
2725 {
2726 logf("tagcache: write fail #4");
2727 error = true;
2728 break ;
2729 }
2730
2731 do_timed_yield();
2732 }
2733 logf("done");
2734
2735 /* Finally write the header. */
2736 tch.magic = TAGCACHE_MAGIC;
2737 tch.entry_count = tempbufidx;
2738 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2739 lseek(fd, 0, SEEK_SET);
2740 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2741
2742 if (index_type != tag_filename)
2743 h->datasize += tch.datasize;
2744 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2745 error_exit:
2746
2747 close(fd);
2748 close(masterfd);
2749
2750 if (error)
2751 return -2;
2752
2753 return 1;
2754 }
2755
2756 static bool commit(void)
2757 {
2758 struct tagcache_header tch;
2759 struct master_header tcmh;
2760 int i, len, rc;
2761 int tmpfd;
2762 int masterfd;
2763 #ifdef HAVE_DIRCACHE
2764 bool dircache_buffer_stolen = false;
2765 #endif
2766 bool local_allocation = false;
2767
2768 logf("committing tagcache");
2769
2770 while (write_lock)
2771 sleep(1);
2772
2773 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2774 if (tmpfd < 0)
2775 {
2776 logf("nothing to commit");
2777 return true;
2778 }
2779
2780
2781 /* Load the header. */
2782 len = sizeof(struct tagcache_header);
2783 rc = read(tmpfd, &tch, len);
2784
2785 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2786 {
2787 logf("incorrect header");
2788 close(tmpfd);
2789 remove(TAGCACHE_FILE_TEMP);
2790 return false;
2791 }
2792
2793 if (tch.entry_count == 0)
2794 {
2795 logf("nothing to commit");
2796 close(tmpfd);
2797 remove(TAGCACHE_FILE_TEMP);
2798 return true;
2799 }
2800
2801 #ifdef HAVE_EEPROM_SETTINGS
2802 remove(TAGCACHE_STATEFILE);
2803 #endif
2804
2805 /* At first be sure to unload the ramcache! */
2806 #ifdef HAVE_TC_RAMCACHE
2807 tc_stat.ramcache = false;
2808 #endif
2809
2810 read_lock++;
2811
2812 /* Try to steal every buffer we can :) */
2813 if (tempbuf_size == 0)
2814 local_allocation = true;
2815
2816 #ifdef HAVE_DIRCACHE
2817 if (tempbuf_size == 0)
2818 {
2819 /* Try to steal the dircache buffer. */
2820 tempbuf = dircache_steal_buffer(&tempbuf_size);
2821 tempbuf_size &= ~0x03;
2822
2823 if (tempbuf_size > 0)
2824 {
2825 dircache_buffer_stolen = true;
2826 }
2827 }
2828 #endif
2829
2830 #ifdef HAVE_TC_RAMCACHE
2831 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2832 {
2833 tempbuf = (char *)(hdr + 1);
2834 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2835 tempbuf_size &= ~0x03;
2836 }
2837 #endif
2838
2839 /* And finally fail if there are no buffers available. */
2840 if (tempbuf_size == 0)
2841 {
2842 logf("delaying commit until next boot");
2843 tc_stat.commit_delayed = true;
2844 close(tmpfd);
2845 read_lock--;
2846 return false;
2847 }
2848
2849 logf("commit %ld entries...", tch.entry_count);
2850
2851 /* Mark DB dirty so it will stay disabled if commit fails. */
2852 current_tcmh.dirty = true;
2853 update_master_header();
2854
2855 /* Now create the index files. */
2856 tc_stat.commit_step = 0;
2857 tch.datasize = 0;
2858 tc_stat.commit_delayed = false;
2859
2860 for (i = 0; i < TAG_COUNT; i++)
2861 {
2862 int ret;
2863
2864 if (tagcache_is_numeric_tag(i))
2865 continue;
2866
2867 tc_stat.commit_step++;
2868 ret = build_index(i, &tch, tmpfd);
2869 if (ret <= 0)
2870 {
2871 close(tmpfd);
2872 logf("tagcache failed init");
2873 if (ret < 0)
2874 remove_files();
2875 else
2876 tc_stat.commit_delayed = true;
2877 tc_stat.commit_step = 0;
2878 read_lock--;
2879 return false;
2880 }
2881 }
2882
2883 if (!build_numeric_indices(&tch, tmpfd))
2884 {
2885 logf("Failure to commit numeric indices");
2886 close(tmpfd);
2887 remove_files();
2888 tc_stat.commit_step = 0;
2889 read_lock--;
2890 return false;
2891 }
2892
2893 close(tmpfd);
2894 tc_stat.commit_step = 0;
2895
2896 /* Update the master index headers. */
2897 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2898 {
2899 read_lock--;
2900 return false;
2901 }
2902
2903 tcmh.tch.entry_count += tch.entry_count;
2904 tcmh.tch.datasize = sizeof(struct master_header)
2905 + sizeof(struct index_entry) * tcmh.tch.entry_count
2906 + tch.datasize;
2907 tcmh.dirty = false;
2908 tcmh.commitid++;
2909
2910 lseek(masterfd, 0, SEEK_SET);
2911 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2912 close(masterfd);
2913
2914 logf("tagcache committed");
2915 remove(TAGCACHE_FILE_TEMP);
2916 tc_stat.ready = check_all_headers();
2917 tc_stat.readyvalid = true;
2918
2919 if (local_allocation)
2920 {
2921 tempbuf = NULL;
2922 tempbuf_size = 0;
2923 }
2924
2925 #ifdef HAVE_DIRCACHE
2926 /* Rebuild the dircache, if we stole the buffer. */
2927 if (dircache_buffer_stolen)
2928 dircache_build(0);
2929 #endif
2930
2931 #ifdef HAVE_TC_RAMCACHE
2932 /* Reload tagcache. */
2933 if (tc_stat.ramcache_allocated > 0)
2934 tagcache_start_scan();
2935 #endif
2936
2937 read_lock--;
2938
2939 return true;
2940 }
2941
2942 static void allocate_tempbuf(void)
2943 {
2944 /* Yeah, malloc would be really nice now :) */
2945 #ifdef __PCTOOL__
2946 tempbuf_size = 32*1024*1024;
2947 tempbuf = malloc(tempbuf_size);
2948 #else
2949 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2950 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2951 audiobuf += tempbuf_size;
2952 #endif
2953 }
2954
2955 static void free_tempbuf(void)
2956 {
2957 if (tempbuf_size == 0)
2958 return ;
2959
2960 #ifdef __PCTOOL__
2961 free(tempbuf);
2962 #else
2963 audiobuf -= tempbuf_size;
2964 #endif
2965 tempbuf = NULL;
2966 tempbuf_size = 0;
2967 }
2968
2969 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2970 {
2971 struct index_entry idx;
2972
2973 if (!tc_stat.ready)
2974 return false;
2975
2976 if (!tagcache_is_numeric_tag(tag))
2977 return false;
2978
2979 if (!get_index(masterfd, idx_id, &idx, false))
2980 return false;
2981
2982 idx.tag_seek[tag] = data;
2983 idx.flag |= FLAG_DIRTYNUM;
2984
2985 return write_index(masterfd, idx_id, &idx);
2986 }
2987
2988 #if 0
2989 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
2990 int tag, long data)
2991 {
2992 struct master_header myhdr;
2993
2994 if (tcs->masterfd < 0)
2995 {
2996 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
2997 return false;
2998 }
2999
3000 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3001 }
3002 #endif
3003
3004 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3005
3006 static bool command_queue_is_full(void)
3007 {
3008 int next;
3009
3010 next = command_queue_widx + 1;
3011 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3012 next = 0;
3013
3014 return (next == command_queue_ridx);
3015 }
3016
3017 void run_command_queue(bool force)
3018 {
3019 struct master_header myhdr;
3020 int masterfd;
3021
3022 if (COMMAND_QUEUE_IS_EMPTY)
3023 return;
3024
3025 if (!force && !command_queue_is_full()
3026 && current_tick - TAGCACHE_COMMAND_QUEUE_COMMIT_DELAY
3027 < command_queue_timestamp)
3028 {
3029 return;
3030 }
3031
3032 mutex_lock(&command_queue_mutex);
3033
3034 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3035 return;
3036
3037 while (command_queue_ridx != command_queue_widx)
3038 {
3039 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3040
3041 switch (ce->command)
3042 {
3043 case CMD_UPDATE_MASTER_HEADER:
3044 {
3045 close(masterfd);
3046 update_master_header();
3047
3048 /* Re-open the masterfd. */
3049 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3050 return;
3051
3052 break;
3053 }
3054 case CMD_UPDATE_NUMERIC:
3055 {
3056 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3057 break;
3058 }
3059 }
3060
3061 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3062 command_queue_ridx = 0;
3063 }
3064
3065 close(masterfd);
3066
3067 mutex_unlock(&command_queue_mutex);
3068 }
3069
3070 static void queue_command(int cmd, long idx_id, int tag, long data)
3071 {
3072 while (1)
3073 {
3074 int next;
3075
3076 mutex_lock(&command_queue_mutex);
3077 next = command_queue_widx + 1;
3078 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3079 next = 0;
3080
3081 /* Make sure queue is not full. */
3082 if (next != command_queue_ridx)
3083 {
3084 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3085
3086 ce->command = cmd;
3087 ce->idx_id = idx_id;
3088 ce->tag = tag;
3089 ce->data = data;
3090
3091 command_queue_widx = next;
3092 command_queue_timestamp = current_tick;
3093 mutex_unlock(&command_queue_mutex);
3094 break;
3095 }
3096
3097 /* Queue is full, try again later... */
3098 mutex_unlock(&command_queue_mutex);
3099 sleep(1);
3100 }
3101 }
3102
3103 long tagcache_increase_serial(void)
3104 {
3105 long old;
3106
3107 if (!tc_stat.ready)
3108 return -2;
3109
3110 while (read_lock)
3111 sleep(1);
3112
3113 old = current_tcmh.serial++;
3114 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3115
3116 return old;
3117 }
3118
3119 void tagcache_update_numeric(int idx_id, int tag, long data)
3120 {
3121 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3122 }
3123
3124 long tagcache_get_serial(void)
3125 {
3126 return current_tcmh.serial;
3127 }
3128
3129 long tagcache_get_commitid(void)
3130 {
3131 return current_tcmh.commitid;
3132 }
3133
3134 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3135 {
3136 char buf[512];
3137 int i;
3138
3139 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3140 for (i = strlen(buf); i < (long)sizeof(buf)-3; i++)
3141 {
3142 if (*datastr == '\0')
3143 break;
3144
3145 if (*datastr == '"')
3146 {
3147 buf[i] = '\\';
3148 datastr++;
3149 continue;
3150 }
3151
3152 buf[i] = *(datastr++);
3153 }
3154
3155 strcpy(&buf[i], "\" ");
3156
3157 write(fd, buf, i + 2);
3158
3159 return true;
3160 }
3161
3162 static bool read_tag(char *dest, long size,
3163 const char *src, const char *tagstr)
3164 {
3165 int pos;
3166 char current_tag[32];
3167
3168 while (*src != '\0')
3169 {
3170 /* Skip all whitespace */
3171 while (*src == ' ')
3172 src++;
3173
3174 if (*src == '\0')
3175 break;
3176
3177 pos = 0;
3178 /* Read in tag name */
3179 while (*src != '=' && *src != ' ')
3180 {
3181 current_tag[pos] = *src;
3182 src++;
3183 pos++;
3184
3185 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3186 return false;
3187 }
3188 current_tag[pos] = '\0';
3189
3190 /* Read in tag data */
3191
3192 /* Find the start. */
3193 while (*src != '"' && *src != '\0')
3194 src++;
3195
3196 if (*src == '\0' || *(++src) == '\0')
3197 return false;
3198
3199 /* Read the data. */
3200 for (pos = 0; pos < size; pos++)
3201 {
3202 if (*src == '\0')
3203 break;
3204
3205 if (*src == '\\' && *(src+1) == '"')
3206 {
3207 dest[pos] = '"';
3208 src += 2;
3209 continue;
3210 }
3211
3212 dest[pos] = *src;
3213
3214 if (*src == '"')
3215 {
3216 src++;
3217 break;
3218 }
3219
3220 if (*src == '\0')
3221 break;
3222
3223 src++;
3224 }
3225 dest[pos] = '\0';
3226
3227 if (!strcasecmp(tagstr, current_tag))
3228 return true;
3229 }
3230
3231 return false;
3232 }
3233
3234 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3235 {
3236 struct index_entry idx;
3237 char tag_data[TAG_MAXLEN+32];
3238 int idx_id;
3239 long masterfd = (long)parameters;
3240 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3241 tag_commitid };
3242 int i;
3243 (void)line_n;
3244
3245 if (*buf == '#')
3246 return 0;
3247
3248 logf("%d/%s", line_n, buf);
3249 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3250 {
3251 logf("filename missing");
3252 logf("-> %s", buf);
3253 return 0;
3254 }
3255
3256 idx_id = find_index(tag_data);
3257 if (idx_id < 0)
3258 {
3259 logf("entry not found");
3260 return 0;
3261 }
3262
3263 if (!get_index(masterfd, idx_id, &idx, false))
3264 {
3265 logf("failed to retrieve index entry");
3266 return 0;
3267 }
3268
3269 /* Stop if tag has already been modified. */
3270 if (idx.flag & FLAG_DIRTYNUM)
3271 return 0;
3272
3273 logf("import: %s", tag_data);
3274
3275 idx.flag |= FLAG_DIRTYNUM;
3276 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3277 {
3278 int data;
3279
3280 if (!read_tag(tag_data, sizeof tag_data, buf,
3281 tagcache_tag_to_str(import_tags[i])))
3282 {
3283 continue;
3284 }
3285
3286 data = atoi(tag_data);
3287 if (data < 0)
3288 continue;
3289
3290 idx.tag_seek[import_tags[i]] = data;
3291
3292 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3293 current_tcmh.serial = data;
3294 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3295 current_tcmh.commitid = data + 1;
3296 }
3297
3298 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3299 }
3300
3301 #ifndef __PCTOOL__
3302 bool tagcache_import_changelog(void)
3303 {
3304 struct master_header myhdr;
3305 struct tagcache_header tch;
3306 int clfd;
3307 long masterfd;
3308 char buf[2048];
3309
3310 if (!tc_stat.ready)
3311 return false;
3312
3313 while (read_lock)
3314 sleep(1);
3315
3316 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3317 if (clfd < 0)
3318 {
3319 logf("failure to open changelog");
3320 return false;
3321 }
3322
3323 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3324 {
3325 close(clfd);
3326 return false;
3327 }
3328
3329 write_lock++;
3330
3331 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3332
3333 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3334 parse_changelog_line);
3335
3336 close(clfd);
3337 close(masterfd);
3338
3339 if (filenametag_fd >= 0)
3340 close(filenametag_fd);
3341
3342 write_lock--;
3343
3344 update_master_header();
3345
3346 return true;
3347 }
3348 #endif
3349
3350 bool tagcache_create_changelog(struct tagcache_search *tcs)
3351 {
3352 struct master_header myhdr;
3353 struct index_entry idx;
3354 char buf[TAG_MAXLEN+32];
3355 char temp[32];
3356 int clfd;
3357 int i, j;
3358
3359 if (!tc_stat.ready)
3360 return false;
3361
3362 if (!tagcache_search(tcs, tag_filename))
3363 return false;
3364
3365 /* Initialize the changelog */
3366 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3367 if (clfd < 0)
3368 {
3369 logf("failure to open changelog");
3370 return false;
3371 }
3372
3373 if (tcs->masterfd < 0)
3374 {
3375 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3376 return false;
3377 }
3378 else
3379 {
3380 lseek(tcs->masterfd, 0, SEEK_SET);
3381 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3382 }
3383
3384 write(clfd, "## Changelog version 1\n", 23);
3385
3386 for (i = 0; i < myhdr.tch.entry_count; i++)
3387 {
3388 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3389 != sizeof(struct index_entry))
3390 {
3391 logf("read error #9");
3392 tagcache_search_finish(tcs);
3393 close(clfd);
3394 return false;
3395 }
3396
3397 /* Skip until the entry found has been modified. */
3398 if (! (idx.flag & FLAG_DIRTYNUM) )
3399 continue;
3400
3401 /* Skip deleted entries too. */
3402 if (idx.flag & FLAG_DELETED)
3403 continue;
3404
3405 /* Now retrieve all tags. */
3406 for (j = 0; j < TAG_COUNT; j++)
3407 {
3408 if (tagcache_is_numeric_tag(j))
3409 {
3410 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3411 write_tag(clfd, tagcache_tag_to_str(j), temp);
3412 continue;
3413 }
3414
3415 tcs->type = j;
3416 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3417 write_tag(clfd, tagcache_tag_to_str(j), buf);
3418 }
3419
3420 write(clfd, "\n", 1);
3421 do_timed_yield();
3422 }
3423
3424 close(clfd);
3425
3426 tagcache_search_finish(tcs);
3427
3428 return true;
3429 }
3430
3431 static bool delete_entry(long idx_id)
3432 {
3433 int fd = -1;
3434 int masterfd = -1;
3435 int tag, i;
3436 struct index_entry idx, myidx;
3437 struct master_header myhdr;
3438 char buf[TAG_MAXLEN+32];
3439 int in_use[TAG_COUNT];
3440
3441 logf("delete_entry(): %ld", idx_id);
3442
3443 #ifdef HAVE_TC_RAMCACHE
3444 /* At first mark the entry removed from ram cache. */
3445 if (tc_stat.ramcache)
3446 hdr->indices[idx_id].flag |= FLAG_DELETED;
3447 #endif
3448
3449 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3450 return false;
3451
3452 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3453 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3454 != sizeof(struct index_entry))
3455 {
3456 logf("delete_entry(): read error");
3457 goto cleanup;
3458 }
3459
3460 if (myidx.flag & FLAG_DELETED)
3461 {
3462 logf("delete_entry(): already deleted!");
3463 goto cleanup;
3464 }
3465
3466 myidx.flag |= FLAG_DELETED;
3467 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3468 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3469 != sizeof(struct index_entry))
3470 {
3471 logf("delete_entry(): write_error #1");
3472 goto cleanup;
3473 }
3474
3475 /* Now check which tags are no longer in use (if any) */
3476 for (tag = 0; tag < TAG_COUNT; tag++)
3477 in_use[tag] = 0;
3478
3479 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3480 for (i = 0; i < myhdr.tch.entry_count; i++)
3481 {
3482 struct index_entry *idxp;
3483
3484 #ifdef HAVE_TC_RAMCACHE
3485 /* Use RAM DB if available for greater speed */
3486 if (tc_stat.ramcache)
3487 idxp = &hdr->indices[i];
3488 else
3489 #endif
3490 {
3491 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3492 != sizeof(struct index_entry))
3493 {
3494 logf("delete_entry(): read error #2");
3495 goto cleanup;
3496 }
3497 idxp = &idx;
3498 }
3499
3500 if (idxp->flag & FLAG_DELETED)
3501 continue;
3502
3503 for (tag = 0; tag < TAG_COUNT; tag++)
3504 {
3505 if (tagcache_is_numeric_tag(tag))
3506 continue;
3507
3508 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3509 in_use[tag]++;
3510 }
3511 }
3512
3513 /* Now delete all tags no longer in use. */
3514 for (tag = 0; tag < TAG_COUNT; tag++)
3515 {
3516 struct tagcache_header tch;
3517 int oldseek = myidx.tag_seek[tag];
3518
3519 if (tagcache_is_numeric_tag(tag))
3520 continue;
3521
3522 /**
3523 * Replace tag seek with a hash value of the field string data.
3524 * That way runtime statistics of moved or altered files can be
3525 * resurrected.
3526 */
3527 #ifdef HAVE_TC_RAMCACHE
3528 if (tc_stat.ramcache && tag != tag_filename)
3529 {
3530 struct tagfile_entry *tfe;
3531 long *seek = &hdr->indices[idx_id].tag_seek[tag];
3532
3533 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3534 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3535 myidx.tag_seek[tag] = *seek;
3536 }
3537 else
3538 #endif
3539 {
3540 struct tagfile_entry tfe;
3541
3542 /* Open the index file, which contains the tag names. */
3543 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3544 goto cleanup;
3545
3546 /* Skip the header block */
3547 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3548 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3549 != sizeof(struct tagfile_entry))
3550 {
3551 logf("delete_entry(): read error #3");
3552 goto cleanup;
3553 }
3554
3555 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3556 {
3557 logf("delete_entry(): read error #4");
3558 goto cleanup;
3559 }
3560
3561 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3562 }
3563
3564 if (in_use[tag])
3565 {
3566 logf("in use: %d/%d", tag, in_use[tag]);
3567 if (fd >= 0)
3568 {
3569 close(fd);
3570 fd = -1;
3571 }
3572 continue;
3573 }
3574
3575 #ifdef HAVE_TC_RAMCACHE
3576 /* Delete from ram. */
3577 if (tc_stat.ramcache && tag != tag_filename)
3578 {
3579 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3580 tagentry->tag_data[0] = '\0';
3581 }
3582 #endif
3583
3584 /* Open the index file, which contains the tag names. */
3585 if (fd < 0)
3586 {
3587 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3588 goto cleanup;
3589 }
3590
3591 /* Skip the header block */
3592 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3593
3594 /* Debug, print 10 first characters of the tag
3595 read(fd, buf, 10);
3596 buf[10]='\0';
3597 logf("TAG:%s", buf);
3598 lseek(fd, -10, SEEK_CUR);
3599 */
3600
3601 /* Write first data byte in tag as \0 */
3602 write(fd, "", 1);
3603
3604 /* Now tag data has been removed */
3605 close(fd);
3606 fd = -1;
3607 }
3608
3609 /* Write index entry back into master index. */
3610 lseek(masterfd, sizeof(struct master_header) +
3611 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3612 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3613 != sizeof(struct index_entry))
3614 {
3615 logf("delete_entry(): write_error #2");
3616 goto cleanup;
3617 }
3618
3619 close(masterfd);
3620
3621 return true;
3622
3623 cleanup:
3624 if (fd >= 0)
3625 close(fd);
3626 if (masterfd >= 0)
3627 close(masterfd);
3628
3629 return false;
3630 }
3631
3632 #ifndef __PCTOOL__
3633 /**
3634 * Returns true if there is an event waiting in the queue
3635 * that requires the current operation to be aborted.
3636 */
3637 static bool check_event_queue(void)
3638 {
3639 struct queue_event ev;
3640
3641 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3642 switch (ev.id)
3643 {
3644 case Q_STOP_SCAN:
3645 case SYS_POWEROFF:
3646 case SYS_USB_CONNECTED:
3647 /* Put the event back into the queue. */
3648 queue_post(&tagcache_queue, ev.id, ev.data);
3649 return true;
3650 }
3651
3652 return false;
3653 }
3654 #endif
3655
3656 #ifdef HAVE_TC_RAMCACHE
3657 static bool allocate_tagcache(void)
3658 {
3659 struct master_header tcmh;
3660 int fd;
3661
3662 /* Load the header. */
3663 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3664 {
3665 hdr = NULL;
3666 return false;
3667 }
3668
3669 close(fd);
3670
3671 /**
3672 * Now calculate the required cache size plus
3673 * some extra space for alignment fixes.
3674 */
3675 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3676 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3677 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3678 memset(hdr, 0, sizeof(struct ramcache_header));
3679 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3680 hdr->indices = (struct index_entry *)(hdr + 1);
3681 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3682
3683 return true;
3684 }
3685
3686 # ifdef HAVE_EEPROM_SETTINGS
3687 static bool tagcache_dumpload(void)
3688 {
3689 struct statefile_header shdr;
3690 int fd, rc;
3691 long offpos;
3692 int i;
3693
3694 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3695 if (fd < 0)
3696 {
3697 logf("no tagcache statedump");
3698 return false;
3699 }
3700
3701 /* Check the statefile memory placement */
3702 hdr = buffer_alloc(0);
3703 rc = read(fd, &shdr, sizeof(struct statefile_header));
3704 if (rc != sizeof(struct statefile_header)
3705 /* || (long)hdr != (long)shdr.hdr */)
3706 {
3707 logf("incorrect statefile");
3708 hdr = NULL;
3709 close(fd);
3710 return false;
3711 }
3712
3713 offpos = (long)hdr - (long)shdr.hdr;
3714
3715 /* Lets allocate real memory and load it */
3716 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3717 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3718 close(fd);
3719
3720 if (rc != shdr.tc_stat.ramcache_allocated)
3721 {
3722 logf("read failure!");
3723 hdr = NULL;
3724 return false;
3725 }
3726
3727 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3728
3729 /* Now fix the pointers */
3730 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3731 for (i = 0; i < TAG_COUNT; i++)
3732 hdr->tags[i] += offpos;
3733
3734 return true;
3735 }
3736
3737 static bool tagcache_dumpsave(void)
3738 {
3739 struct statefile_header shdr;
3740 int fd;
3741
3742 if (!tc_stat.ramcache)
3743 return false;
3744
3745 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3746 if (fd < 0)
3747 {
3748 logf("failed to create a statedump");
3749 return false;
3750 }
3751
3752 /* Create the header */
3753 shdr.hdr = hdr;
3754 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3755 write(fd, &shdr, sizeof(struct statefile_header));
3756
3757 /* And dump the data too */
3758 write(fd, hdr, tc_stat.ramcache_allocated);
3759 close(fd);
3760
3761 return true;
3762 }
3763 # endif
3764
3765 static bool load_tagcache(void)
3766 {
3767 struct tagcache_header *tch;
3768 long bytesleft = tc_stat.ramcache_allocated;
3769 struct index_entry *idx;
3770 int rc, fd;
3771 char *p;
3772 int i, tag;
3773
3774 # ifdef HAVE_DIRCACHE
3775 while (dircache_is_initializing())
3776 sleep(1);
3777 # endif
3778
3779 logf("loading tagcache to ram...");
3780
3781 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3782 if (fd < 0)
3783 {
3784 logf("tagcache open failed");
3785 return false;
3786 }
3787
3788 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3789 != sizeof(struct master_header)
3790 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3791 {
3792 logf("incorrect header");
3793 return false;
3794 }
3795
3796 idx = hdr->indices;
3797
3798 /* Load the master index table. */
3799 for (i = 0; i < hdr->h.tch.entry_count; i++)
3800 {
3801 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3802 if (rc != sizeof(struct index_entry))
3803 {
3804 logf("read error #10");
3805 close(fd);
3806 return false;
3807 }
3808
3809 bytesleft -= sizeof(struct index_entry);
3810 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3811 {
3812 logf("too big tagcache.");
3813 close(fd);
3814 return false;
3815 }
3816
3817 idx++;
3818 }
3819
3820 close(fd);
3821
3822 /* Load the tags. */
3823 p = (char *)idx;
3824 for (tag = 0; tag < TAG_COUNT; tag++)
3825 {
3826 struct tagfile_entry *fe;
3827 char buf[TAG_MAXLEN+32];
3828
3829 if (tagcache_is_numeric_tag(tag))
3830 continue ;
3831
3832 //p = ((void *)p+1);
3833 p = (char *)((long)p & ~0x03) + 0x04;
3834 hdr->tags[tag] = p;
3835
3836 /* Check the header. */
3837 tch = (struct tagcache_header *)p;
3838 p += sizeof(struct tagcache_header);
3839
3840 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3841 return false;
3842
3843 for (hdr->entry_count[tag] = 0;
3844 hdr->entry_count[tag] < tch->entry_count;
3845 hdr->entry_count[tag]++)
3846 {
3847 long pos;
3848
3849 if (do_timed_yield())
3850 {
3851 /* Abort if we got a critical event in queue */
3852 if (check_event_queue())
3853 return false;
3854 }
3855
3856 fe = (struct tagfile_entry *)p;
3857 pos = lseek(fd, 0, SEEK_CUR);
3858 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3859 if (rc != sizeof(struct tagfile_entry))
3860 {
3861 /* End of lookup table. */
3862 logf("read error #11");
3863 close(fd);
3864 return false;
3865 }
3866
3867 /* We have a special handling for the filename tags. */
3868 if (tag == tag_filename)
3869 {
3870 # ifdef HAVE_DIRCACHE
3871 const struct dirent *dc;
3872 # endif
3873
3874 // FIXME: This is wrong!
3875 // idx = &hdr->indices[hdr->entry_count[i]];
3876 idx = &hdr->indices[fe->idx_id];
3877
3878 if (fe->tag_length >= (long)sizeof(buf)-1)
3879 {
3880 read(fd, buf, 10);
3881 buf[10] = '\0';
3882 logf("TAG:%s", buf);
3883 logf("too long filename");
3884 close(fd);
3885 return false;
3886 }
3887
3888 rc = read(fd, buf, fe->tag_length);
3889 if (rc != fe->tag_length)
3890 {
3891 logf("read error #12");
3892 close(fd);
3893 return false;
3894 }
3895
3896 /* Check if the entry has already been removed */
3897 if (idx->flag & FLAG_DELETED)
3898 continue;
3899
3900 /* This flag must not be used yet. */
3901 if (idx->flag & FLAG_DIRCACHE)
3902 {
3903 logf("internal error!");
3904 close(fd);
3905 return false;
3906 }
3907
3908 if (idx->tag_seek[tag] != pos)
3909 {
3910 logf("corrupt data structures!");
3911 close(fd);
3912 return false;
3913 }
3914
3915 # ifdef HAVE_DIRCACHE
3916 if (dircache_is_enabled())
3917 {
3918 dc = dircache_get_entry_ptr(buf);
3919 if (dc == NULL)
3920 {
3921 logf("Entry no longer valid.");
3922 logf("-> %s", buf);
3923 delete_entry(fe->idx_id);
3924 continue ;
3925 }
3926
3927 idx->flag |= FLAG_DIRCACHE;
3928 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3929 idx->tag_seek[tag_filename] = (long)dc;
3930 }
3931 else
3932 # endif
3933 {
3934
3935 /* Enabled for flash based targets. Too slow otherwise. */
3936 # ifdef HAVE_FLASH_STORAGE
3937 /* Check if entry has been removed. */
3938 if (global_settings.tagcache_autoupdate)
3939 {
3940 int testfd;
3941
3942 testfd = open(buf, O_RDONLY);
3943 if (testfd < 0)
3944 {
3945 logf("Entry no longer valid.");
3946 logf("-> %s", buf);
3947 delete_entry(fe->idx_id);
3948 continue;
3949 }
3950 close(testfd);
3951 }
3952 # endif
3953 }
3954
3955 continue ;
3956 }
3957
3958 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3959 if (bytesleft < 0)
3960 {
3961 logf("too big tagcache #2");
3962 logf("tl: %d", fe->tag_length);
3963 logf("bl: %ld", bytesleft);
3964 close(fd);
3965 return false;
3966 }
3967
3968 p = fe->tag_data;
3969 rc = read(fd, fe->tag_data, fe->tag_length);
3970 p += rc;
3971
3972 if (rc != fe->tag_length)
3973 {
3974 logf("read error #13");
3975 logf("rc=0x%04x", rc); // 0x431
3976 logf("len=0x%04x", fe->tag_length); // 0x4000
3977 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3978 logf("tag=0x%02x", tag); // 0x00
3979 close(fd);
3980 return false;
3981 }
3982 }
3983 close(fd);
3984 }
3985
3986 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
3987 logf("tagcache loaded into ram!");
3988
3989 return true;
3990 }
3991 #endif /* HAVE_TC_RAMCACHE */
3992
3993 static bool check_deleted_files(void)
3994 {
3995 int fd, testfd;
3996 char buf[TAG_MAXLEN+32];
3997 struct tagfile_entry tfe;
3998
3999 logf("reverse scan...");
4000 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4001 fd = open(buf, O_RDONLY);
4002
4003 if (fd < 0)
4004 {
4005 logf("%s open fail", buf);
4006 return false;
4007 }
4008
4009 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4010 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4011 == sizeof(struct tagfile_entry)
4012 #ifndef __PCTOOL__
4013 && !check_event_queue()
4014 #endif
4015 )
4016 {
4017 if (tfe.tag_length >= (long)sizeof(buf)-1)
4018 {
4019 logf("too long tag");
4020 close(fd);
4021 return false;
4022 }
4023
4024 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4025 {
4026 logf("read error #14");
4027 close(fd);
4028 return false;
4029 }
4030
4031 /* Check if the file has already deleted from the db. */
4032 if (*buf == '\0')
4033 continue;
4034
4035 /* Now check if the file exists. */
4036 testfd = open(buf, O_RDONLY);
4037 if (testfd < 0)
4038 {
4039 logf("Entry no longer valid.");
4040 logf("-> %s / %d", buf, tfe.tag_length);
4041 delete_entry(tfe.idx_id);
4042 }
4043 close(testfd);
4044 }
4045
4046 close(fd);
4047
4048 logf("done");
4049
4050 return true;
4051 }
4052
4053 static bool check_dir(const char *dirname)
4054 {
4055 DIR *dir;
4056 int len;
4057 int success = false;
4058 char newpath[MAX_PATH];
4059
4060 dir = opendir(dirname);
4061 if (!dir)
4062 {
4063 logf("tagcache: opendir() failed");
4064 return false;
4065 }
4066
4067 /* check for a database.ignore file */
4068 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4069 if (file_exists(newpath))
4070 {
4071 closedir(dir);
4072 return false;
4073 }
4074
4075 /* Recursively scan the dir. */
4076 #ifdef __PCTOOL__
4077 while (1)
4078 #else
4079 while (!check_event_queue())
4080 #endif
4081 {
4082 struct dirent *entry;
4083
4084 entry = readdir(dir);
4085
4086 if (entry == NULL)
4087 {
4088 success = true;
4089 break ;
4090 }
4091
4092 if (!strcmp((char *)entry->d_name, ".") ||
4093 !strcmp((char *)entry->d_name, ".."))
4094 continue;
4095
4096 yield();
4097
4098 len = strlen(curpath);
4099 snprintf(&curpath[len], curpath_size - len, "/%s",
4100 entry->d_name);
4101
4102 processed_dir_count++;
4103 if (entry->attribute & ATTR_DIRECTORY)
4104 check_dir(curpath);
4105 else
4106 {
4107 tc_stat.curentry = curpath;
4108
4109 /* Add a new entry to the temporary db file. */
4110 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4111 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4112 , dir->internal_entry
4113 #endif
4114 );
4115
4116 /* Wait until current path for debug screen is read and unset. */
4117 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4118 yield();
4119
4120 tc_stat.curentry = NULL;
4121 }
4122
4123 curpath[len] = '\0';
4124 }
4125
4126 closedir(dir);
4127
4128 return success;
4129 }
4130
4131 void tagcache_screensync_event(void)
4132 {
4133 tc_stat.curentry = NULL;
4134 }
4135
4136 void tagcache_screensync_enable(bool state)
4137 {
4138 tc_stat.syncscreen = state;
4139 }
4140
4141 void build_tagcache(const char *path)
4142 {
4143 struct tagcache_header header;
4144 bool ret;
4145
4146 curpath[0] = '\0';
4147 data_size = 0;
4148 total_entry_count = 0;
4149 processed_dir_count = 0;
4150
4151 #ifdef HAVE_DIRCACHE
4152 while (dircache_is_initializing())
4153 sleep(1);
4154 #endif
4155
4156 logf("updating tagcache");
4157
4158 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4159 if (cachefd >= 0)
4160 {
4161 logf("skipping, cache already waiting for commit");
4162 close(cachefd);
4163 return ;
4164 }
4165
4166 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4167 if (cachefd < 0)
4168 {
4169 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4170 return ;
4171 }
4172
4173 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4174
4175 cpu_boost(true);
4176
4177 logf("Scanning files...");
4178 /* Scan for new files. */
4179 memset(&header, 0, sizeof(struct tagcache_header));
4180 write(cachefd, &header, sizeof(struct tagcache_header));
4181
4182 if (strcmp("/", path) != 0)
4183 strcpy(curpath, path);
4184 ret = check_dir(path);
4185
4186 /* Write the header. */
4187 header.magic = TAGCACHE_MAGIC;
4188 header.datasize = data_size;
4189 header.entry_count = total_entry_count;
4190 lseek(cachefd, 0, SEEK_SET);
4191 write(cachefd, &header, sizeof(struct tagcache_header));
4192 close(cachefd);
4193
4194 if (filenametag_fd >= 0)
4195 {
4196 close(filenametag_fd);
4197 filenametag_fd = -1;
4198 }
4199
4200 if (!ret)
4201 {
4202 logf("Aborted.");
4203 cpu_boost(false);
4204 return ;
4205 }
4206
4207 /* Commit changes to the database. */
4208 #ifdef __PCTOOL__
4209 allocate_tempbuf();
4210 #endif
4211 if (commit())
4212 {
4213 remove(TAGCACHE_FILE_TEMP);
4214 logf("tagcache built!");
4215 }
4216 #ifdef __PCTOOL__
4217 free_tempbuf();
4218 #endif
4219
4220 #ifdef HAVE_TC_RAMCACHE
4221 if (hdr)
4222 {
4223 /* Import runtime statistics if we just initialized the db. */
4224 if (hdr->h.serial == 0)
4225 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4226 }
4227 #endif
4228
4229 cpu_boost(false);
4230 }
4231
4232 #ifdef HAVE_TC_RAMCACHE
4233 static void load_ramcache(void)
4234 {
4235 if (!hdr)
4236 return ;
4237
4238 cpu_boost(true);
4239
4240 /* At first we should load the cache (if exists). */
4241 tc_stat.ramcache = load_tagcache();
4242
4243 if (!tc_stat.ramcache)
4244 {
4245 /* If loading failed, it must indicate some problem with the db
4246 * so disable it entirely to prevent further issues. */
4247 tc_stat.ready = false;
4248 hdr = NULL;
4249 }
4250
4251 cpu_boost(false);
4252 }
4253
4254 void tagcache_unload_ramcache(void)
4255 {
4256 tc_stat.ramcache = false;
4257 /* Just to make sure there is no statefile present. */
4258 // remove(TAGCACHE_STATEFILE);
4259 }
4260 #endif
4261
4262 #ifndef __PCTOOL__
4263 static void tagcache_thread(void)
4264 {
4265 struct queue_event ev;
4266 bool check_done = false;
4267
4268 /* If the previous cache build/update was interrupted, commit
4269 * the changes first in foreground. */
4270 cpu_boost(true);
4271 allocate_tempbuf();
4272 commit();
4273 free_tempbuf();
4274
4275 #ifdef HAVE_TC_RAMCACHE
4276 # ifdef HAVE_EEPROM_SETTINGS
4277 if (firmware_settings.initialized && firmware_settings.disk_clean)
4278 check_done = tagcache_dumpload();
4279
4280 remove(TAGCACHE_STATEFILE);
4281 # endif
4282
4283 /* Allocate space for the tagcache if found on disk. */
4284 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4285 allocate_tagcache();
4286 #endif
4287
4288 cpu_boost(false);
4289 tc_stat.initialized = true;
4290
4291 /* Don't delay bootup with the header check but do it on background. */
4292 sleep(HZ);
4293 tc_stat.ready = check_all_headers();
4294 tc_stat.readyvalid = true;
4295
4296 while (1)
4297 {
4298 run_command_queue(false);
4299
4300 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4301
4302 switch (ev.id)
4303 {
4304 case Q_IMPORT_CHANGELOG:
4305 tagcache_import_changelog();
4306 break;
4307
4308 case Q_REBUILD:
4309 remove_files();
4310 build_tagcache("/");
4311 break;
4312
4313 case Q_UPDATE:
4314 build_tagcache("/");
4315 #ifdef HAVE_TC_RAMCACHE
4316 load_ramcache();
4317 #endif
4318 check_deleted_files();
4319 break ;
4320
4321 case Q_START_SCAN:
4322 check_done = false;
4323 case SYS_TIMEOUT:
4324 if (check_done || !tc_stat.ready)
4325 break ;
4326
4327 #ifdef HAVE_TC_RAMCACHE
4328 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4329 {
4330 load_ramcache();
4331 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4332 build_tagcache("/");
4333 }
4334 else
4335 #endif
4336 if (global_settings.tagcache_autoupdate)
4337 {
4338 build_tagcache("/");
4339 /* Don't do auto removal without dircache or flash
4340 * storage (very slow). */
4341 #ifdef HAVE_FLASH_STORAGE
4342 check_deleted_files();
4343 #else
4344 # ifdef HAVE_DIRCACHE
4345 if (dircache_is_enabled())
4346 check_deleted_files();
4347 # endif
4348 #endif
4349 }
4350
4351 logf("tagcache check done");
4352
4353 check_done = true;
4354 break ;
4355
4356 case Q_STOP_SCAN:
4357 break ;
4358
4359 case SYS_POWEROFF:
4360 break ;
4361
4362 #ifndef SIMULATOR
4363 case SYS_USB_CONNECTED:
4364 logf("USB: TagCache");
4365 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4366 usb_wait_for_disconnect(&tagcache_queue);
4367 break ;
4368 #endif
4369 }
4370 }
4371 }
4372
4373 bool tagcache_prepare_shutdown(void)
4374 {
4375 if (tagcache_get_commit_step() > 0)
4376 return false;
4377
4378 tagcache_stop_scan();
4379 while (read_lock || write_lock)
4380 sleep(1);
4381
4382 return true;
4383 }
4384
4385 void tagcache_shutdown(void)
4386 {
4387 /* Flush the command queue. */
4388 run_command_queue(true);
4389
4390 #ifdef HAVE_EEPROM_SETTINGS
4391 if (tc_stat.ramcache)
4392 tagcache_dumpsave();
4393 #endif
4394 }
4395
4396 static int get_progress(void)
4397 {
4398 int total_count = -1;
4399
4400 #ifdef HAVE_DIRCACHE
4401 if (dircache_is_enabled())
4402 {
4403 total_count = dircache_get_entry_count();
4404 }
4405 else
4406 #endif
4407 #ifdef HAVE_TC_RAMCACHE
4408 {
4409 if (hdr && tc_stat.ramcache)
4410 total_count = hdr->h.tch.entry_count;
4411 }
4412 #endif
4413
4414 if (total_count < 0)
4415 return -1;
4416
4417 return processed_dir_count * 100 / total_count;
4418 }
4419
4420 struct tagcache_stat* tagcache_get_stat(void)
4421 {
4422 tc_stat.progress = get_progress();
4423 tc_stat.processed_entries = processed_dir_count;
4424
4425 return &tc_stat;
4426 }
4427
4428 void tagcache_start_scan(void)
4429 {
4430 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4431 }
4432
4433 bool tagcache_update(void)
4434 {
4435 if (!tc_stat.ready)
4436 return false;
4437
4438 queue_post(&tagcache_queue, Q_UPDATE, 0);
4439 return false;
4440 }
4441
4442 bool tagcache_rebuild()
4443 {
4444 queue_post(&tagcache_queue, Q_REBUILD, 0);
4445 return false;
4446 }
4447
4448 void tagcache_stop_scan(void)
4449 {
4450 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4451 }
4452
4453 #ifdef HAVE_TC_RAMCACHE
4454 bool tagcache_is_ramcache(void)
4455 {
4456 return tc_stat.ramcache;
4457 }
4458 #endif
4459
4460 #endif /* !__PCTOOL__ */
4461
4462
4463 void tagcache_init(void)
4464 {
4465 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4466 memset(&current_tcmh, 0, sizeof(struct master_header));
4467 filenametag_fd = -1;
4468 write_lock = read_lock = 0;
4469
4470 #ifndef __PCTOOL__
4471 mutex_init(&command_queue_mutex);
4472 queue_init(&tagcache_queue, true);
4473 create_thread(tagcache_thread, tagcache_stack,
4474 sizeof(tagcache_stack), 0, tagcache_thread_name
4475 IF_PRIO(, PRIORITY_BACKGROUND)
4476 IF_COP(, CPU));
4477 #else
4478 tc_stat.initialized = true;
4479 allocate_tempbuf();
4480 commit();
4481 free_tempbuf();
4482 tc_stat.ready = check_all_headers();
4483 #endif
4484 }
4485
4486 #ifdef __PCTOOL__
4487 void tagcache_reverse_scan(void)
4488 {
4489 logf("Checking for deleted files");
4490 check_deleted_files();
4491 }
4492 #endif
4493
4494 bool tagcache_is_initialized(void)
4495 {
4496 return tc_stat.initialized;
4497 }
4498 bool tagcache_is_usable(void)
4499 {
4500 return tc_stat.initialized && tc_stat.ready;
4501 }
4502 int tagcache_get_commit_step(void)
4503 {
4504 return tc_stat.commit_step;
4505 }
4506 int tagcache_get_max_commit_step(void)
4507 {
4508 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4509 }
4510
Open Source Jukebox Firmware