?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add 2008 to the copyright notice.
[Rockbox.git] / apps / tagcache.c
blob926ba7b3c7a48eab0951f5f410ab40f2e3fdb91f
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "ata_idle_notify.h"
61 #include "thread.h"
62 #include "kernel.h"
63 #include "system.h"
64 #include "logf.h"
65 #include "string.h"
66 #include "usb.h"
67 #include "metadata.h"
68 #include "id3.h"
69 #include "tagcache.h"
70 #include "buffer.h"
71 #include "crc32.h"
72 #include "misc.h"
73 #include "settings.h"
74 #include "dir.h"
75 #include "structec.h"
76 #ifndef __PCTOOL__
77 #include "atoi.h"
78 #include "splash.h"
79 #include "lang.h"
80 #include "eeprom_settings.h"
81 #endif
82
83 #ifdef __PCTOOL__
84 #define yield() do { } while(0)
85 #define sim_sleep(timeout) do { } while(0)
86 #define do_timed_yield() do { } while(0)
87 #endif
88
89 #ifndef __PCTOOL__
90 /* Tag Cache thread. */
91 static struct event_queue tagcache_queue;
92 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
93 static const char tagcache_thread_name[] = "tagcache";
94 #endif
95
96 #define UNTAGGED "<Untagged>"
97
98 /* Previous path when scanning directory tree recursively. */
99 static char curpath[TAG_MAXLEN+32];
100 static long curpath_size = sizeof(curpath);
101
102 /* Used when removing duplicates. */
103 static char *tempbuf; /* Allocated when needed. */
104 static long tempbufidx; /* Current location in buffer. */
105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
106 static long tempbuf_left; /* Buffer space left. */
107 static long tempbuf_pos;
108
109 /* Tags we want to get sorted (loaded to the tempbuf). */
110 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
111 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
112
113 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
114 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
115 tag_composer, tag_comment, tag_albumartist, tag_grouping };
116
117 /* Numeric tags (we can use these tags with conditional clauses). */
118 static const int numeric_tags[] = { tag_year, tag_discnumber,
119 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
120 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
121 tag_virt_length_min, tag_virt_length_sec,
122 tag_virt_playtime_min, tag_virt_playtime_sec,
123 tag_virt_entryage, tag_virt_autoscore };
124
125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
126 static const char *tags_str[] = { "artist", "album", "genre", "title",
127 "filename", "composer", "comment", "albumartist", "grouping", "year",
128 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
129 "playtime", "lastplayed", "commitid", "mtime" };
130
131 /* Status information of the tagcache. */
132 static struct tagcache_stat tc_stat;
133
134 /* Queue commands. */
135 enum tagcache_queue {
136 Q_STOP_SCAN = 0,
137 Q_START_SCAN,
138 Q_IMPORT_CHANGELOG,
139 Q_UPDATE,
140 Q_REBUILD,
141
142 /* Internal tagcache command queue. */
143 CMD_UPDATE_MASTER_HEADER,
144 CMD_UPDATE_NUMERIC,
145 };
146
147 struct tagcache_command_entry {
148 long command;
149 long idx_id;
150 long tag;
151 long data;
152 };
153
154 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
155 static volatile int command_queue_widx = 0;
156 static volatile int command_queue_ridx = 0;
157 static struct mutex command_queue_mutex;
158
159 /* Tag database structures. */
160
161 /* Variable-length tag entry in tag files. */
162 struct tagfile_entry {
163 short tag_length; /* Length of the data in bytes including '\0' */
164 short idx_id; /* Corresponding entry location in index file of not unique tags */
165 char tag_data[0]; /* Begin of the tag data */
166 };
167
168 /* Fixed-size tag entry in master db index. */
169 struct index_entry {
170 long tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
171 long flag; /* Status flags */
172 };
173
174 /* Header is the same in every file. */
175 struct tagcache_header {
176 long magic; /* Header version number */
177 long datasize; /* Data size in bytes */
178 long entry_count; /* Number of entries in this file */
179 };
180
181 struct master_header {
182 struct tagcache_header tch;
183 long serial; /* Increasing counting number */
184 long commitid; /* Number of commits so far */
185 long dirty;
186 };
187
188 /* For the endianess correction */
189 static const char *tagfile_entry_ec = "ss";
190 static const char *index_entry_ec = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
191 static const char *tagcache_header_ec = "lll";
192 static const char *master_header_ec = "llllll";
193
194 static struct master_header current_tcmh;
195
196 #ifdef HAVE_TC_RAMCACHE
197 /* Header is created when loading database to ram. */
198 struct ramcache_header {
199 struct master_header h; /* Header from the master index */
200 struct index_entry *indices; /* Master index file content */
201 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
202 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
203 };
204
205 # ifdef HAVE_EEPROM_SETTINGS
206 struct statefile_header {
207 struct ramcache_header *hdr;
208 struct tagcache_stat tc_stat;
209 };
210 # endif
211
212 /* Pointer to allocated ramcache_header */
213 static struct ramcache_header *hdr;
214 #endif
215
216 /**
217 * Full tag entries stored in a temporary file waiting
218 * for commit to the cache. */
219 struct temp_file_entry {
220 long tag_offset[TAG_COUNT];
221 short tag_length[TAG_COUNT];
222 long flag;
223
224 long data_length;
225 };
226
227 struct tempbuf_id_list {
228 long id;
229 struct tempbuf_id_list *next;
230 };
231
232 struct tempbuf_searchidx {
233 long idx_id;
234 char *str;
235 int seek;
236 struct tempbuf_id_list idlist;
237 };
238
239 /* Lookup buffer for fixing messed up index while after sorting. */
240 static long commit_entry_count;
241 static long lookup_buffer_depth;
242 static struct tempbuf_searchidx **lookup;
243
244 /* Used when building the temporary file. */
245 static int cachefd = -1, filenametag_fd;
246 static int total_entry_count = 0;
247 static int data_size = 0;
248 static int processed_dir_count;
249
250 /* Thread safe locking */
251 static volatile int write_lock;
252 static volatile int read_lock;
253
254 static bool delete_entry(long idx_id);
255
256 const char* tagcache_tag_to_str(int tag)
257 {
258 return tags_str[tag];
259 }
260
261 bool tagcache_is_numeric_tag(int type)
262 {
263 int i;
264
265 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
266 {
267 if (type == numeric_tags[i])
268 return true;
269 }
270
271 return false;
272 }
273
274 bool tagcache_is_unique_tag(int type)
275 {
276 int i;
277
278 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
279 {
280 if (type == unique_tags[i])
281 return true;
282 }
283
284 return false;
285 }
286
287 bool tagcache_is_sorted_tag(int type)
288 {
289 int i;
290
291 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
292 {
293 if (type == sorted_tags[i])
294 return true;
295 }
296
297 return false;
298 }
299
300 #ifdef HAVE_DIRCACHE
301 /**
302 * Returns true if specified flag is still present, i.e., dircache
303 * has not been reloaded.
304 */
305 static bool is_dircache_intact(void)
306 {
307 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
308 }
309 #endif
310
311 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
312 {
313 int fd;
314 char buf[MAX_PATH];
315 int rc;
316
317 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
318 return -1;
319
320 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
321
322 fd = open(buf, write ? O_RDWR : O_RDONLY);
323 if (fd < 0)
324 {
325 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
326 tc_stat.ready = false;
327 return fd;
328 }
329
330 /* Check the header. */
331 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
332 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
333 {
334 logf("header error");
335 tc_stat.ready = false;
336 close(fd);
337 return -2;
338 }
339
340 return fd;
341 }
342
343 static int open_master_fd(struct master_header *hdr, bool write)
344 {
345 int fd;
346 int rc;
347
348 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
349 if (fd < 0)
350 {
351 logf("master file open failed for R/W");
352 tc_stat.ready = false;
353 return fd;
354 }
355
356 tc_stat.econ = false;
357
358 /* Check the header. */
359 rc = read(fd, hdr, sizeof(struct master_header));
360 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
361 {
362 /* Success. */
363 return fd;
364 }
365
366 /* Trying to read again, this time with endianess correction enabled. */
367 lseek(fd, 0, SEEK_SET);
368
369 rc = ecread(fd, hdr, 1, master_header_ec, true);
370 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
371 {
372 logf("header error");
373 tc_stat.ready = false;
374 close(fd);
375 return -2;
376 }
377
378 tc_stat.econ = true;
379
380 return fd;
381 }
382
383 #ifndef __PCTOOL__
384 static bool do_timed_yield(void)
385 {
386 /* Sorting can lock up for quite a while, so yield occasionally */
387 static long wakeup_tick = 0;
388 if (current_tick >= wakeup_tick)
389 {
390 wakeup_tick = current_tick + (HZ/4);
391 yield();
392 return true;
393 }
394 return false;
395 }
396 #endif
397
398 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
399 static long find_entry_ram(const char *filename,
400 const struct dirent *dc)
401 {
402 static long last_pos = 0;
403 int i;
404
405 /* Check if we tagcache is loaded into ram. */
406 if (!tc_stat.ramcache)
407 return -1;
408
409 if (dc == NULL)
410 dc = dircache_get_entry_ptr(filename);
411
412 if (dc == NULL)
413 {
414 logf("tagcache: file not found.");
415 return -1;
416 }
417
418 try_again:
419
420 if (last_pos > 0)
421 i = last_pos;
422 else
423 i = 0;
424
425 for (; i < hdr->h.tch.entry_count; i++)
426 {
427 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
428 {
429 last_pos = MAX(0, i - 3);
430 return i;
431 }
432
433 do_timed_yield();
434 }
435
436 if (last_pos > 0)
437 {
438 last_pos = 0;
439 goto try_again;
440 }
441
442 return -1;
443 }
444 #endif
445
446 static long find_entry_disk(const char *filename)
447 {
448 struct tagcache_header tch;
449 static long last_pos = -1;
450 long pos_history[POS_HISTORY_COUNT];
451 long pos_history_idx = 0;
452 bool found = false;
453 struct tagfile_entry tfe;
454 int fd;
455 char buf[TAG_MAXLEN+32];
456 int i;
457 int pos = -1;
458
459 if (!tc_stat.ready)
460 return -2;
461
462 fd = filenametag_fd;
463 if (fd < 0)
464 {
465 last_pos = -1;
466 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
467 return -1;
468 }
469
470 check_again:
471
472 if (last_pos > 0)
473 lseek(fd, last_pos, SEEK_SET);
474 else
475 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
476
477 while (true)
478 {
479 pos = lseek(fd, 0, SEEK_CUR);
480 for (i = pos_history_idx-1; i >= 0; i--)
481 pos_history[i+1] = pos_history[i];
482 pos_history[0] = pos;
483
484 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
485 != sizeof(struct tagfile_entry))
486 {
487 break ;
488 }
489
490 if (tfe.tag_length >= (long)sizeof(buf))
491 {
492 logf("too long tag #1");
493 close(fd);
494 last_pos = -1;
495 return -2;
496 }
497
498 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
499 {
500 logf("read error #2");
501 close(fd);
502 last_pos = -1;
503 return -3;
504 }
505
506 if (!strcasecmp(filename, buf))
507 {
508 last_pos = pos_history[pos_history_idx];
509 found = true;
510 break ;
511 }
512
513 if (pos_history_idx < POS_HISTORY_COUNT - 1)
514 pos_history_idx++;
515 }
516
517 /* Not found? */
518 if (!found)
519 {
520 if (last_pos > 0)
521 {
522 last_pos = -1;
523 logf("seek again");
524 goto check_again;
525 }
526
527 if (fd != filenametag_fd)
528 close(fd);
529 return -4;
530 }
531
532 if (fd != filenametag_fd)
533 close(fd);
534
535 return tfe.idx_id;
536 }
537
538 static int find_index(const char *filename)
539 {
540 long idx_id = -1;
541
542 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
543 if (tc_stat.ramcache && is_dircache_intact())
544 idx_id = find_entry_ram(filename, NULL);
545 #endif
546
547 if (idx_id < 0)
548 idx_id = find_entry_disk(filename);
549
550 return idx_id;
551 }
552
553 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
554 {
555 int idx_id;
556
557 if (!tc_stat.ready)
558 return false;
559
560 idx_id = find_index(filename);
561 if (idx_id < 0)
562 return false;
563
564 if (!tagcache_search(tcs, tag_filename))
565 return false;
566
567 tcs->entry_count = 0;
568 tcs->idx_id = idx_id;
569
570 return true;
571 }
572
573 static bool get_index(int masterfd, int idxid,
574 struct index_entry *idx, bool use_ram)
575 {
576 bool localfd = false;
577
578 if (idxid < 0)
579 {
580 logf("Incorrect idxid: %d", idxid);
581 return false;
582 }
583
584 #ifdef HAVE_TC_RAMCACHE
585 if (tc_stat.ramcache && use_ram)
586 {
587 if (hdr->indices[idxid].flag & FLAG_DELETED)
588 return false;
589
590 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
591 return true;
592 }
593 #else
594 (void)use_ram;
595 #endif
596
597 if (masterfd < 0)
598 {
599 struct master_header tcmh;
600
601 localfd = true;
602 masterfd = open_master_fd(&tcmh, false);
603 if (masterfd < 0)
604 return false;
605 }
606
607 lseek(masterfd, idxid * sizeof(struct index_entry)
608 + sizeof(struct master_header), SEEK_SET);
609 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
610 != sizeof(struct index_entry))
611 {
612 logf("read error #3");
613 if (localfd)
614 close(masterfd);
615
616 return false;
617 }
618
619 if (localfd)
620 close(masterfd);
621
622 if (idx->flag & FLAG_DELETED)
623 return false;
624
625 return true;
626 }
627
628 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
629 {
630 /* We need to exclude all memory only flags & tags when writing to disk. */
631 if (idx->flag & FLAG_DIRCACHE)
632 {
633 logf("memory only flags!");
634 return false;
635 }
636
637 #ifdef HAVE_TC_RAMCACHE
638 /* Only update numeric data. Writing the whole index to RAM by memcpy
639 * destroys dircache pointers!
640 */
641 if (tc_stat.ramcache)
642 {
643 int tag;
644 struct index_entry *idx_ram = &hdr->indices[idxid];
645
646 for (tag = 0; tag < TAG_COUNT; tag++)
647 {
648 if (tagcache_is_numeric_tag(tag))
649 {
650 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
651 }
652 }
653
654 /* Don't touch the dircache flag. */
655 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
656 }
657 #endif
658
659 lseek(masterfd, idxid * sizeof(struct index_entry)
660 + sizeof(struct master_header), SEEK_SET);
661 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
662 != sizeof(struct index_entry))
663 {
664 logf("write error #3");
665 logf("idxid: %d", idxid);
666 return false;
667 }
668
669 return true;
670 }
671
672 static bool open_files(struct tagcache_search *tcs, int tag)
673 {
674 if (tcs->idxfd[tag] < 0)
675 {
676 char fn[MAX_PATH];
677
678 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
679 tcs->idxfd[tag] = open(fn, O_RDONLY);
680 }
681
682 if (tcs->idxfd[tag] < 0)
683 {
684 logf("File not open!");
685 return false;
686 }
687
688 return true;
689 }
690
691 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
692 int tag, char *buf, long size)
693 {
694 struct tagfile_entry tfe;
695 long seek;
696
697 *buf = '\0';
698
699 if (tagcache_is_numeric_tag(tag))
700 return false;
701
702 seek = idx->tag_seek[tag];
703 if (seek < 0)
704 {
705 logf("Retrieve failed");
706 return false;
707 }
708
709 #ifdef HAVE_TC_RAMCACHE
710 if (tcs->ramsearch)
711 {
712 struct tagfile_entry *ep;
713
714 # ifdef HAVE_DIRCACHE
715 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
716 && is_dircache_intact())
717 {
718 dircache_copy_path((struct dirent *)seek,
719 buf, size);
720 return true;
721 }
722 else
723 # endif
724 if (tag != tag_filename)
725 {
726 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
727 strncpy(buf, ep->tag_data, size-1);
728
729 return true;
730 }
731 }
732 #endif
733
734 if (!open_files(tcs, tag))
735 return false;
736
737 lseek(tcs->idxfd[tag], seek, SEEK_SET);
738 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
739 != sizeof(struct tagfile_entry))
740 {
741 logf("read error #5");
742 return false;
743 }
744
745 if (tfe.tag_length >= size)
746 {
747 logf("too small buffer");
748 return false;
749 }
750
751 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
752 tfe.tag_length)
753 {
754 logf("read error #6");
755 return false;
756 }
757
758 buf[tfe.tag_length] = '\0';
759
760 return true;
761 }
762
763 static long check_virtual_tags(int tag, const struct index_entry *idx)
764 {
765 long data = 0;
766
767 switch (tag)
768 {
769 case tag_virt_length_sec:
770 data = (idx->tag_seek[tag_length]/1000) % 60;
771 break;
772
773 case tag_virt_length_min:
774 data = (idx->tag_seek[tag_length]/1000) / 60;
775 break;
776
777 case tag_virt_playtime_sec:
778 data = (idx->tag_seek[tag_playtime]/1000) % 60;
779 break;
780
781 case tag_virt_playtime_min:
782 data = (idx->tag_seek[tag_playtime]/1000) / 60;
783 break;
784
785 case tag_virt_autoscore:
786 if (idx->tag_seek[tag_length] == 0
787 || idx->tag_seek[tag_playcount] == 0)
788 {
789 data = 0;
790 }
791 else
792 {
793 data = 100 * idx->tag_seek[tag_playtime]
794 / idx->tag_seek[tag_length]
795 / idx->tag_seek[tag_playcount];
796 }
797 break;
798
799 /* How many commits before the file has been added to the DB. */
800 case tag_virt_entryage:
801 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
802 break;
803
804 default:
805 data = idx->tag_seek[tag];
806 }
807
808 return data;
809 }
810
811 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
812 {
813 struct index_entry idx;
814
815 if (!tc_stat.ready)
816 return false;
817
818 if (!tagcache_is_numeric_tag(tag))
819 return -1;
820
821 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
822 return -2;
823
824 return check_virtual_tags(tag, &idx);
825 }
826
827 inline static bool str_ends_with(const char *str1, const char *str2)
828 {
829 int str_len = strlen(str1);
830 int clause_len = strlen(str2);
831
832 if (clause_len > str_len)
833 return false;
834
835 return !strcasecmp(&str1[str_len - clause_len], str2);
836 }
837
838 inline static bool str_oneof(const char *str, const char *list)
839 {
840 const char *sep;
841 int l, len = strlen(str);
842
843 while (*list)
844 {
845 sep = strchr(list, '|');
846 l = sep ? (long)sep - (long)list : (int)strlen(list);
847 if ((l==len) && !strncasecmp(str, list, len))
848 return true;
849 list += sep ? l + 1 : l;
850 }
851
852 return false;
853 }
854
855 static bool check_against_clause(long numeric, const char *str,
856 const struct tagcache_search_clause *clause)
857 {
858 if (clause->numeric)
859 {
860 switch (clause->type)
861 {
862 case clause_is:
863 return numeric == clause->numeric_data;
864 case clause_is_not:
865 return numeric != clause->numeric_data;
866 case clause_gt:
867 return numeric > clause->numeric_data;
868 case clause_gteq:
869 return numeric >= clause->numeric_data;
870 case clause_lt:
871 return numeric < clause->numeric_data;
872 case clause_lteq:
873 return numeric <= clause->numeric_data;
874 default:
875 logf("Incorrect numeric tag: %d", clause->type);
876 }
877 }
878 else
879 {
880 switch (clause->type)
881 {
882 case clause_is:
883 return !strcasecmp(clause->str, str);
884 case clause_is_not:
885 return strcasecmp(clause->str, str);
886 case clause_gt:
887 return 0>strcasecmp(clause->str, str);
888 case clause_gteq:
889 return 0>=strcasecmp(clause->str, str);
890 case clause_lt:
891 return 0<strcasecmp(clause->str, str);
892 case clause_lteq:
893 return 0<=strcasecmp(clause->str, str);
894 case clause_contains:
895 return (strcasestr(str, clause->str) != NULL);
896 case clause_not_contains:
897 return (strcasestr(str, clause->str) == NULL);
898 case clause_begins_with:
899 return (strcasestr(str, clause->str) == str);
900 case clause_not_begins_with:
901 return (strcasestr(str, clause->str) != str);
902 case clause_ends_with:
903 return str_ends_with(str, clause->str);
904 case clause_not_ends_with:
905 return !str_ends_with(str, clause->str);
906 case clause_oneof:
907 return str_oneof(str, clause->str);
908
909 default:
910 logf("Incorrect tag: %d", clause->type);
911 }
912 }
913
914 return false;
915 }
916
917 static bool check_clauses(struct tagcache_search *tcs,
918 struct index_entry *idx,
919 struct tagcache_search_clause **clause, int count)
920 {
921 int i;
922
923 #ifdef HAVE_TC_RAMCACHE
924 if (tcs->ramsearch)
925 {
926 /* Go through all conditional clauses. */
927 for (i = 0; i < count; i++)
928 {
929 struct tagfile_entry *tfe;
930 int seek;
931 char buf[256];
932 char *str = NULL;
933
934 seek = check_virtual_tags(clause[i]->tag, idx);
935
936 if (!tagcache_is_numeric_tag(clause[i]->tag))
937 {
938 if (clause[i]->tag == tag_filename)
939 {
940 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
941 str = buf;
942 }
943 else
944 {
945 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
946 str = tfe->tag_data;
947 }
948 }
949
950 if (!check_against_clause(seek, str, clause[i]))
951 return false;
952 }
953 }
954 else
955 #endif
956 {
957 /* Check for conditions. */
958 for (i = 0; i < count; i++)
959 {
960 struct tagfile_entry tfe;
961 int seek;
962 char str[256];
963
964 seek = check_virtual_tags(clause[i]->tag, idx);
965
966 memset(str, 0, sizeof str);
967 if (!tagcache_is_numeric_tag(clause[i]->tag))
968 {
969 int fd = tcs->idxfd[clause[i]->tag];
970 lseek(fd, seek, SEEK_SET);
971 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
972 if (tfe.tag_length >= (int)sizeof(str))
973 {
974 logf("Too long tag read!");
975 break ;
976 }
977
978 read(fd, str, tfe.tag_length);
979
980 /* Check if entry has been deleted. */
981 if (str[0] == '\0')
982 break;
983 }
984
985 if (!check_against_clause(seek, str, clause[i]))
986 return false;
987 }
988 }
989
990 return true;
991 }
992
993 bool tagcache_check_clauses(struct tagcache_search *tcs,
994 struct tagcache_search_clause **clause, int count)
995 {
996 struct index_entry idx;
997
998 if (count == 0)
999 return true;
1000
1001 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1002 return false;
1003
1004 return check_clauses(tcs, &idx, clause, count);
1005 }
1006
1007 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1008 {
1009 int i;
1010
1011 /* If uniq buffer is not defined we must return true for search to work. */
1012 if (tcs->unique_list == NULL
1013 || (!tagcache_is_unique_tag(tcs->type)
1014 && !tagcache_is_numeric_tag(tcs->type)))
1015 {
1016 return true;
1017 }
1018
1019 for (i = 0; i < tcs->unique_list_count; i++)
1020 {
1021 /* Return false if entry is found. */
1022 if (tcs->unique_list[i] == id)
1023 return false;
1024 }
1025
1026 if (tcs->unique_list_count < tcs->unique_list_capacity)
1027 {
1028 tcs->unique_list[i] = id;
1029 tcs->unique_list_count++;
1030 }
1031
1032 return true;
1033 }
1034
1035 static bool build_lookup_list(struct tagcache_search *tcs)
1036 {
1037 struct index_entry entry;
1038 int i;
1039
1040 tcs->seek_list_count = 0;
1041
1042 #ifdef HAVE_TC_RAMCACHE
1043 if (tcs->ramsearch)
1044 {
1045 int j;
1046
1047 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1048 {
1049 struct index_entry *idx = &hdr->indices[i];
1050 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1051 break ;
1052
1053 /* Skip deleted files. */
1054 if (idx->flag & FLAG_DELETED)
1055 continue;
1056
1057 /* Go through all filters.. */
1058 for (j = 0; j < tcs->filter_count; j++)
1059 {
1060 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1061 {
1062 break ;
1063 }
1064 }
1065
1066 if (j < tcs->filter_count)
1067 continue ;
1068
1069 /* Check for conditions. */
1070 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1071 continue;
1072
1073 /* Add to the seek list if not already in uniq buffer. */
1074 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1075 continue;
1076
1077 /* Lets add it. */
1078 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1079 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1080 tcs->seek_list_count++;
1081 }
1082
1083 tcs->seek_pos = i;
1084
1085 return tcs->seek_list_count > 0;
1086 }
1087 #endif
1088
1089 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1090 sizeof(struct master_header), SEEK_SET);
1091
1092 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1093 == sizeof(struct index_entry))
1094 {
1095 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1096 break ;
1097
1098 tcs->seek_pos++;
1099
1100 /* Check if entry has been deleted. */
1101 if (entry.flag & FLAG_DELETED)
1102 continue;
1103
1104 /* Go through all filters.. */
1105 for (i = 0; i < tcs->filter_count; i++)
1106 {
1107 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1108 break ;
1109 }
1110
1111 if (i < tcs->filter_count)
1112 continue ;
1113
1114 /* Check for conditions. */
1115 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1116 continue;
1117
1118 /* Add to the seek list if not already in uniq buffer. */
1119 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1120 continue;
1121
1122 /* Lets add it. */
1123 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1124 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1125 tcs->seek_list_count++;
1126
1127 yield();
1128 }
1129
1130 return tcs->seek_list_count > 0;
1131 }
1132
1133
1134 static void remove_files(void)
1135 {
1136 int i;
1137 char buf[MAX_PATH];
1138
1139 tc_stat.ready = false;
1140 tc_stat.ramcache = false;
1141 tc_stat.econ = false;
1142 remove(TAGCACHE_FILE_MASTER);
1143 for (i = 0; i < TAG_COUNT; i++)
1144 {
1145 if (tagcache_is_numeric_tag(i))
1146 continue;
1147
1148 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1149 remove(buf);
1150 }
1151 }
1152
1153
1154 static bool check_all_headers(void)
1155 {
1156 struct master_header myhdr;
1157 struct tagcache_header tch;
1158 int tag;
1159 int fd;
1160
1161 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1162 return false;
1163
1164 close(fd);
1165 if (myhdr.dirty)
1166 {
1167 logf("tagcache is dirty!");
1168 return false;
1169 }
1170
1171 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1172
1173 for (tag = 0; tag < TAG_COUNT; tag++)
1174 {
1175 if (tagcache_is_numeric_tag(tag))
1176 continue;
1177
1178 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1179 return false;
1180
1181 close(fd);
1182 }
1183
1184 return true;
1185 }
1186
1187 bool tagcache_search(struct tagcache_search *tcs, int tag)
1188 {
1189 struct tagcache_header tag_hdr;
1190 struct master_header master_hdr;
1191 int i;
1192
1193 if (tcs->initialized)
1194 tagcache_search_finish(tcs);
1195
1196 while (read_lock)
1197 sleep(1);
1198
1199 memset(tcs, 0, sizeof(struct tagcache_search));
1200 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1201 return false;
1202
1203 tcs->position = sizeof(struct tagcache_header);
1204 tcs->type = tag;
1205 tcs->seek_pos = 0;
1206 tcs->seek_list_count = 0;
1207 tcs->filter_count = 0;
1208 tcs->masterfd = -1;
1209
1210 for (i = 0; i < TAG_COUNT; i++)
1211 tcs->idxfd[i] = -1;
1212
1213 #ifndef HAVE_TC_RAMCACHE
1214 tcs->ramsearch = false;
1215 #else
1216 tcs->ramsearch = tc_stat.ramcache;
1217 if (tcs->ramsearch)
1218 {
1219 tcs->entry_count = hdr->entry_count[tcs->type];
1220 }
1221 else
1222 #endif
1223 {
1224 if (!tagcache_is_numeric_tag(tcs->type))
1225 {
1226 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1227 if (tcs->idxfd[tcs->type] < 0)
1228 return false;
1229 }
1230
1231 /* Always open as R/W so we can pass tcs to functions that modify data also
1232 * without failing. */
1233 tcs->masterfd = open_master_fd(&master_hdr, true);
1234
1235 if (tcs->masterfd < 0)
1236 return false;
1237 }
1238
1239 tcs->valid = true;
1240 tcs->initialized = true;
1241 write_lock++;
1242
1243 return true;
1244 }
1245
1246 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1247 void *buffer, long length)
1248 {
1249 tcs->unique_list = (unsigned long *)buffer;
1250 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1251 tcs->unique_list_count = 0;
1252 }
1253
1254 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1255 int tag, int seek)
1256 {
1257 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1258 return false;
1259
1260 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1261 return false;
1262
1263 tcs->filter_tag[tcs->filter_count] = tag;
1264 tcs->filter_seek[tcs->filter_count] = seek;
1265 tcs->filter_count++;
1266
1267 return true;
1268 }
1269
1270 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1271 struct tagcache_search_clause *clause)
1272 {
1273 int i;
1274
1275 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1276 {
1277 logf("Too many clauses");
1278 return false;
1279 }
1280
1281 /* Check if there is already a similar filter in present (filters are
1282 * much faster than clauses).
1283 */
1284 for (i = 0; i < tcs->filter_count; i++)
1285 {
1286 if (tcs->filter_tag[i] == clause->tag)
1287 return true;
1288 }
1289
1290 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1291 {
1292 char buf[MAX_PATH];
1293
1294 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1295 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1296 }
1297
1298 tcs->clause[tcs->clause_count] = clause;
1299 tcs->clause_count++;
1300
1301 return true;
1302 }
1303
1304 /* TODO: Remove this mess. */
1305 #ifdef HAVE_DIRCACHE
1306 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1307 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1308 #else
1309 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1310 #endif
1311
1312 static bool get_next(struct tagcache_search *tcs)
1313 {
1314 static char buf[TAG_MAXLEN+32];
1315 struct tagfile_entry entry;
1316 long flag = 0;
1317
1318 if (!tcs->valid || !tc_stat.ready)
1319 return false;
1320
1321 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1322 #ifdef HAVE_TC_RAMCACHE
1323 && !tcs->ramsearch
1324 #endif
1325 )
1326 return false;
1327
1328 /* Relative fetch. */
1329 if (tcs->filter_count > 0 || tcs->clause_count > 0
1330 || tagcache_is_numeric_tag(tcs->type))
1331 {
1332 /* Check for end of list. */
1333 if (tcs->seek_list_count == 0)
1334 {
1335 /* Try to fetch more. */
1336 if (!build_lookup_list(tcs))
1337 {
1338 tcs->valid = false;
1339 return false;
1340 }
1341 }
1342
1343 tcs->seek_list_count--;
1344 flag = tcs->seek_flags[tcs->seek_list_count];
1345
1346 /* Seek stream to the correct position and continue to direct fetch. */
1347 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1348 && !tagcache_is_numeric_tag(tcs->type))
1349 {
1350 if (!open_files(tcs, tcs->type))
1351 return false;
1352
1353 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1354 }
1355 else
1356 tcs->position = tcs->seek_list[tcs->seek_list_count];
1357 }
1358
1359 if (tagcache_is_numeric_tag(tcs->type))
1360 {
1361 snprintf(buf, sizeof(buf), "%d", tcs->position);
1362 tcs->result_seek = tcs->position;
1363 tcs->result = buf;
1364 tcs->result_len = strlen(buf) + 1;
1365 return true;
1366 }
1367
1368 /* Direct fetch. */
1369 #ifdef HAVE_TC_RAMCACHE
1370 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1371 {
1372 struct tagfile_entry *ep;
1373
1374 if (tcs->entry_count == 0)
1375 {
1376 tcs->valid = false;
1377 return false;
1378 }
1379 tcs->entry_count--;
1380
1381 tcs->result_seek = tcs->position;
1382
1383 # ifdef HAVE_DIRCACHE
1384 if (tcs->type == tag_filename)
1385 {
1386 dircache_copy_path((struct dirent *)tcs->position,
1387 buf, sizeof buf);
1388 tcs->result = buf;
1389 tcs->result_len = strlen(buf) + 1;
1390 tcs->idx_id = FLAG_GET_ATTR(flag);
1391 tcs->ramresult = false;
1392
1393 return true;
1394 }
1395 # endif
1396
1397 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1398 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1399 tcs->result = ep->tag_data;
1400 tcs->result_len = strlen(tcs->result) + 1;
1401 tcs->idx_id = ep->idx_id;
1402 tcs->ramresult = true;
1403
1404 return true;
1405 }
1406 else
1407 #endif
1408 {
1409 if (!open_files(tcs, tcs->type))
1410 return false;
1411
1412 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1413 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1414 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1415 {
1416 /* End of data. */
1417 tcs->valid = false;
1418 return false;
1419 }
1420 }
1421
1422 if (entry.tag_length > (long)sizeof(buf))
1423 {
1424 tcs->valid = false;
1425 logf("too long tag #2");
1426 return false;
1427 }
1428
1429 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1430 {
1431 tcs->valid = false;
1432 logf("read error #4");
1433 return false;
1434 }
1435
1436 tcs->result = buf;
1437 tcs->result_len = strlen(tcs->result) + 1;
1438 tcs->idx_id = entry.idx_id;
1439 tcs->ramresult = false;
1440
1441 return true;
1442 }
1443
1444 bool tagcache_get_next(struct tagcache_search *tcs)
1445 {
1446 while (get_next(tcs))
1447 {
1448 if (tcs->result_len > 1)
1449 return true;
1450 }
1451
1452 return false;
1453 }
1454
1455 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1456 int tag, char *buf, long size)
1457 {
1458 struct index_entry idx;
1459
1460 *buf = '\0';
1461 if (!get_index(tcs->masterfd, idxid, &idx, true))
1462 return false;
1463
1464 return retrieve(tcs, &idx, tag, buf, size);
1465 }
1466
1467 static bool update_master_header(void)
1468 {
1469 struct master_header myhdr;
1470 int fd;
1471
1472 if (!tc_stat.ready)
1473 return false;
1474
1475 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1476 return false;
1477
1478 myhdr.serial = current_tcmh.serial;
1479 myhdr.commitid = current_tcmh.commitid;
1480
1481 /* Write it back */
1482 lseek(fd, 0, SEEK_SET);
1483 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1484 close(fd);
1485
1486 #ifdef HAVE_TC_RAMCACHE
1487 if (hdr)
1488 {
1489 hdr->h.serial = current_tcmh.serial;
1490 hdr->h.commitid = current_tcmh.commitid;
1491 }
1492 #endif
1493
1494 return true;
1495 }
1496
1497 #if 0
1498
1499 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1500 {
1501 struct tagentry *entry;
1502
1503 if (tcs->type != tag_title)
1504 return ;
1505
1506 /* We will need reserve buffer for this. */
1507 if (tcs->ramcache)
1508 {
1509 struct tagfile_entry *ep;
1510
1511 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1512 tcs->seek_list[tcs->seek_list_count];
1513 }
1514
1515 entry = find_entry_ram();
1516
1517 }
1518 #endif
1519
1520 void tagcache_search_finish(struct tagcache_search *tcs)
1521 {
1522 int i;
1523
1524 if (!tcs->initialized)
1525 return;
1526
1527 if (tcs->masterfd >= 0)
1528 {
1529 close(tcs->masterfd);
1530 tcs->masterfd = -1;
1531 }
1532
1533 for (i = 0; i < TAG_COUNT; i++)
1534 {
1535 if (tcs->idxfd[i] >= 0)
1536 {
1537 close(tcs->idxfd[i]);
1538 tcs->idxfd[i] = -1;
1539 }
1540 }
1541
1542 tcs->ramsearch = false;
1543 tcs->valid = false;
1544 tcs->initialized = 0;
1545 if (write_lock > 0)
1546 write_lock--;
1547 }
1548
1549 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1550 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1551 {
1552 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1553 }
1554
1555 static long get_tag_numeric(const struct index_entry *entry, int tag)
1556 {
1557 return check_virtual_tags(tag, entry);
1558 }
1559
1560 static char* get_tag_string(const struct index_entry *entry, int tag)
1561 {
1562 char* s = get_tag(entry, tag)->tag_data;
1563 return strcmp(s, UNTAGGED) ? s : NULL;
1564 }
1565
1566 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1567 {
1568 struct index_entry *entry;
1569 int idx_id;
1570
1571 if (!tc_stat.ready)
1572 return false;
1573
1574 /* Find the corresponding entry in tagcache. */
1575 idx_id = find_entry_ram(filename, NULL);
1576 if (idx_id < 0 || !tc_stat.ramcache)
1577 return false;
1578
1579 entry = &hdr->indices[idx_id];
1580
1581 id3->title = get_tag_string(entry, tag_title);
1582 id3->artist = get_tag_string(entry, tag_artist);
1583 id3->album = get_tag_string(entry, tag_album);
1584 id3->genre_string = get_tag_string(entry, tag_genre);
1585 id3->composer = get_tag_string(entry, tag_composer);
1586 id3->comment = get_tag_string(entry, tag_comment);
1587 id3->albumartist = get_tag_string(entry, tag_albumartist);
1588 id3->grouping = get_tag_string(entry, tag_grouping);
1589
1590 id3->playcount = get_tag_numeric(entry, tag_playcount);
1591 id3->rating = get_tag_numeric(entry, tag_rating);
1592 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1593 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1594 id3->year = get_tag_numeric(entry, tag_year);
1595
1596 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1597 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1598 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1599 if (id3->bitrate == 0)
1600 id3->bitrate = 1;
1601
1602 return true;
1603 }
1604 #endif
1605
1606 static inline void write_item(const char *item)
1607 {
1608 int len = strlen(item) + 1;
1609
1610 data_size += len;
1611 write(cachefd, item, len);
1612 }
1613
1614 static int check_if_empty(char **tag)
1615 {
1616 int length;
1617
1618 if (*tag == NULL || **tag == '\0')
1619 {
1620 *tag = UNTAGGED;
1621 return sizeof(UNTAGGED); /* Tag length */
1622 }
1623
1624 length = strlen(*tag);
1625 if (length > TAG_MAXLEN)
1626 {
1627 logf("over length tag: %s", *tag);
1628 length = TAG_MAXLEN;
1629 (*tag)[length] = '\0';
1630 }
1631
1632 return length + 1;
1633 }
1634
1635 #define ADD_TAG(entry,tag,data) \
1636 /* Adding tag */ \
1637 entry.tag_offset[tag] = offset; \
1638 entry.tag_length[tag] = check_if_empty(data); \
1639 offset += entry.tag_length[tag]
1640
1641 static void add_tagcache(char *path, unsigned long mtime
1642 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1643 ,const struct dirent *dc
1644 #endif
1645 )
1646 {
1647 struct mp3entry id3;
1648 struct temp_file_entry entry;
1649 bool ret;
1650 int fd;
1651 int idx_id = -1;
1652 char tracknumfix[3];
1653 int offset = 0;
1654 int path_length = strlen(path);
1655 bool has_albumartist;
1656 bool has_grouping;
1657
1658 if (cachefd < 0)
1659 return ;
1660
1661 /* Check for overlength file path. */
1662 if (path_length > TAG_MAXLEN)
1663 {
1664 /* Path can't be shortened. */
1665 logf("Too long path: %s", path);
1666 return ;
1667 }
1668
1669 /* Check if the file is supported. */
1670 if (probe_file_format(path) == AFMT_UNKNOWN)
1671 return ;
1672
1673 /* Check if the file is already cached. */
1674 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1675 if (tc_stat.ramcache && is_dircache_intact())
1676 {
1677 idx_id = find_entry_ram(path, dc);
1678 }
1679 else
1680 #endif
1681 {
1682 if (filenametag_fd >= 0)
1683 {
1684 idx_id = find_entry_disk(path);
1685 }
1686 }
1687
1688 /* Check if file has been modified. */
1689 if (idx_id >= 0)
1690 {
1691 struct index_entry idx;
1692
1693 if (!get_index(-1, idx_id, &idx, true))
1694 {
1695 logf("failed to retrieve index entry");
1696 return ;
1697 }
1698
1699 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1700 {
1701 /* No changes to file. */
1702 return ;
1703 }
1704
1705 /* Metadata might have been changed. Delete the entry. */
1706 logf("Re-adding: %s", path);
1707 if (!delete_entry(idx_id))
1708 {
1709 logf("delete_entry failed: %d", idx_id);
1710 return ;
1711 }
1712 }
1713
1714 fd = open(path, O_RDONLY);
1715 if (fd < 0)
1716 {
1717 logf("open fail: %s", path);
1718 return ;
1719 }
1720
1721 memset(&id3, 0, sizeof(struct mp3entry));
1722 memset(&entry, 0, sizeof(struct temp_file_entry));
1723 memset(&tracknumfix, 0, sizeof(tracknumfix));
1724 ret = get_metadata(&id3, fd, path);
1725 close(fd);
1726
1727 if (!ret)
1728 return ;
1729
1730 logf("-> %s", path);
1731
1732 /* Generate track number if missing. */
1733 if (id3.tracknum <= 0)
1734 {
1735 const char *p = strrchr(path, '.');
1736
1737 if (p == NULL)
1738 p = &path[strlen(path)-1];
1739
1740 while (*p != '/')
1741 {
1742 if (isdigit(*p) && isdigit(*(p-1)))
1743 {
1744 tracknumfix[1] = *p--;
1745 tracknumfix[0] = *p;
1746 break;
1747 }
1748 p--;
1749 }
1750
1751 if (tracknumfix[0] != '\0')
1752 {
1753 id3.tracknum = atoi(tracknumfix);
1754 /* Set a flag to indicate track number has been generated. */
1755 entry.flag |= FLAG_TRKNUMGEN;
1756 }
1757 else
1758 {
1759 /* Unable to generate track number. */
1760 id3.tracknum = -1;
1761 }
1762 }
1763
1764 /* Numeric tags */
1765 entry.tag_offset[tag_year] = id3.year;
1766 entry.tag_offset[tag_discnumber] = id3.discnum;
1767 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1768 entry.tag_offset[tag_length] = id3.length;
1769 entry.tag_offset[tag_bitrate] = id3.bitrate;
1770 entry.tag_offset[tag_mtime] = mtime;
1771
1772 /* String tags. */
1773 has_albumartist = id3.albumartist != NULL
1774 && strlen(id3.albumartist) > 0;
1775 has_grouping = id3.grouping != NULL
1776 && strlen(id3.grouping) > 0;
1777
1778 ADD_TAG(entry, tag_filename, &path);
1779 ADD_TAG(entry, tag_title, &id3.title);
1780 ADD_TAG(entry, tag_artist, &id3.artist);
1781 ADD_TAG(entry, tag_album, &id3.album);
1782 ADD_TAG(entry, tag_genre, &id3.genre_string);
1783 ADD_TAG(entry, tag_composer, &id3.composer);
1784 ADD_TAG(entry, tag_comment, &id3.comment);
1785 if (has_albumartist)
1786 {
1787 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1788 }
1789 else
1790 {
1791 ADD_TAG(entry, tag_albumartist, &id3.artist);
1792 }
1793 if (has_grouping)
1794 {
1795 ADD_TAG(entry, tag_grouping, &id3.grouping);
1796 }
1797 else
1798 {
1799 ADD_TAG(entry, tag_grouping, &id3.title);
1800 }
1801 entry.data_length = offset;
1802
1803 /* Write the header */
1804 write(cachefd, &entry, sizeof(struct temp_file_entry));
1805
1806 /* And tags also... Correct order is critical */
1807 write_item(path);
1808 write_item(id3.title);
1809 write_item(id3.artist);
1810 write_item(id3.album);
1811 write_item(id3.genre_string);
1812 write_item(id3.composer);
1813 write_item(id3.comment);
1814 if (has_albumartist)
1815 {
1816 write_item(id3.albumartist);
1817 }
1818 else
1819 {
1820 write_item(id3.artist);
1821 }
1822 if (has_grouping)
1823 {
1824 write_item(id3.grouping);
1825 }
1826 else
1827 {
1828 write_item(id3.title);
1829 }
1830 total_entry_count++;
1831 }
1832
1833 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1834 {
1835 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1836 int len = strlen(str)+1;
1837 int i;
1838 unsigned crc32;
1839 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1840 char buf[TAG_MAXLEN+32];
1841
1842 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1843 buf[i] = tolower(str[i]);
1844 buf[i] = '\0';
1845
1846 crc32 = crc_32(buf, i, 0xffffffff);
1847
1848 if (unique)
1849 {
1850 /* Check if the crc does not exist -> entry does not exist for sure. */
1851 for (i = 0; i < tempbufidx; i++)
1852 {
1853 if (crcbuf[-i] != crc32)
1854 continue;
1855
1856 if (!strcasecmp(str, index[i].str))
1857 {
1858 if (id < 0 || id >= lookup_buffer_depth)
1859 {
1860 logf("lookup buf overf.: %d", id);
1861 return false;
1862 }
1863
1864 lookup[id] = &index[i];
1865 return true;
1866 }
1867 }
1868 }
1869
1870 /* Insert to CRC buffer. */
1871 crcbuf[-tempbufidx] = crc32;
1872 tempbuf_left -= 4;
1873
1874 /* Insert it to the buffer. */
1875 tempbuf_left -= len;
1876 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1877 return false;
1878
1879 if (id >= lookup_buffer_depth)
1880 {
1881 logf("lookup buf overf. #2: %d", id);
1882 return false;
1883 }
1884
1885 if (id >= 0)
1886 {
1887 lookup[id] = &index[tempbufidx];
1888 index[tempbufidx].idlist.id = id;
1889 }
1890 else
1891 index[tempbufidx].idlist.id = -1;
1892
1893 index[tempbufidx].idlist.next = NULL;
1894 index[tempbufidx].idx_id = idx_id;
1895 index[tempbufidx].seek = -1;
1896 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1897 memcpy(index[tempbufidx].str, str, len);
1898 tempbuf_pos += len;
1899 tempbufidx++;
1900
1901 return true;
1902 }
1903
1904 static int compare(const void *p1, const void *p2)
1905 {
1906 do_timed_yield();
1907
1908 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1909 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1910
1911 if (strcmp(e1->str, UNTAGGED) == 0)
1912 {
1913 if (strcmp(e2->str, UNTAGGED) == 0)
1914 return 0;
1915 return -1;
1916 }
1917 else if (strcmp(e2->str, UNTAGGED) == 0)
1918 return 1;
1919
1920 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1921 }
1922
1923 static int tempbuf_sort(int fd)
1924 {
1925 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1926 struct tagfile_entry fe;
1927 int i;
1928 int length;
1929
1930 /* Generate reverse lookup entries. */
1931 for (i = 0; i < lookup_buffer_depth; i++)
1932 {
1933 struct tempbuf_id_list *idlist;
1934
1935 if (!lookup[i])
1936 continue;
1937
1938 if (lookup[i]->idlist.id == i)
1939 continue;
1940
1941 idlist = &lookup[i]->idlist;
1942 while (idlist->next != NULL)
1943 idlist = idlist->next;
1944
1945 tempbuf_left -= sizeof(struct tempbuf_id_list);
1946 if (tempbuf_left - 4 < 0)
1947 return -1;
1948
1949 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1950 if (tempbuf_pos & 0x03)
1951 {
1952 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1953 tempbuf_left -= 3;
1954 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1955 }
1956 tempbuf_pos += sizeof(struct tempbuf_id_list);
1957
1958 idlist = idlist->next;
1959 idlist->id = i;
1960 idlist->next = NULL;
1961
1962 do_timed_yield();
1963 }
1964
1965 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1966 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1967
1968 for (i = 0; i < tempbufidx; i++)
1969 {
1970 struct tempbuf_id_list *idlist = &index[i].idlist;
1971
1972 /* Fix the lookup list. */
1973 while (idlist != NULL)
1974 {
1975 if (idlist->id >= 0)
1976 lookup[idlist->id] = &index[i];
1977 idlist = idlist->next;
1978 }
1979
1980 index[i].seek = lseek(fd, 0, SEEK_CUR);
1981 length = strlen(index[i].str) + 1;
1982 fe.tag_length = length;
1983 fe.idx_id = index[i].idx_id;
1984
1985 /* Check the chunk alignment. */
1986 if ((fe.tag_length + sizeof(struct tagfile_entry))
1987 % TAGFILE_ENTRY_CHUNK_LENGTH)
1988 {
1989 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1990 ((fe.tag_length + sizeof(struct tagfile_entry))
1991 % TAGFILE_ENTRY_CHUNK_LENGTH);
1992 }
1993
1994 #ifdef TAGCACHE_STRICT_ALIGN
1995 /* Make sure the entry is long aligned. */
1996 if (index[i].seek & 0x03)
1997 {
1998 logf("tempbuf_sort: alignment error!");
1999 return -3;
2000 }
2001 #endif
2002
2003 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2004 sizeof(struct tagfile_entry))
2005 {
2006 logf("tempbuf_sort: write error #1");
2007 return -1;
2008 }
2009
2010 if (write(fd, index[i].str, length) != length)
2011 {
2012 logf("tempbuf_sort: write error #2");
2013 return -2;
2014 }
2015
2016 /* Write some padding. */
2017 if (fe.tag_length - length > 0)
2018 write(fd, "XXXXXXXX", fe.tag_length - length);
2019 }
2020
2021 return i;
2022 }
2023
2024 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2025 {
2026 if (id < 0 || id >= lookup_buffer_depth)
2027 return NULL;
2028
2029 return lookup[id];
2030 }
2031
2032
2033 inline static int tempbuf_find_location(int id)
2034 {
2035 struct tempbuf_searchidx *entry;
2036
2037 entry = tempbuf_locate(id);
2038 if (entry == NULL)
2039 return -1;
2040
2041 return entry->seek;
2042 }
2043
2044 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2045 {
2046 struct master_header tcmh;
2047 struct index_entry idx;
2048 int masterfd;
2049 int masterfd_pos;
2050 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2051 int max_entries;
2052 int entries_processed = 0;
2053 int i, j;
2054 char buf[TAG_MAXLEN];
2055
2056 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2057
2058 logf("Building numeric indices...");
2059 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2060
2061 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2062 return false;
2063
2064 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2065 SEEK_CUR);
2066 if (masterfd_pos == filesize(masterfd))
2067 {
2068 logf("we can't append!");
2069 close(masterfd);
2070 return false;
2071 }
2072
2073 while (entries_processed < h->entry_count)
2074 {
2075 int count = MIN(h->entry_count - entries_processed, max_entries);
2076
2077 /* Read in as many entries as possible. */
2078 for (i = 0; i < count; i++)
2079 {
2080 struct temp_file_entry *tfe = &entrybuf[i];
2081 int datastart;
2082
2083 /* Read in numeric data. */
2084 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2085 sizeof(struct temp_file_entry))
2086 {
2087 logf("read fail #1");
2088 close(masterfd);
2089 return false;
2090 }
2091
2092 datastart = lseek(tmpfd, 0, SEEK_CUR);
2093
2094 /**
2095 * Read string data from the following tags:
2096 * - tag_filename
2097 * - tag_artist
2098 * - tag_album
2099 * - tag_title
2100 *
2101 * A crc32 hash is calculated from the read data
2102 * and stored back to the data offset field kept in memory.
2103 */
2104 #define tmpdb_read_string_tag(tag) \
2105 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2106 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2107 { \
2108 logf("read fail: buffer overflow"); \
2109 close(masterfd); \
2110 return false; \
2111 } \
2112 \
2113 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2114 tfe->tag_length[tag]) \
2115 { \
2116 logf("read fail #2"); \
2117 close(masterfd); \
2118 return false; \
2119 } \
2120 \
2121 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2122 lseek(tmpfd, datastart, SEEK_SET)
2123
2124 tmpdb_read_string_tag(tag_filename);
2125 tmpdb_read_string_tag(tag_artist);
2126 tmpdb_read_string_tag(tag_album);
2127 tmpdb_read_string_tag(tag_title);
2128
2129 /* Seek to the end of the string data. */
2130 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2131 }
2132
2133 /* Backup the master index position. */
2134 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2135 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2136
2137 /* Check if we can resurrect some deleted runtime statistics data. */
2138 for (i = 0; i < tcmh.tch.entry_count; i++)
2139 {
2140 /* Read the index entry. */
2141 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2142 != sizeof(struct index_entry))
2143 {
2144 logf("read fail #3");
2145 close(masterfd);
2146 return false;
2147 }
2148
2149 /**
2150 * Skip unless the entry is marked as being deleted
2151 * or the data has already been resurrected.
2152 */
2153 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2154 continue;
2155
2156 /* Now try to match the entry. */
2157 /**
2158 * To succesfully match a song, the following conditions
2159 * must apply:
2160 *
2161 * For numeric fields: tag_length
2162 * - Full identical match is required
2163 *
2164 * If tag_filename matches, no further checking necessary.
2165 *
2166 * For string hashes: tag_artist, tag_album, tag_title
2167 * - Two of these must match
2168 */
2169 for (j = 0; j < count; j++)
2170 {
2171 struct temp_file_entry *tfe = &entrybuf[j];
2172
2173 /* Try to match numeric fields first. */
2174 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2175 continue;
2176
2177 /* Now it's time to do the hash matching. */
2178 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2179 {
2180 int match_count = 0;
2181
2182 /* No filename match, check if we can match two other tags. */
2183 #define tmpdb_match(tag) \
2184 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2185 match_count++
2186
2187 tmpdb_match(tag_artist);
2188 tmpdb_match(tag_album);
2189 tmpdb_match(tag_title);
2190
2191 if (match_count < 2)
2192 {
2193 /* Still no match found, give up. */
2194 continue;
2195 }
2196 }
2197
2198 /* A match found, now copy & resurrect the statistical data. */
2199 #define tmpdb_copy_tag(tag) \
2200 tfe->tag_offset[tag] = idx.tag_seek[tag]
2201
2202 tmpdb_copy_tag(tag_playcount);
2203 tmpdb_copy_tag(tag_rating);
2204 tmpdb_copy_tag(tag_playtime);
2205 tmpdb_copy_tag(tag_lastplayed);
2206 tmpdb_copy_tag(tag_commitid);
2207
2208 /* Avoid processing this entry again. */
2209 idx.flag |= FLAG_RESURRECTED;
2210
2211 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2212 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2213 != sizeof(struct index_entry))
2214 {
2215 logf("masterfd writeback fail #1");
2216 close(masterfd);
2217 return false;
2218 }
2219
2220 logf("Entry resurrected");
2221 }
2222 }
2223
2224
2225 /* Restore the master index position. */
2226 lseek(masterfd, masterfd_pos, SEEK_SET);
2227
2228 /* Commit the data to the index. */
2229 for (i = 0; i < count; i++)
2230 {
2231 int loc = lseek(masterfd, 0, SEEK_CUR);
2232
2233 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2234 != sizeof(struct index_entry))
2235 {
2236 logf("read fail #3");
2237 close(masterfd);
2238 return false;
2239 }
2240
2241 for (j = 0; j < TAG_COUNT; j++)
2242 {
2243 if (!tagcache_is_numeric_tag(j))
2244 continue;
2245
2246 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2247 }
2248 idx.flag = entrybuf[i].flag;
2249
2250 if (idx.tag_seek[tag_commitid])
2251 {
2252 /* Data has been resurrected. */
2253 idx.flag |= FLAG_DIRTYNUM;
2254 }
2255 else if (tc_stat.ready && current_tcmh.commitid > 0)
2256 {
2257 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2258 idx.flag |= FLAG_DIRTYNUM;
2259 }
2260
2261 /* Write back the updated index. */
2262 lseek(masterfd, loc, SEEK_SET);
2263 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2264 != sizeof(struct index_entry))
2265 {
2266 logf("write fail");
2267 close(masterfd);
2268 return false;
2269 }
2270 }
2271
2272 entries_processed += count;
2273 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2274 }
2275
2276 close(masterfd);
2277
2278 return true;
2279 }
2280
2281 /**
2282 * Return values:
2283 * > 0 success
2284 * == 0 temporary failure
2285 * < 0 fatal error
2286 */
2287 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2288 {
2289 int i;
2290 struct tagcache_header tch;
2291 struct master_header tcmh;
2292 struct index_entry idxbuf[IDX_BUF_DEPTH];
2293 int idxbuf_pos;
2294 char buf[TAG_MAXLEN+32];
2295 int fd = -1, masterfd;
2296 bool error = false;
2297 int init;
2298 int masterfd_pos;
2299
2300 logf("Building index: %d", index_type);
2301
2302 /* Check the number of entries we need to allocate ram for. */
2303 commit_entry_count = h->entry_count + 1;
2304
2305 masterfd = open_master_fd(&tcmh, false);
2306 if (masterfd >= 0)
2307 {
2308 commit_entry_count += tcmh.tch.entry_count;
2309 close(masterfd);
2310 }
2311 else
2312 remove_files(); /* Just to be sure we are clean. */
2313
2314 /* Open the index file, which contains the tag names. */
2315 fd = open_tag_fd(&tch, index_type, true);
2316 if (fd >= 0)
2317 {
2318 logf("tch.datasize=%ld", tch.datasize);
2319 lookup_buffer_depth = 1 +
2320 /* First part */ commit_entry_count +
2321 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2322 }
2323 else
2324 {
2325 lookup_buffer_depth = 1 +
2326 /* First part */ commit_entry_count +
2327 /* Second part */ 0;
2328 }
2329
2330 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2331 logf("commit_entry_count=%ld", commit_entry_count);
2332
2333 /* Allocate buffer for all index entries from both old and new
2334 * tag files. */
2335 tempbufidx = 0;
2336 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2337
2338 /* Allocate lookup buffer. The first portion of commit_entry_count
2339 * contains the new tags in the temporary file and the second
2340 * part for locating entries already in the db.
2341 *
2342 * New tags Old tags
2343 * +---------+---------------------------+
2344 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2345 * +---------+---------------------------+
2346 *
2347 * Old tags are inserted to a temporary buffer with position:
2348 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2349 * And new tags with index:
2350 * tempbuf_insert(idx, ...);
2351 *
2352 * The buffer is sorted and written into tag file:
2353 * tempbuf_sort(...);
2354 * leaving master index locations messed up.
2355 *
2356 * That is fixed using the lookup buffer for old tags:
2357 * new_seek = tempbuf_find_location(old_seek, ...);
2358 * and for new tags:
2359 * new_seek = tempbuf_find_location(idx);
2360 */
2361 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2362 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2363 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2364
2365 /* And calculate the remaining data space used mainly for storing
2366 * tag data (strings). */
2367 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2368 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2369 {
2370 logf("Buffer way too small!");
2371 return 0;
2372 }
2373
2374 if (fd >= 0)
2375 {
2376 /**
2377 * If tag file contains unique tags (sorted index), we will load
2378 * it entirely into memory so we can resort it later for use with
2379 * chunked browsing.
2380 */
2381 if (tagcache_is_sorted_tag(index_type))
2382 {
2383 logf("loading tags...");
2384 for (i = 0; i < tch.entry_count; i++)
2385 {
2386 struct tagfile_entry entry;
2387 int loc = lseek(fd, 0, SEEK_CUR);
2388 bool ret;
2389
2390 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2391 != sizeof(struct tagfile_entry))
2392 {
2393 logf("read error #7");
2394 close(fd);
2395 return -2;
2396 }
2397
2398 if (entry.tag_length >= (int)sizeof(buf))
2399 {
2400 logf("too long tag #3");
2401 close(fd);
2402 return -2;
2403 }
2404
2405 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2406 {
2407 logf("read error #8");
2408 close(fd);
2409 return -2;
2410 }
2411
2412 /* Skip deleted entries. */
2413 if (buf[0] == '\0')
2414 continue;
2415
2416 /**
2417 * Save the tag and tag id in the memory buffer. Tag id
2418 * is saved so we can later reindex the master lookup
2419 * table when the index gets resorted.
2420 */
2421 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2422 + commit_entry_count, entry.idx_id,
2423 tagcache_is_unique_tag(index_type));
2424 if (!ret)
2425 {
2426 close(fd);
2427 return -3;
2428 }
2429 do_timed_yield();
2430 }
2431 logf("done");
2432 }
2433 else
2434 tempbufidx = tch.entry_count;
2435 }
2436 else
2437 {
2438 /**
2439 * Creating new index file to store the tags. No need to preload
2440 * anything whether the index type is sorted or not.
2441 */
2442 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2443 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2444 if (fd < 0)
2445 {
2446 logf("%s open fail", buf);
2447 return -2;
2448 }
2449
2450 tch.magic = TAGCACHE_MAGIC;
2451 tch.entry_count = 0;
2452 tch.datasize = 0;
2453
2454 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2455 != sizeof(struct tagcache_header))
2456 {
2457 logf("header write failed");
2458 close(fd);
2459 return -2;
2460 }
2461 }
2462
2463 /* Loading the tag lookup file as "master file". */
2464 logf("Loading index file");
2465 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2466
2467 if (masterfd < 0)
2468 {
2469 logf("Creating new DB");
2470 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2471
2472 if (masterfd < 0)
2473 {
2474 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2475 close(fd);
2476 return -2;
2477 }
2478
2479 /* Write the header (write real values later). */
2480 memset(&tcmh, 0, sizeof(struct master_header));
2481 tcmh.tch = *h;
2482 tcmh.tch.entry_count = 0;
2483 tcmh.tch.datasize = 0;
2484 tcmh.dirty = true;
2485 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2486 init = true;
2487 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2488 }
2489 else
2490 {
2491 /**
2492 * Master file already exists so we need to process the current
2493 * file first.
2494 */
2495 init = false;
2496
2497 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2498 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2499 {
2500 logf("header error");
2501 close(fd);
2502 close(masterfd);
2503 return -2;
2504 }
2505
2506 /**
2507 * If we reach end of the master file, we need to expand it to
2508 * hold new tags. If the current index is not sorted, we can
2509 * simply append new data to end of the file.
2510 * However, if the index is sorted, we need to update all tag
2511 * pointers in the master file for the current index.
2512 */
2513 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2514 SEEK_CUR);
2515 if (masterfd_pos == filesize(masterfd))
2516 {
2517 logf("appending...");
2518 init = true;
2519 }
2520 }
2521
2522 /**
2523 * Load new unique tags in memory to be sorted later and added
2524 * to the master lookup file.
2525 */
2526 if (tagcache_is_sorted_tag(index_type))
2527 {
2528 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2529 /* h is the header of the temporary file containing new tags. */
2530 logf("inserting new tags...");
2531 for (i = 0; i < h->entry_count; i++)
2532 {
2533 struct temp_file_entry entry;
2534
2535 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2536 sizeof(struct temp_file_entry))
2537 {
2538 logf("read fail #3");
2539 error = true;
2540 goto error_exit;
2541 }
2542
2543 /* Read data. */
2544 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2545 {
2546 logf("too long entry!");
2547 error = true;
2548 goto error_exit;
2549 }
2550
2551 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2552 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2553 entry.tag_length[index_type])
2554 {
2555 logf("read fail #4");
2556 error = true;
2557 goto error_exit;
2558 }
2559
2560 if (tagcache_is_unique_tag(index_type))
2561 error = !tempbuf_insert(buf, i, -1, true);
2562 else
2563 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2564
2565 if (error)
2566 {
2567 logf("insert error");
2568 goto error_exit;
2569 }
2570
2571 /* Skip to next. */
2572 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2573 entry.tag_length[index_type], SEEK_CUR);
2574 do_timed_yield();
2575 }
2576 logf("done");
2577
2578 /* Sort the buffer data and write it to the index file. */
2579 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2580 i = tempbuf_sort(fd);
2581 if (i < 0)
2582 goto error_exit;
2583 logf("sorted %d tags", i);
2584
2585 /**
2586 * Now update all indexes in the master lookup file.
2587 */
2588 logf("updating indices...");
2589 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2590 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2591 {
2592 int j;
2593 int loc = lseek(masterfd, 0, SEEK_CUR);
2594
2595 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2596
2597 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2598 != (int)sizeof(struct index_entry)*idxbuf_pos)
2599 {
2600 logf("read fail #5");
2601 error = true;
2602 goto error_exit ;
2603 }
2604 lseek(masterfd, loc, SEEK_SET);
2605
2606 for (j = 0; j < idxbuf_pos; j++)
2607 {
2608 if (idxbuf[j].flag & FLAG_DELETED)
2609 {
2610 /* We can just ignore deleted entries. */
2611 // idxbuf[j].tag_seek[index_type] = 0;
2612 continue;
2613 }
2614
2615 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2616 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2617 + commit_entry_count);
2618
2619 if (idxbuf[j].tag_seek[index_type] < 0)
2620 {
2621 logf("update error: %d/%d/%ld",
2622 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2623 error = true;
2624 goto error_exit;
2625 }
2626
2627 do_timed_yield();
2628 }
2629
2630 /* Write back the updated index. */
2631 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2632 index_entry_ec, tc_stat.econ) !=
2633 (int)sizeof(struct index_entry)*idxbuf_pos)
2634 {
2635 logf("write fail");
2636 error = true;
2637 goto error_exit;
2638 }
2639 }
2640 logf("done");
2641 }
2642
2643 /**
2644 * Walk through the temporary file containing the new tags.
2645 */
2646 // build_normal_index(h, tmpfd, masterfd, idx);
2647 logf("updating new indices...");
2648 lseek(masterfd, masterfd_pos, SEEK_SET);
2649 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2650 lseek(fd, 0, SEEK_END);
2651 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2652 {
2653 int j;
2654
2655 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2656 if (init)
2657 {
2658 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2659 }
2660 else
2661 {
2662 int loc = lseek(masterfd, 0, SEEK_CUR);
2663
2664 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2665 != (int)sizeof(struct index_entry)*idxbuf_pos)
2666 {
2667 logf("read fail #6");
2668 error = true;
2669 break ;
2670 }
2671 lseek(masterfd, loc, SEEK_SET);
2672 }
2673
2674 /* Read entry headers. */
2675 for (j = 0; j < idxbuf_pos; j++)
2676 {
2677 if (!tagcache_is_sorted_tag(index_type))
2678 {
2679 struct temp_file_entry entry;
2680 struct tagfile_entry fe;
2681
2682 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2683 sizeof(struct temp_file_entry))
2684 {
2685 logf("read fail #7");
2686 error = true;
2687 break ;
2688 }
2689
2690 /* Read data. */
2691 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2692 {
2693 logf("too long entry!");
2694 logf("length=%d", entry.tag_length[index_type]);
2695 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2696 error = true;
2697 break ;
2698 }
2699
2700 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2701 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2702 entry.tag_length[index_type])
2703 {
2704 logf("read fail #8");
2705 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2706 logf("length=0x%02x", entry.tag_length[index_type]);
2707 error = true;
2708 break ;
2709 }
2710
2711 /* Write to index file. */
2712 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2713 fe.tag_length = entry.tag_length[index_type];
2714 fe.idx_id = tcmh.tch.entry_count + i + j;
2715 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2716 write(fd, buf, fe.tag_length);
2717 tempbufidx++;
2718
2719 /* Skip to next. */
2720 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2721 entry.tag_length[index_type], SEEK_CUR);
2722 }
2723 else
2724 {
2725 /* Locate the correct entry from the sorted array. */
2726 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2727 if (idxbuf[j].tag_seek[index_type] < 0)
2728 {
2729 logf("entry not found (%d)", j);
2730 error = true;
2731 break ;
2732 }
2733 }
2734 }
2735
2736 /* Write index. */
2737 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2738 index_entry_ec, tc_stat.econ) !=
2739 (int)sizeof(struct index_entry)*idxbuf_pos)
2740 {
2741 logf("tagcache: write fail #4");
2742 error = true;
2743 break ;
2744 }
2745
2746 do_timed_yield();
2747 }
2748 logf("done");
2749
2750 /* Finally write the header. */
2751 tch.magic = TAGCACHE_MAGIC;
2752 tch.entry_count = tempbufidx;
2753 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2754 lseek(fd, 0, SEEK_SET);
2755 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2756
2757 if (index_type != tag_filename)
2758 h->datasize += tch.datasize;
2759 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2760 error_exit:
2761
2762 close(fd);
2763 close(masterfd);
2764
2765 if (error)
2766 return -2;
2767
2768 return 1;
2769 }
2770
2771 static bool commit(void)
2772 {
2773 struct tagcache_header tch;
2774 struct master_header tcmh;
2775 int i, len, rc;
2776 int tmpfd;
2777 int masterfd;
2778 #ifdef HAVE_DIRCACHE
2779 bool dircache_buffer_stolen = false;
2780 #endif
2781 bool local_allocation = false;
2782
2783 logf("committing tagcache");
2784
2785 while (write_lock)
2786 sleep(1);
2787
2788 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2789 if (tmpfd < 0)
2790 {
2791 logf("nothing to commit");
2792 return true;
2793 }
2794
2795
2796 /* Load the header. */
2797 len = sizeof(struct tagcache_header);
2798 rc = read(tmpfd, &tch, len);
2799
2800 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2801 {
2802 logf("incorrect header");
2803 close(tmpfd);
2804 remove(TAGCACHE_FILE_TEMP);
2805 return false;
2806 }
2807
2808 if (tch.entry_count == 0)
2809 {
2810 logf("nothing to commit");
2811 close(tmpfd);
2812 remove(TAGCACHE_FILE_TEMP);
2813 return true;
2814 }
2815
2816 #ifdef HAVE_EEPROM_SETTINGS
2817 remove(TAGCACHE_STATEFILE);
2818 #endif
2819
2820 /* At first be sure to unload the ramcache! */
2821 #ifdef HAVE_TC_RAMCACHE
2822 tc_stat.ramcache = false;
2823 #endif
2824
2825 read_lock++;
2826
2827 /* Try to steal every buffer we can :) */
2828 if (tempbuf_size == 0)
2829 local_allocation = true;
2830
2831 #ifdef HAVE_DIRCACHE
2832 if (tempbuf_size == 0)
2833 {
2834 /* Try to steal the dircache buffer. */
2835 tempbuf = dircache_steal_buffer(&tempbuf_size);
2836 tempbuf_size &= ~0x03;
2837
2838 if (tempbuf_size > 0)
2839 {
2840 dircache_buffer_stolen = true;
2841 }
2842 }
2843 #endif
2844
2845 #ifdef HAVE_TC_RAMCACHE
2846 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2847 {
2848 tempbuf = (char *)(hdr + 1);
2849 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2850 tempbuf_size &= ~0x03;
2851 }
2852 #endif
2853
2854 /* And finally fail if there are no buffers available. */
2855 if (tempbuf_size == 0)
2856 {
2857 logf("delaying commit until next boot");
2858 tc_stat.commit_delayed = true;
2859 close(tmpfd);
2860 read_lock--;
2861 return false;
2862 }
2863
2864 logf("commit %ld entries...", tch.entry_count);
2865
2866 /* Mark DB dirty so it will stay disabled if commit fails. */
2867 current_tcmh.dirty = true;
2868 update_master_header();
2869
2870 /* Now create the index files. */
2871 tc_stat.commit_step = 0;
2872 tch.datasize = 0;
2873 tc_stat.commit_delayed = false;
2874
2875 for (i = 0; i < TAG_COUNT; i++)
2876 {
2877 int ret;
2878
2879 if (tagcache_is_numeric_tag(i))
2880 continue;
2881
2882 tc_stat.commit_step++;
2883 ret = build_index(i, &tch, tmpfd);
2884 if (ret <= 0)
2885 {
2886 close(tmpfd);
2887 logf("tagcache failed init");
2888 if (ret < 0)
2889 remove_files();
2890 else
2891 tc_stat.commit_delayed = true;
2892 tc_stat.commit_step = 0;
2893 read_lock--;
2894 return false;
2895 }
2896 }
2897
2898 if (!build_numeric_indices(&tch, tmpfd))
2899 {
2900 logf("Failure to commit numeric indices");
2901 close(tmpfd);
2902 remove_files();
2903 tc_stat.commit_step = 0;
2904 read_lock--;
2905 return false;
2906 }
2907
2908 close(tmpfd);
2909 tc_stat.commit_step = 0;
2910
2911 /* Update the master index headers. */
2912 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2913 {
2914 read_lock--;
2915 return false;
2916 }
2917
2918 tcmh.tch.entry_count += tch.entry_count;
2919 tcmh.tch.datasize = sizeof(struct master_header)
2920 + sizeof(struct index_entry) * tcmh.tch.entry_count
2921 + tch.datasize;
2922 tcmh.dirty = false;
2923 tcmh.commitid++;
2924
2925 lseek(masterfd, 0, SEEK_SET);
2926 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2927 close(masterfd);
2928
2929 logf("tagcache committed");
2930 remove(TAGCACHE_FILE_TEMP);
2931 tc_stat.ready = check_all_headers();
2932 tc_stat.readyvalid = true;
2933
2934 if (local_allocation)
2935 {
2936 tempbuf = NULL;
2937 tempbuf_size = 0;
2938 }
2939
2940 #ifdef HAVE_DIRCACHE
2941 /* Rebuild the dircache, if we stole the buffer. */
2942 if (dircache_buffer_stolen)
2943 dircache_build(0);
2944 #endif
2945
2946 #ifdef HAVE_TC_RAMCACHE
2947 /* Reload tagcache. */
2948 if (tc_stat.ramcache_allocated > 0)
2949 tagcache_start_scan();
2950 #endif
2951
2952 read_lock--;
2953
2954 return true;
2955 }
2956
2957 static void allocate_tempbuf(void)
2958 {
2959 /* Yeah, malloc would be really nice now :) */
2960 #ifdef __PCTOOL__
2961 tempbuf_size = 32*1024*1024;
2962 tempbuf = malloc(tempbuf_size);
2963 #else
2964 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2965 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2966 audiobuf += tempbuf_size;
2967 #endif
2968 }
2969
2970 static void free_tempbuf(void)
2971 {
2972 if (tempbuf_size == 0)
2973 return ;
2974
2975 #ifdef __PCTOOL__
2976 free(tempbuf);
2977 #else
2978 audiobuf -= tempbuf_size;
2979 #endif
2980 tempbuf = NULL;
2981 tempbuf_size = 0;
2982 }
2983
2984 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2985 {
2986 struct index_entry idx;
2987
2988 if (!tc_stat.ready)
2989 return false;
2990
2991 if (!tagcache_is_numeric_tag(tag))
2992 return false;
2993
2994 if (!get_index(masterfd, idx_id, &idx, false))
2995 return false;
2996
2997 idx.tag_seek[tag] = data;
2998 idx.flag |= FLAG_DIRTYNUM;
2999
3000 return write_index(masterfd, idx_id, &idx);
3001 }
3002
3003 #if 0
3004 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3005 int tag, long data)
3006 {
3007 struct master_header myhdr;
3008
3009 if (tcs->masterfd < 0)
3010 {
3011 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3012 return false;
3013 }
3014
3015 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3016 }
3017 #endif
3018
3019 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3020
3021 static bool command_queue_is_full(void)
3022 {
3023 int next;
3024
3025 next = command_queue_widx + 1;
3026 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3027 next = 0;
3028
3029 return (next == command_queue_ridx);
3030 }
3031 bool command_queue_sync_callback(void)
3032 {
3033
3034 struct master_header myhdr;
3035 int masterfd;
3036
3037 mutex_lock(&command_queue_mutex);
3038
3039 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3040 return false;
3041
3042 while (command_queue_ridx != command_queue_widx)
3043 {
3044 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3045
3046 switch (ce->command)
3047 {
3048 case CMD_UPDATE_MASTER_HEADER:
3049 {
3050 close(masterfd);
3051 update_master_header();
3052
3053 /* Re-open the masterfd. */
3054 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3055 return true;
3056
3057 break;
3058 }
3059 case CMD_UPDATE_NUMERIC:
3060 {
3061 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3062 break;
3063 }
3064 }
3065
3066 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3067 command_queue_ridx = 0;
3068 }
3069
3070 close(masterfd);
3071
3072 tc_stat.queue_length = 0;
3073 mutex_unlock(&command_queue_mutex);
3074 return true;
3075 }
3076
3077 void run_command_queue(bool force)
3078 {
3079 if (COMMAND_QUEUE_IS_EMPTY)
3080 return;
3081
3082 if (force || command_queue_is_full())
3083 command_queue_sync_callback();
3084 else
3085 register_ata_idle_func(command_queue_sync_callback);
3086 }
3087
3088 static void queue_command(int cmd, long idx_id, int tag, long data)
3089 {
3090 while (1)
3091 {
3092 int next;
3093
3094 mutex_lock(&command_queue_mutex);
3095 next = command_queue_widx + 1;
3096 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3097 next = 0;
3098
3099 /* Make sure queue is not full. */
3100 if (next != command_queue_ridx)
3101 {
3102 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3103
3104 ce->command = cmd;
3105 ce->idx_id = idx_id;
3106 ce->tag = tag;
3107 ce->data = data;
3108
3109 command_queue_widx = next;
3110
3111 tc_stat.queue_length++;
3112
3113 mutex_unlock(&command_queue_mutex);
3114 break;
3115 }
3116
3117 /* Queue is full, try again later... */
3118 mutex_unlock(&command_queue_mutex);
3119 sleep(1);
3120 }
3121 }
3122
3123 long tagcache_increase_serial(void)
3124 {
3125 long old;
3126
3127 if (!tc_stat.ready)
3128 return -2;
3129
3130 while (read_lock)
3131 sleep(1);
3132
3133 old = current_tcmh.serial++;
3134 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3135
3136 return old;
3137 }
3138
3139 void tagcache_update_numeric(int idx_id, int tag, long data)
3140 {
3141 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3142 }
3143
3144 long tagcache_get_serial(void)
3145 {
3146 return current_tcmh.serial;
3147 }
3148
3149 long tagcache_get_commitid(void)
3150 {
3151 return current_tcmh.commitid;
3152 }
3153
3154 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3155 {
3156 char buf[512];
3157 int i;
3158
3159 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3160 for (i = strlen(buf); i < (long)sizeof(buf)-3; i++)
3161 {
3162 if (*datastr == '\0')
3163 break;
3164
3165 if (*datastr == '"')
3166 {
3167 buf[i] = '\\';
3168 datastr++;
3169 continue;
3170 }
3171
3172 buf[i] = *(datastr++);
3173 }
3174
3175 strcpy(&buf[i], "\" ");
3176
3177 write(fd, buf, i + 2);
3178
3179 return true;
3180 }
3181
3182 static bool read_tag(char *dest, long size,
3183 const char *src, const char *tagstr)
3184 {
3185 int pos;
3186 char current_tag[32];
3187
3188 while (*src != '\0')
3189 {
3190 /* Skip all whitespace */
3191 while (*src == ' ')
3192 src++;
3193
3194 if (*src == '\0')
3195 break;
3196
3197 pos = 0;
3198 /* Read in tag name */
3199 while (*src != '=' && *src != ' ')
3200 {
3201 current_tag[pos] = *src;
3202 src++;
3203 pos++;
3204
3205 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3206 return false;
3207 }
3208 current_tag[pos] = '\0';
3209
3210 /* Read in tag data */
3211
3212 /* Find the start. */
3213 while (*src != '"' && *src != '\0')
3214 src++;
3215
3216 if (*src == '\0' || *(++src) == '\0')
3217 return false;
3218
3219 /* Read the data. */
3220 for (pos = 0; pos < size; pos++)
3221 {
3222 if (*src == '\0')
3223 break;
3224
3225 if (*src == '\\' && *(src+1) == '"')
3226 {
3227 dest[pos] = '"';
3228 src += 2;
3229 continue;
3230 }
3231
3232 dest[pos] = *src;
3233
3234 if (*src == '"')
3235 {
3236 src++;
3237 break;
3238 }
3239
3240 if (*src == '\0')
3241 break;
3242
3243 src++;
3244 }
3245 dest[pos] = '\0';
3246
3247 if (!strcasecmp(tagstr, current_tag))
3248 return true;
3249 }
3250
3251 return false;
3252 }
3253
3254 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3255 {
3256 struct index_entry idx;
3257 char tag_data[TAG_MAXLEN+32];
3258 int idx_id;
3259 long masterfd = (long)parameters;
3260 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3261 tag_commitid };
3262 int i;
3263 (void)line_n;
3264
3265 if (*buf == '#')
3266 return 0;
3267
3268 logf("%d/%s", line_n, buf);
3269 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3270 {
3271 logf("filename missing");
3272 logf("-> %s", buf);
3273 return 0;
3274 }
3275
3276 idx_id = find_index(tag_data);
3277 if (idx_id < 0)
3278 {
3279 logf("entry not found");
3280 return 0;
3281 }
3282
3283 if (!get_index(masterfd, idx_id, &idx, false))
3284 {
3285 logf("failed to retrieve index entry");
3286 return 0;
3287 }
3288
3289 /* Stop if tag has already been modified. */
3290 if (idx.flag & FLAG_DIRTYNUM)
3291 return 0;
3292
3293 logf("import: %s", tag_data);
3294
3295 idx.flag |= FLAG_DIRTYNUM;
3296 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3297 {
3298 int data;
3299
3300 if (!read_tag(tag_data, sizeof tag_data, buf,
3301 tagcache_tag_to_str(import_tags[i])))
3302 {
3303 continue;
3304 }
3305
3306 data = atoi(tag_data);
3307 if (data < 0)
3308 continue;
3309
3310 idx.tag_seek[import_tags[i]] = data;
3311
3312 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3313 current_tcmh.serial = data;
3314 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3315 current_tcmh.commitid = data + 1;
3316 }
3317
3318 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3319 }
3320
3321 #ifndef __PCTOOL__
3322 bool tagcache_import_changelog(void)
3323 {
3324 struct master_header myhdr;
3325 struct tagcache_header tch;
3326 int clfd;
3327 long masterfd;
3328 char buf[2048];
3329
3330 if (!tc_stat.ready)
3331 return false;
3332
3333 while (read_lock)
3334 sleep(1);
3335
3336 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3337 if (clfd < 0)
3338 {
3339 logf("failure to open changelog");
3340 return false;
3341 }
3342
3343 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3344 {
3345 close(clfd);
3346 return false;
3347 }
3348
3349 write_lock++;
3350
3351 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3352
3353 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3354 parse_changelog_line);
3355
3356 close(clfd);
3357 close(masterfd);
3358
3359 if (filenametag_fd >= 0)
3360 close(filenametag_fd);
3361
3362 write_lock--;
3363
3364 update_master_header();
3365
3366 return true;
3367 }
3368 #endif
3369
3370 bool tagcache_create_changelog(struct tagcache_search *tcs)
3371 {
3372 struct master_header myhdr;
3373 struct index_entry idx;
3374 char buf[TAG_MAXLEN+32];
3375 char temp[32];
3376 int clfd;
3377 int i, j;
3378
3379 if (!tc_stat.ready)
3380 return false;
3381
3382 if (!tagcache_search(tcs, tag_filename))
3383 return false;
3384
3385 /* Initialize the changelog */
3386 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3387 if (clfd < 0)
3388 {
3389 logf("failure to open changelog");
3390 return false;
3391 }
3392
3393 if (tcs->masterfd < 0)
3394 {
3395 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3396 return false;
3397 }
3398 else
3399 {
3400 lseek(tcs->masterfd, 0, SEEK_SET);
3401 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3402 }
3403
3404 write(clfd, "## Changelog version 1\n", 23);
3405
3406 for (i = 0; i < myhdr.tch.entry_count; i++)
3407 {
3408 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3409 != sizeof(struct index_entry))
3410 {
3411 logf("read error #9");
3412 tagcache_search_finish(tcs);
3413 close(clfd);
3414 return false;
3415 }
3416
3417 /* Skip until the entry found has been modified. */
3418 if (! (idx.flag & FLAG_DIRTYNUM) )
3419 continue;
3420
3421 /* Skip deleted entries too. */
3422 if (idx.flag & FLAG_DELETED)
3423 continue;
3424
3425 /* Now retrieve all tags. */
3426 for (j = 0; j < TAG_COUNT; j++)
3427 {
3428 if (tagcache_is_numeric_tag(j))
3429 {
3430 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3431 write_tag(clfd, tagcache_tag_to_str(j), temp);
3432 continue;
3433 }
3434
3435 tcs->type = j;
3436 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3437 write_tag(clfd, tagcache_tag_to_str(j), buf);
3438 }
3439
3440 write(clfd, "\n", 1);
3441 do_timed_yield();
3442 }
3443
3444 close(clfd);
3445
3446 tagcache_search_finish(tcs);
3447
3448 return true;
3449 }
3450
3451 static bool delete_entry(long idx_id)
3452 {
3453 int fd = -1;
3454 int masterfd = -1;
3455 int tag, i;
3456 struct index_entry idx, myidx;
3457 struct master_header myhdr;
3458 char buf[TAG_MAXLEN+32];
3459 int in_use[TAG_COUNT];
3460
3461 logf("delete_entry(): %ld", idx_id);
3462
3463 #ifdef HAVE_TC_RAMCACHE
3464 /* At first mark the entry removed from ram cache. */
3465 if (tc_stat.ramcache)
3466 hdr->indices[idx_id].flag |= FLAG_DELETED;
3467 #endif
3468
3469 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3470 return false;
3471
3472 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3473 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3474 != sizeof(struct index_entry))
3475 {
3476 logf("delete_entry(): read error");
3477 goto cleanup;
3478 }
3479
3480 if (myidx.flag & FLAG_DELETED)
3481 {
3482 logf("delete_entry(): already deleted!");
3483 goto cleanup;
3484 }
3485
3486 myidx.flag |= FLAG_DELETED;
3487 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3488 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3489 != sizeof(struct index_entry))
3490 {
3491 logf("delete_entry(): write_error #1");
3492 goto cleanup;
3493 }
3494
3495 /* Now check which tags are no longer in use (if any) */
3496 for (tag = 0; tag < TAG_COUNT; tag++)
3497 in_use[tag] = 0;
3498
3499 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3500 for (i = 0; i < myhdr.tch.entry_count; i++)
3501 {
3502 struct index_entry *idxp;
3503
3504 #ifdef HAVE_TC_RAMCACHE
3505 /* Use RAM DB if available for greater speed */
3506 if (tc_stat.ramcache)
3507 idxp = &hdr->indices[i];
3508 else
3509 #endif
3510 {
3511 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3512 != sizeof(struct index_entry))
3513 {
3514 logf("delete_entry(): read error #2");
3515 goto cleanup;
3516 }
3517 idxp = &idx;
3518 }
3519
3520 if (idxp->flag & FLAG_DELETED)
3521 continue;
3522
3523 for (tag = 0; tag < TAG_COUNT; tag++)
3524 {
3525 if (tagcache_is_numeric_tag(tag))
3526 continue;
3527
3528 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3529 in_use[tag]++;
3530 }
3531 }
3532
3533 /* Now delete all tags no longer in use. */
3534 for (tag = 0; tag < TAG_COUNT; tag++)
3535 {
3536 struct tagcache_header tch;
3537 int oldseek = myidx.tag_seek[tag];
3538
3539 if (tagcache_is_numeric_tag(tag))
3540 continue;
3541
3542 /**
3543 * Replace tag seek with a hash value of the field string data.
3544 * That way runtime statistics of moved or altered files can be
3545 * resurrected.
3546 */
3547 #ifdef HAVE_TC_RAMCACHE
3548 if (tc_stat.ramcache && tag != tag_filename)
3549 {
3550 struct tagfile_entry *tfe;
3551 long *seek = &hdr->indices[idx_id].tag_seek[tag];
3552
3553 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3554 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3555 myidx.tag_seek[tag] = *seek;
3556 }
3557 else
3558 #endif
3559 {
3560 struct tagfile_entry tfe;
3561
3562 /* Open the index file, which contains the tag names. */
3563 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3564 goto cleanup;
3565
3566 /* Skip the header block */
3567 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3568 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3569 != sizeof(struct tagfile_entry))
3570 {
3571 logf("delete_entry(): read error #3");
3572 goto cleanup;
3573 }
3574
3575 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3576 {
3577 logf("delete_entry(): read error #4");
3578 goto cleanup;
3579 }
3580
3581 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3582 }
3583
3584 if (in_use[tag])
3585 {
3586 logf("in use: %d/%d", tag, in_use[tag]);
3587 if (fd >= 0)
3588 {
3589 close(fd);
3590 fd = -1;
3591 }
3592 continue;
3593 }
3594
3595 #ifdef HAVE_TC_RAMCACHE
3596 /* Delete from ram. */
3597 if (tc_stat.ramcache && tag != tag_filename)
3598 {
3599 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3600 tagentry->tag_data[0] = '\0';
3601 }
3602 #endif
3603
3604 /* Open the index file, which contains the tag names. */
3605 if (fd < 0)
3606 {
3607 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3608 goto cleanup;
3609 }
3610
3611 /* Skip the header block */
3612 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3613
3614 /* Debug, print 10 first characters of the tag
3615 read(fd, buf, 10);
3616 buf[10]='\0';
3617 logf("TAG:%s", buf);
3618 lseek(fd, -10, SEEK_CUR);
3619 */
3620
3621 /* Write first data byte in tag as \0 */
3622 write(fd, "", 1);
3623
3624 /* Now tag data has been removed */
3625 close(fd);
3626 fd = -1;
3627 }
3628
3629 /* Write index entry back into master index. */
3630 lseek(masterfd, sizeof(struct master_header) +
3631 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3632 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3633 != sizeof(struct index_entry))
3634 {
3635 logf("delete_entry(): write_error #2");
3636 goto cleanup;
3637 }
3638
3639 close(masterfd);
3640
3641 return true;
3642
3643 cleanup:
3644 if (fd >= 0)
3645 close(fd);
3646 if (masterfd >= 0)
3647 close(masterfd);
3648
3649 return false;
3650 }
3651
3652 #ifndef __PCTOOL__
3653 /**
3654 * Returns true if there is an event waiting in the queue
3655 * that requires the current operation to be aborted.
3656 */
3657 static bool check_event_queue(void)
3658 {
3659 struct queue_event ev;
3660
3661 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3662 switch (ev.id)
3663 {
3664 case Q_STOP_SCAN:
3665 case SYS_POWEROFF:
3666 case SYS_USB_CONNECTED:
3667 /* Put the event back into the queue. */
3668 queue_post(&tagcache_queue, ev.id, ev.data);
3669 return true;
3670 }
3671
3672 return false;
3673 }
3674 #endif
3675
3676 #ifdef HAVE_TC_RAMCACHE
3677 static bool allocate_tagcache(void)
3678 {
3679 struct master_header tcmh;
3680 int fd;
3681
3682 /* Load the header. */
3683 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3684 {
3685 hdr = NULL;
3686 return false;
3687 }
3688
3689 close(fd);
3690
3691 /**
3692 * Now calculate the required cache size plus
3693 * some extra space for alignment fixes.
3694 */
3695 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3696 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3697 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3698 memset(hdr, 0, sizeof(struct ramcache_header));
3699 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3700 hdr->indices = (struct index_entry *)(hdr + 1);
3701 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3702
3703 return true;
3704 }
3705
3706 # ifdef HAVE_EEPROM_SETTINGS
3707 static bool tagcache_dumpload(void)
3708 {
3709 struct statefile_header shdr;
3710 int fd, rc;
3711 long offpos;
3712 int i;
3713
3714 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3715 if (fd < 0)
3716 {
3717 logf("no tagcache statedump");
3718 return false;
3719 }
3720
3721 /* Check the statefile memory placement */
3722 hdr = buffer_alloc(0);
3723 rc = read(fd, &shdr, sizeof(struct statefile_header));
3724 if (rc != sizeof(struct statefile_header)
3725 /* || (long)hdr != (long)shdr.hdr */)
3726 {
3727 logf("incorrect statefile");
3728 hdr = NULL;
3729 close(fd);
3730 return false;
3731 }
3732
3733 offpos = (long)hdr - (long)shdr.hdr;
3734
3735 /* Lets allocate real memory and load it */
3736 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3737 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3738 close(fd);
3739
3740 if (rc != shdr.tc_stat.ramcache_allocated)
3741 {
3742 logf("read failure!");
3743 hdr = NULL;
3744 return false;
3745 }
3746
3747 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3748
3749 /* Now fix the pointers */
3750 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3751 for (i = 0; i < TAG_COUNT; i++)
3752 hdr->tags[i] += offpos;
3753
3754 return true;
3755 }
3756
3757 static bool tagcache_dumpsave(void)
3758 {
3759 struct statefile_header shdr;
3760 int fd;
3761
3762 if (!tc_stat.ramcache)
3763 return false;
3764
3765 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3766 if (fd < 0)
3767 {
3768 logf("failed to create a statedump");
3769 return false;
3770 }
3771
3772 /* Create the header */
3773 shdr.hdr = hdr;
3774 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3775 write(fd, &shdr, sizeof(struct statefile_header));
3776
3777 /* And dump the data too */
3778 write(fd, hdr, tc_stat.ramcache_allocated);
3779 close(fd);
3780
3781 return true;
3782 }
3783 # endif
3784
3785 static bool load_tagcache(void)
3786 {
3787 struct tagcache_header *tch;
3788 long bytesleft = tc_stat.ramcache_allocated;
3789 struct index_entry *idx;
3790 int rc, fd;
3791 char *p;
3792 int i, tag;
3793
3794 # ifdef HAVE_DIRCACHE
3795 while (dircache_is_initializing())
3796 sleep(1);
3797 # endif
3798
3799 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3800
3801 logf("loading tagcache to ram...");
3802
3803 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3804 if (fd < 0)
3805 {
3806 logf("tagcache open failed");
3807 return false;
3808 }
3809
3810 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3811 != sizeof(struct master_header)
3812 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3813 {
3814 logf("incorrect header");
3815 return false;
3816 }
3817
3818 idx = hdr->indices;
3819
3820 /* Load the master index table. */
3821 for (i = 0; i < hdr->h.tch.entry_count; i++)
3822 {
3823 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3824 if (rc != sizeof(struct index_entry))
3825 {
3826 logf("read error #10");
3827 close(fd);
3828 return false;
3829 }
3830
3831 bytesleft -= sizeof(struct index_entry);
3832 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3833 {
3834 logf("too big tagcache.");
3835 close(fd);
3836 return false;
3837 }
3838
3839 idx++;
3840 }
3841
3842 close(fd);
3843
3844 /* Load the tags. */
3845 p = (char *)idx;
3846 for (tag = 0; tag < TAG_COUNT; tag++)
3847 {
3848 struct tagfile_entry *fe;
3849 char buf[TAG_MAXLEN+32];
3850
3851 if (tagcache_is_numeric_tag(tag))
3852 continue ;
3853
3854 //p = ((void *)p+1);
3855 p = (char *)((long)p & ~0x03) + 0x04;
3856 hdr->tags[tag] = p;
3857
3858 /* Check the header. */
3859 tch = (struct tagcache_header *)p;
3860 p += sizeof(struct tagcache_header);
3861
3862 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3863 return false;
3864
3865 for (hdr->entry_count[tag] = 0;
3866 hdr->entry_count[tag] < tch->entry_count;
3867 hdr->entry_count[tag]++)
3868 {
3869 long pos;
3870
3871 if (do_timed_yield())
3872 {
3873 /* Abort if we got a critical event in queue */
3874 if (check_event_queue())
3875 return false;
3876 }
3877
3878 fe = (struct tagfile_entry *)p;
3879 pos = lseek(fd, 0, SEEK_CUR);
3880 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3881 if (rc != sizeof(struct tagfile_entry))
3882 {
3883 /* End of lookup table. */
3884 logf("read error #11");
3885 close(fd);
3886 return false;
3887 }
3888
3889 /* We have a special handling for the filename tags. */
3890 if (tag == tag_filename)
3891 {
3892 # ifdef HAVE_DIRCACHE
3893 const struct dirent *dc;
3894 # endif
3895
3896 // FIXME: This is wrong!
3897 // idx = &hdr->indices[hdr->entry_count[i]];
3898 idx = &hdr->indices[fe->idx_id];
3899
3900 if (fe->tag_length >= (long)sizeof(buf)-1)
3901 {
3902 read(fd, buf, 10);
3903 buf[10] = '\0';
3904 logf("TAG:%s", buf);
3905 logf("too long filename");
3906 close(fd);
3907 return false;
3908 }
3909
3910 rc = read(fd, buf, fe->tag_length);
3911 if (rc != fe->tag_length)
3912 {
3913 logf("read error #12");
3914 close(fd);
3915 return false;
3916 }
3917
3918 /* Check if the entry has already been removed */
3919 if (idx->flag & FLAG_DELETED)
3920 continue;
3921
3922 /* This flag must not be used yet. */
3923 if (idx->flag & FLAG_DIRCACHE)
3924 {
3925 logf("internal error!");
3926 close(fd);
3927 return false;
3928 }
3929
3930 if (idx->tag_seek[tag] != pos)
3931 {
3932 logf("corrupt data structures!");
3933 close(fd);
3934 return false;
3935 }
3936
3937 # ifdef HAVE_DIRCACHE
3938 if (dircache_is_enabled())
3939 {
3940 dc = dircache_get_entry_ptr(buf);
3941 if (dc == NULL)
3942 {
3943 logf("Entry no longer valid.");
3944 logf("-> %s", buf);
3945 delete_entry(fe->idx_id);
3946 continue ;
3947 }
3948
3949 idx->flag |= FLAG_DIRCACHE;
3950 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3951 idx->tag_seek[tag_filename] = (long)dc;
3952 }
3953 else
3954 # endif
3955 {
3956 /* This will be very slow unless dircache is enabled
3957 or target is flash based, but do it anyway for
3958 consistency. */
3959 /* Check if entry has been removed. */
3960 if (global_settings.tagcache_autoupdate)
3961 {
3962 if (!file_exists(buf))
3963 {
3964 logf("Entry no longer valid.");
3965 logf("-> %s", buf);
3966 delete_entry(fe->idx_id);
3967 continue;
3968 }
3969 }
3970 }
3971
3972 continue ;
3973 }
3974
3975 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3976 if (bytesleft < 0)
3977 {
3978 logf("too big tagcache #2");
3979 logf("tl: %d", fe->tag_length);
3980 logf("bl: %ld", bytesleft);
3981 close(fd);
3982 return false;
3983 }
3984
3985 p = fe->tag_data;
3986 rc = read(fd, fe->tag_data, fe->tag_length);
3987 p += rc;
3988
3989 if (rc != fe->tag_length)
3990 {
3991 logf("read error #13");
3992 logf("rc=0x%04x", rc); // 0x431
3993 logf("len=0x%04x", fe->tag_length); // 0x4000
3994 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3995 logf("tag=0x%02x", tag); // 0x00
3996 close(fd);
3997 return false;
3998 }
3999 }
4000 close(fd);
4001 }
4002
4003 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4004 logf("tagcache loaded into ram!");
4005
4006 return true;
4007 }
4008 #endif /* HAVE_TC_RAMCACHE */
4009
4010 static bool check_deleted_files(void)
4011 {
4012 int fd;
4013 char buf[TAG_MAXLEN+32];
4014 struct tagfile_entry tfe;
4015
4016 logf("reverse scan...");
4017 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4018 fd = open(buf, O_RDONLY);
4019
4020 if (fd < 0)
4021 {
4022 logf("%s open fail", buf);
4023 return false;
4024 }
4025
4026 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4027 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4028 == sizeof(struct tagfile_entry)
4029 #ifndef __PCTOOL__
4030 && !check_event_queue()
4031 #endif
4032 )
4033 {
4034 if (tfe.tag_length >= (long)sizeof(buf)-1)
4035 {
4036 logf("too long tag");
4037 close(fd);
4038 return false;
4039 }
4040
4041 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4042 {
4043 logf("read error #14");
4044 close(fd);
4045 return false;
4046 }
4047
4048 /* Check if the file has already deleted from the db. */
4049 if (*buf == '\0')
4050 continue;
4051
4052 /* Now check if the file exists. */
4053 if (!file_exists(buf))
4054 {
4055 logf("Entry no longer valid.");
4056 logf("-> %s / %d", buf, tfe.tag_length);
4057 delete_entry(tfe.idx_id);
4058 }
4059 }
4060
4061 close(fd);
4062
4063 logf("done");
4064
4065 return true;
4066 }
4067
4068 static bool check_dir(const char *dirname, int add_files)
4069 {
4070 DIR *dir;
4071 int len;
4072 int success = false;
4073 int ignore, unignore;
4074 char newpath[MAX_PATH];
4075
4076 dir = opendir(dirname);
4077 if (!dir)
4078 {
4079 logf("tagcache: opendir() failed");
4080 return false;
4081 }
4082
4083 /* check for a database.ignore file */
4084 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4085 ignore = file_exists(newpath);
4086 /* check for a database.unignore file */
4087 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4088 unignore = file_exists(newpath);
4089
4090 /* don't do anything if both ignore and unignore are there */
4091 if (ignore != unignore)
4092 add_files = unignore;
4093
4094 /* Recursively scan the dir. */
4095 #ifdef __PCTOOL__
4096 while (1)
4097 #else
4098 while (!check_event_queue())
4099 #endif
4100 {
4101 struct dirent *entry;
4102
4103 entry = readdir(dir);
4104
4105 if (entry == NULL)
4106 {
4107 success = true;
4108 break ;
4109 }
4110
4111 if (!strcmp((char *)entry->d_name, ".") ||
4112 !strcmp((char *)entry->d_name, ".."))
4113 continue;
4114
4115 yield();
4116
4117 len = strlen(curpath);
4118 snprintf(&curpath[len], curpath_size - len, "/%s",
4119 entry->d_name);
4120
4121 processed_dir_count++;
4122 if (entry->attribute & ATTR_DIRECTORY)
4123 check_dir(curpath, add_files);
4124 else if (add_files)
4125 {
4126 tc_stat.curentry = curpath;
4127
4128 /* Add a new entry to the temporary db file. */
4129 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4130 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4131 , dir->internal_entry
4132 #endif
4133 );
4134
4135 /* Wait until current path for debug screen is read and unset. */
4136 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4137 yield();
4138
4139 tc_stat.curentry = NULL;
4140 }
4141
4142 curpath[len] = '\0';
4143 }
4144
4145 closedir(dir);
4146
4147 return success;
4148 }
4149
4150 void tagcache_screensync_event(void)
4151 {
4152 tc_stat.curentry = NULL;
4153 }
4154
4155 void tagcache_screensync_enable(bool state)
4156 {
4157 tc_stat.syncscreen = state;
4158 }
4159
4160 void build_tagcache(const char *path)
4161 {
4162 struct tagcache_header header;
4163 bool ret;
4164
4165 curpath[0] = '\0';
4166 data_size = 0;
4167 total_entry_count = 0;
4168 processed_dir_count = 0;
4169
4170 #ifdef HAVE_DIRCACHE
4171 while (dircache_is_initializing())
4172 sleep(1);
4173 #endif
4174
4175 logf("updating tagcache");
4176
4177 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4178 if (cachefd >= 0)
4179 {
4180 logf("skipping, cache already waiting for commit");
4181 close(cachefd);
4182 return ;
4183 }
4184
4185 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4186 if (cachefd < 0)
4187 {
4188 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4189 return ;
4190 }
4191
4192 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4193
4194 cpu_boost(true);
4195
4196 logf("Scanning files...");
4197 /* Scan for new files. */
4198 memset(&header, 0, sizeof(struct tagcache_header));
4199 write(cachefd, &header, sizeof(struct tagcache_header));
4200
4201 if (strcmp("/", path) != 0)
4202 strcpy(curpath, path);
4203 ret = check_dir(path, true);
4204
4205 /* Write the header. */
4206 header.magic = TAGCACHE_MAGIC;
4207 header.datasize = data_size;
4208 header.entry_count = total_entry_count;
4209 lseek(cachefd, 0, SEEK_SET);
4210 write(cachefd, &header, sizeof(struct tagcache_header));
4211 close(cachefd);
4212
4213 if (filenametag_fd >= 0)
4214 {
4215 close(filenametag_fd);
4216 filenametag_fd = -1;
4217 }
4218
4219 if (!ret)
4220 {
4221 logf("Aborted.");
4222 cpu_boost(false);
4223 return ;
4224 }
4225
4226 /* Commit changes to the database. */
4227 #ifdef __PCTOOL__
4228 allocate_tempbuf();
4229 #endif
4230 if (commit())
4231 {
4232 remove(TAGCACHE_FILE_TEMP);
4233 logf("tagcache built!");
4234 }
4235 #ifdef __PCTOOL__
4236 free_tempbuf();
4237 #endif
4238
4239 #ifdef HAVE_TC_RAMCACHE
4240 if (hdr)
4241 {
4242 /* Import runtime statistics if we just initialized the db. */
4243 if (hdr->h.serial == 0)
4244 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4245 }
4246 #endif
4247
4248 cpu_boost(false);
4249 }
4250
4251 #ifdef HAVE_TC_RAMCACHE
4252 static void load_ramcache(void)
4253 {
4254 if (!hdr)
4255 return ;
4256
4257 cpu_boost(true);
4258
4259 /* At first we should load the cache (if exists). */
4260 tc_stat.ramcache = load_tagcache();
4261
4262 if (!tc_stat.ramcache)
4263 {
4264 /* If loading failed, it must indicate some problem with the db
4265 * so disable it entirely to prevent further issues. */
4266 tc_stat.ready = false;
4267 hdr = NULL;
4268 }
4269
4270 cpu_boost(false);
4271 }
4272
4273 void tagcache_unload_ramcache(void)
4274 {
4275 tc_stat.ramcache = false;
4276 /* Just to make sure there is no statefile present. */
4277 // remove(TAGCACHE_STATEFILE);
4278 }
4279 #endif
4280
4281 #ifndef __PCTOOL__
4282 static void tagcache_thread(void)
4283 {
4284 struct queue_event ev;
4285 bool check_done = false;
4286
4287 /* If the previous cache build/update was interrupted, commit
4288 * the changes first in foreground. */
4289 cpu_boost(true);
4290 allocate_tempbuf();
4291 commit();
4292 free_tempbuf();
4293
4294 #ifdef HAVE_TC_RAMCACHE
4295 # ifdef HAVE_EEPROM_SETTINGS
4296 if (firmware_settings.initialized && firmware_settings.disk_clean)
4297 check_done = tagcache_dumpload();
4298
4299 remove(TAGCACHE_STATEFILE);
4300 # endif
4301
4302 /* Allocate space for the tagcache if found on disk. */
4303 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4304 allocate_tagcache();
4305 #endif
4306
4307 cpu_boost(false);
4308 tc_stat.initialized = true;
4309
4310 /* Don't delay bootup with the header check but do it on background. */
4311 sleep(HZ);
4312 tc_stat.ready = check_all_headers();
4313 tc_stat.readyvalid = true;
4314
4315 while (1)
4316 {
4317 run_command_queue(false);
4318
4319 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4320
4321 switch (ev.id)
4322 {
4323 case Q_IMPORT_CHANGELOG:
4324 tagcache_import_changelog();
4325 break;
4326
4327 case Q_REBUILD:
4328 remove_files();
4329 build_tagcache("/");
4330 break;
4331
4332 case Q_UPDATE:
4333 build_tagcache("/");
4334 #ifdef HAVE_TC_RAMCACHE
4335 load_ramcache();
4336 #endif
4337 check_deleted_files();
4338 break ;
4339
4340 case Q_START_SCAN:
4341 check_done = false;
4342 case SYS_TIMEOUT:
4343 if (check_done || !tc_stat.ready)
4344 break ;
4345
4346 #ifdef HAVE_TC_RAMCACHE
4347 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4348 {
4349 load_ramcache();
4350 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4351 build_tagcache("/");
4352 }
4353 else
4354 #endif
4355 if (global_settings.tagcache_autoupdate)
4356 {
4357 build_tagcache("/");
4358
4359 /* This will be very slow unless dircache is enabled
4360 or target is flash based, but do it anyway for
4361 consistency. */
4362 check_deleted_files();
4363 }
4364
4365 logf("tagcache check done");
4366
4367 check_done = true;
4368 break ;
4369
4370 case Q_STOP_SCAN:
4371 break ;
4372
4373 case SYS_POWEROFF:
4374 break ;
4375
4376 #ifndef SIMULATOR
4377 case SYS_USB_CONNECTED:
4378 logf("USB: TagCache");
4379 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4380 usb_wait_for_disconnect(&tagcache_queue);
4381 break ;
4382 #endif
4383 }
4384 }
4385 }
4386
4387 bool tagcache_prepare_shutdown(void)
4388 {
4389 if (tagcache_get_commit_step() > 0)
4390 return false;
4391
4392 tagcache_stop_scan();
4393 while (read_lock || write_lock)
4394 sleep(1);
4395
4396 return true;
4397 }
4398
4399 void tagcache_shutdown(void)
4400 {
4401 /* Flush the command queue. */
4402 run_command_queue(true);
4403
4404 #ifdef HAVE_EEPROM_SETTINGS
4405 if (tc_stat.ramcache)
4406 tagcache_dumpsave();
4407 #endif
4408 }
4409
4410 static int get_progress(void)
4411 {
4412 int total_count = -1;
4413
4414 #ifdef HAVE_DIRCACHE
4415 if (dircache_is_enabled())
4416 {
4417 total_count = dircache_get_entry_count();
4418 }
4419 else
4420 #endif
4421 #ifdef HAVE_TC_RAMCACHE
4422 {
4423 if (hdr && tc_stat.ramcache)
4424 total_count = hdr->h.tch.entry_count;
4425 }
4426 #endif
4427
4428 if (total_count < 0)
4429 return -1;
4430
4431 return processed_dir_count * 100 / total_count;
4432 }
4433
4434 struct tagcache_stat* tagcache_get_stat(void)
4435 {
4436 tc_stat.progress = get_progress();
4437 tc_stat.processed_entries = processed_dir_count;
4438
4439 return &tc_stat;
4440 }
4441
4442 void tagcache_start_scan(void)
4443 {
4444 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4445 }
4446
4447 bool tagcache_update(void)
4448 {
4449 if (!tc_stat.ready)
4450 return false;
4451
4452 queue_post(&tagcache_queue, Q_UPDATE, 0);
4453 return false;
4454 }
4455
4456 bool tagcache_rebuild()
4457 {
4458 queue_post(&tagcache_queue, Q_REBUILD, 0);
4459 return false;
4460 }
4461
4462 void tagcache_stop_scan(void)
4463 {
4464 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4465 }
4466
4467 #ifdef HAVE_TC_RAMCACHE
4468 bool tagcache_is_ramcache(void)
4469 {
4470 return tc_stat.ramcache;
4471 }
4472 #endif
4473
4474 #endif /* !__PCTOOL__ */
4475
4476
4477 void tagcache_init(void)
4478 {
4479 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4480 memset(&current_tcmh, 0, sizeof(struct master_header));
4481 filenametag_fd = -1;
4482 write_lock = read_lock = 0;
4483
4484 #ifndef __PCTOOL__
4485 mutex_init(&command_queue_mutex);
4486 queue_init(&tagcache_queue, true);
4487 create_thread(tagcache_thread, tagcache_stack,
4488 sizeof(tagcache_stack), 0, tagcache_thread_name
4489 IF_PRIO(, PRIORITY_BACKGROUND)
4490 IF_COP(, CPU));
4491 #else
4492 tc_stat.initialized = true;
4493 allocate_tempbuf();
4494 commit();
4495 free_tempbuf();
4496 tc_stat.ready = check_all_headers();
4497 #endif
4498 }
4499
4500 #ifdef __PCTOOL__
4501 void tagcache_reverse_scan(void)
4502 {
4503 logf("Checking for deleted files");
4504 check_deleted_files();
4505 }
4506 #endif
4507
4508 bool tagcache_is_initialized(void)
4509 {
4510 return tc_stat.initialized;
4511 }
4512 bool tagcache_is_usable(void)
4513 {
4514 return tc_stat.initialized && tc_stat.ready;
4515 }
4516 int tagcache_get_commit_step(void)
4517 {
4518 return tc_stat.commit_step;
4519 }
4520 int tagcache_get_max_commit_step(void)
4521 {
4522 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4523 }
4524
Open Source Jukebox Firmware