?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Be a bit more accurate in the manual: use better options to include or exclude parts...
[kugel-rb.git] / apps / tagcache.c
blobcfa0d7073735598091c1268fd156b0937d529e6e
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "ata_idle_notify.h"
61 #include "thread.h"
62 #include "kernel.h"
63 #include "system.h"
64 #include "logf.h"
65 #include "string.h"
66 #include "usb.h"
67 #include "metadata.h"
68 #include "id3.h"
69 #include "tagcache.h"
70 #include "buffer.h"
71 #include "crc32.h"
72 #include "misc.h"
73 #include "settings.h"
74 #include "dir.h"
75 #include "structec.h"
76 #include "tagcache.h"
77
78 #ifndef __PCTOOL__
79 #include "splash.h"
80 #include "lang.h"
81 #include "eeprom_settings.h"
82 #endif
83
84 #ifdef __PCTOOL__
85 #define yield() do { } while(0)
86 #define sim_sleep(timeout) do { } while(0)
87 #define do_timed_yield() do { } while(0)
88 #endif
89
90 #ifndef __PCTOOL__
91 /* Tag Cache thread. */
92 static struct event_queue tagcache_queue;
93 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
94 static const char tagcache_thread_name[] = "tagcache";
95 #endif
96
97 #define UNTAGGED "<Untagged>"
98
99 /* Previous path when scanning directory tree recursively. */
100 static char curpath[TAG_MAXLEN+32];
101 static long curpath_size = sizeof(curpath);
102
103 /* Used when removing duplicates. */
104 static char *tempbuf; /* Allocated when needed. */
105 static long tempbufidx; /* Current location in buffer. */
106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
107 static long tempbuf_left; /* Buffer space left. */
108 static long tempbuf_pos;
109
110 /* Tags we want to get sorted (loaded to the tempbuf). */
111 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
112 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
113
114 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
115 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
116 tag_composer, tag_comment, tag_albumartist, tag_grouping };
117
118 /* Numeric tags (we can use these tags with conditional clauses). */
119 static const int numeric_tags[] = { tag_year, tag_discnumber,
120 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
121 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
122 tag_virt_length_min, tag_virt_length_sec,
123 tag_virt_playtime_min, tag_virt_playtime_sec,
124 tag_virt_entryage, tag_virt_autoscore };
125
126 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
127 static const char *tags_str[] = { "artist", "album", "genre", "title",
128 "filename", "composer", "comment", "albumartist", "grouping", "year",
129 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
130 "playtime", "lastplayed", "commitid", "mtime" };
131
132 /* Status information of the tagcache. */
133 static struct tagcache_stat tc_stat;
134
135 /* Queue commands. */
136 enum tagcache_queue {
137 Q_STOP_SCAN = 0,
138 Q_START_SCAN,
139 Q_IMPORT_CHANGELOG,
140 Q_UPDATE,
141 Q_REBUILD,
142
143 /* Internal tagcache command queue. */
144 CMD_UPDATE_MASTER_HEADER,
145 CMD_UPDATE_NUMERIC,
146 };
147
148 struct tagcache_command_entry {
149 int32_t command;
150 int32_t idx_id;
151 int32_t tag;
152 int32_t data;
153 };
154
155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
156 static volatile int command_queue_widx = 0;
157 static volatile int command_queue_ridx = 0;
158 static struct mutex command_queue_mutex;
159
160 /* Tag database structures. */
161
162 /* Variable-length tag entry in tag files. */
163 struct tagfile_entry {
164 short tag_length; /* Length of the data in bytes including '\0' */
165 short idx_id; /* Corresponding entry location in index file of not unique tags */
166 char tag_data[0]; /* Begin of the tag data */
167 };
168
169 /* Fixed-size tag entry in master db index. */
170 struct index_entry {
171 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
172 int32_t flag; /* Status flags */
173 };
174
175 /* Header is the same in every file. */
176 struct tagcache_header {
177 int32_t magic; /* Header version number */
178 int32_t datasize; /* Data size in bytes */
179 int32_t entry_count; /* Number of entries in this file */
180 };
181
182 struct master_header {
183 struct tagcache_header tch;
184 int32_t serial; /* Increasing counting number */
185 int32_t commitid; /* Number of commits so far */
186 int32_t dirty;
187 };
188
189 /* For the endianess correction */
190 static const char *tagfile_entry_ec = "ss";
191 static const char *index_entry_ec = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
192 static const char *tagcache_header_ec = "lll";
193 static const char *master_header_ec = "llllll";
194
195 static struct master_header current_tcmh;
196
197 #ifdef HAVE_TC_RAMCACHE
198 /* Header is created when loading database to ram. */
199 struct ramcache_header {
200 struct master_header h; /* Header from the master index */
201 struct index_entry *indices; /* Master index file content */
202 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
203 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
204 };
205
206 # ifdef HAVE_EEPROM_SETTINGS
207 struct statefile_header {
208 struct ramcache_header *hdr;
209 struct tagcache_stat tc_stat;
210 };
211 # endif
212
213 /* Pointer to allocated ramcache_header */
214 static struct ramcache_header *hdr;
215 #endif
216
217 /**
218 * Full tag entries stored in a temporary file waiting
219 * for commit to the cache. */
220 struct temp_file_entry {
221 long tag_offset[TAG_COUNT];
222 short tag_length[TAG_COUNT];
223 long flag;
224
225 long data_length;
226 };
227
228 struct tempbuf_id_list {
229 long id;
230 struct tempbuf_id_list *next;
231 };
232
233 struct tempbuf_searchidx {
234 long idx_id;
235 char *str;
236 int seek;
237 struct tempbuf_id_list idlist;
238 };
239
240 /* Lookup buffer for fixing messed up index while after sorting. */
241 static long commit_entry_count;
242 static long lookup_buffer_depth;
243 static struct tempbuf_searchidx **lookup;
244
245 /* Used when building the temporary file. */
246 static int cachefd = -1, filenametag_fd;
247 static int total_entry_count = 0;
248 static int data_size = 0;
249 static int processed_dir_count;
250
251 /* Thread safe locking */
252 static volatile int write_lock;
253 static volatile int read_lock;
254
255 static bool delete_entry(long idx_id);
256
257 const char* tagcache_tag_to_str(int tag)
258 {
259 return tags_str[tag];
260 }
261
262 bool tagcache_is_numeric_tag(int type)
263 {
264 int i;
265
266 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
267 {
268 if (type == numeric_tags[i])
269 return true;
270 }
271
272 return false;
273 }
274
275 bool tagcache_is_unique_tag(int type)
276 {
277 int i;
278
279 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
280 {
281 if (type == unique_tags[i])
282 return true;
283 }
284
285 return false;
286 }
287
288 bool tagcache_is_sorted_tag(int type)
289 {
290 int i;
291
292 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
293 {
294 if (type == sorted_tags[i])
295 return true;
296 }
297
298 return false;
299 }
300
301 #ifdef HAVE_DIRCACHE
302 /**
303 * Returns true if specified flag is still present, i.e., dircache
304 * has not been reloaded.
305 */
306 static bool is_dircache_intact(void)
307 {
308 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
309 }
310 #endif
311
312 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
313 {
314 int fd;
315 char buf[MAX_PATH];
316 int rc;
317
318 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
319 return -1;
320
321 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
322
323 fd = open(buf, write ? O_RDWR : O_RDONLY);
324 if (fd < 0)
325 {
326 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
327 tc_stat.ready = false;
328 return fd;
329 }
330
331 /* Check the header. */
332 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
333 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
334 {
335 logf("header error");
336 tc_stat.ready = false;
337 close(fd);
338 return -2;
339 }
340
341 return fd;
342 }
343
344 static int open_master_fd(struct master_header *hdr, bool write)
345 {
346 int fd;
347 int rc;
348
349 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
350 if (fd < 0)
351 {
352 logf("master file open failed for R/W");
353 tc_stat.ready = false;
354 return fd;
355 }
356
357 tc_stat.econ = false;
358
359 /* Check the header. */
360 rc = read(fd, hdr, sizeof(struct master_header));
361 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
362 {
363 /* Success. */
364 return fd;
365 }
366
367 /* Trying to read again, this time with endianess correction enabled. */
368 lseek(fd, 0, SEEK_SET);
369
370 rc = ecread(fd, hdr, 1, master_header_ec, true);
371 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
372 {
373 logf("header error");
374 tc_stat.ready = false;
375 close(fd);
376 return -2;
377 }
378
379 tc_stat.econ = true;
380
381 return fd;
382 }
383
384 #ifndef __PCTOOL__
385 static bool do_timed_yield(void)
386 {
387 /* Sorting can lock up for quite a while, so yield occasionally */
388 static long wakeup_tick = 0;
389 if (current_tick >= wakeup_tick)
390 {
391 wakeup_tick = current_tick + (HZ/4);
392 yield();
393 return true;
394 }
395 return false;
396 }
397 #endif
398
399 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
400 static long find_entry_ram(const char *filename,
401 const struct dirent *dc)
402 {
403 static long last_pos = 0;
404 int i;
405
406 /* Check if we tagcache is loaded into ram. */
407 if (!tc_stat.ramcache)
408 return -1;
409
410 if (dc == NULL)
411 dc = dircache_get_entry_ptr(filename);
412
413 if (dc == NULL)
414 {
415 logf("tagcache: file not found.");
416 return -1;
417 }
418
419 try_again:
420
421 if (last_pos > 0)
422 i = last_pos;
423 else
424 i = 0;
425
426 for (; i < hdr->h.tch.entry_count; i++)
427 {
428 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
429 {
430 last_pos = MAX(0, i - 3);
431 return i;
432 }
433
434 do_timed_yield();
435 }
436
437 if (last_pos > 0)
438 {
439 last_pos = 0;
440 goto try_again;
441 }
442
443 return -1;
444 }
445 #endif
446
447 static long find_entry_disk(const char *filename)
448 {
449 struct tagcache_header tch;
450 static long last_pos = -1;
451 long pos_history[POS_HISTORY_COUNT];
452 long pos_history_idx = 0;
453 bool found = false;
454 struct tagfile_entry tfe;
455 int fd;
456 char buf[TAG_MAXLEN+32];
457 int i;
458 int pos = -1;
459
460 if (!tc_stat.ready)
461 return -2;
462
463 fd = filenametag_fd;
464 if (fd < 0)
465 {
466 last_pos = -1;
467 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
468 return -1;
469 }
470
471 check_again:
472
473 if (last_pos > 0)
474 lseek(fd, last_pos, SEEK_SET);
475 else
476 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
477
478 while (true)
479 {
480 pos = lseek(fd, 0, SEEK_CUR);
481 for (i = pos_history_idx-1; i >= 0; i--)
482 pos_history[i+1] = pos_history[i];
483 pos_history[0] = pos;
484
485 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
486 != sizeof(struct tagfile_entry))
487 {
488 break ;
489 }
490
491 if (tfe.tag_length >= (long)sizeof(buf))
492 {
493 logf("too long tag #1");
494 close(fd);
495 last_pos = -1;
496 return -2;
497 }
498
499 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
500 {
501 logf("read error #2");
502 close(fd);
503 last_pos = -1;
504 return -3;
505 }
506
507 if (!strcasecmp(filename, buf))
508 {
509 last_pos = pos_history[pos_history_idx];
510 found = true;
511 break ;
512 }
513
514 if (pos_history_idx < POS_HISTORY_COUNT - 1)
515 pos_history_idx++;
516 }
517
518 /* Not found? */
519 if (!found)
520 {
521 if (last_pos > 0)
522 {
523 last_pos = -1;
524 logf("seek again");
525 goto check_again;
526 }
527
528 if (fd != filenametag_fd)
529 close(fd);
530 return -4;
531 }
532
533 if (fd != filenametag_fd)
534 close(fd);
535
536 return tfe.idx_id;
537 }
538
539 static int find_index(const char *filename)
540 {
541 long idx_id = -1;
542
543 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
544 if (tc_stat.ramcache && is_dircache_intact())
545 idx_id = find_entry_ram(filename, NULL);
546 #endif
547
548 if (idx_id < 0)
549 idx_id = find_entry_disk(filename);
550
551 return idx_id;
552 }
553
554 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
555 {
556 int idx_id;
557
558 if (!tc_stat.ready)
559 return false;
560
561 idx_id = find_index(filename);
562 if (idx_id < 0)
563 return false;
564
565 if (!tagcache_search(tcs, tag_filename))
566 return false;
567
568 tcs->entry_count = 0;
569 tcs->idx_id = idx_id;
570
571 return true;
572 }
573
574 static bool get_index(int masterfd, int idxid,
575 struct index_entry *idx, bool use_ram)
576 {
577 bool localfd = false;
578
579 if (idxid < 0)
580 {
581 logf("Incorrect idxid: %d", idxid);
582 return false;
583 }
584
585 #ifdef HAVE_TC_RAMCACHE
586 if (tc_stat.ramcache && use_ram)
587 {
588 if (hdr->indices[idxid].flag & FLAG_DELETED)
589 return false;
590
591 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
592 return true;
593 }
594 #else
595 (void)use_ram;
596 #endif
597
598 if (masterfd < 0)
599 {
600 struct master_header tcmh;
601
602 localfd = true;
603 masterfd = open_master_fd(&tcmh, false);
604 if (masterfd < 0)
605 return false;
606 }
607
608 lseek(masterfd, idxid * sizeof(struct index_entry)
609 + sizeof(struct master_header), SEEK_SET);
610 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
611 != sizeof(struct index_entry))
612 {
613 logf("read error #3");
614 if (localfd)
615 close(masterfd);
616
617 return false;
618 }
619
620 if (localfd)
621 close(masterfd);
622
623 if (idx->flag & FLAG_DELETED)
624 return false;
625
626 return true;
627 }
628
629 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
630 {
631 /* We need to exclude all memory only flags & tags when writing to disk. */
632 if (idx->flag & FLAG_DIRCACHE)
633 {
634 logf("memory only flags!");
635 return false;
636 }
637
638 #ifdef HAVE_TC_RAMCACHE
639 /* Only update numeric data. Writing the whole index to RAM by memcpy
640 * destroys dircache pointers!
641 */
642 if (tc_stat.ramcache)
643 {
644 int tag;
645 struct index_entry *idx_ram = &hdr->indices[idxid];
646
647 for (tag = 0; tag < TAG_COUNT; tag++)
648 {
649 if (tagcache_is_numeric_tag(tag))
650 {
651 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
652 }
653 }
654
655 /* Don't touch the dircache flag. */
656 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
657 }
658 #endif
659
660 lseek(masterfd, idxid * sizeof(struct index_entry)
661 + sizeof(struct master_header), SEEK_SET);
662 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
663 != sizeof(struct index_entry))
664 {
665 logf("write error #3");
666 logf("idxid: %d", idxid);
667 return false;
668 }
669
670 return true;
671 }
672
673 static bool open_files(struct tagcache_search *tcs, int tag)
674 {
675 if (tcs->idxfd[tag] < 0)
676 {
677 char fn[MAX_PATH];
678
679 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
680 tcs->idxfd[tag] = open(fn, O_RDONLY);
681 }
682
683 if (tcs->idxfd[tag] < 0)
684 {
685 logf("File not open!");
686 return false;
687 }
688
689 return true;
690 }
691
692 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
693 int tag, char *buf, long size)
694 {
695 struct tagfile_entry tfe;
696 long seek;
697
698 *buf = '\0';
699
700 if (tagcache_is_numeric_tag(tag))
701 return false;
702
703 seek = idx->tag_seek[tag];
704 if (seek < 0)
705 {
706 logf("Retrieve failed");
707 return false;
708 }
709
710 #ifdef HAVE_TC_RAMCACHE
711 if (tcs->ramsearch)
712 {
713 struct tagfile_entry *ep;
714
715 # ifdef HAVE_DIRCACHE
716 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
717 && is_dircache_intact())
718 {
719 dircache_copy_path((struct dirent *)seek,
720 buf, size);
721 return true;
722 }
723 else
724 # endif
725 if (tag != tag_filename)
726 {
727 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
728 strncpy(buf, ep->tag_data, size-1);
729
730 return true;
731 }
732 }
733 #endif
734
735 if (!open_files(tcs, tag))
736 return false;
737
738 lseek(tcs->idxfd[tag], seek, SEEK_SET);
739 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
740 != sizeof(struct tagfile_entry))
741 {
742 logf("read error #5");
743 return false;
744 }
745
746 if (tfe.tag_length >= size)
747 {
748 logf("too small buffer");
749 return false;
750 }
751
752 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
753 tfe.tag_length)
754 {
755 logf("read error #6");
756 return false;
757 }
758
759 buf[tfe.tag_length] = '\0';
760
761 return true;
762 }
763
764 static long check_virtual_tags(int tag, const struct index_entry *idx)
765 {
766 long data = 0;
767
768 switch (tag)
769 {
770 case tag_virt_length_sec:
771 data = (idx->tag_seek[tag_length]/1000) % 60;
772 break;
773
774 case tag_virt_length_min:
775 data = (idx->tag_seek[tag_length]/1000) / 60;
776 break;
777
778 case tag_virt_playtime_sec:
779 data = (idx->tag_seek[tag_playtime]/1000) % 60;
780 break;
781
782 case tag_virt_playtime_min:
783 data = (idx->tag_seek[tag_playtime]/1000) / 60;
784 break;
785
786 case tag_virt_autoscore:
787 if (idx->tag_seek[tag_length] == 0
788 || idx->tag_seek[tag_playcount] == 0)
789 {
790 data = 0;
791 }
792 else
793 {
794 data = 100 * idx->tag_seek[tag_playtime]
795 / idx->tag_seek[tag_length]
796 / idx->tag_seek[tag_playcount];
797 }
798 break;
799
800 /* How many commits before the file has been added to the DB. */
801 case tag_virt_entryage:
802 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
803 break;
804
805 default:
806 data = idx->tag_seek[tag];
807 }
808
809 return data;
810 }
811
812 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
813 {
814 struct index_entry idx;
815
816 if (!tc_stat.ready)
817 return false;
818
819 if (!tagcache_is_numeric_tag(tag))
820 return -1;
821
822 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
823 return -2;
824
825 return check_virtual_tags(tag, &idx);
826 }
827
828 inline static bool str_ends_with(const char *str1, const char *str2)
829 {
830 int str_len = strlen(str1);
831 int clause_len = strlen(str2);
832
833 if (clause_len > str_len)
834 return false;
835
836 return !strcasecmp(&str1[str_len - clause_len], str2);
837 }
838
839 inline static bool str_oneof(const char *str, const char *list)
840 {
841 const char *sep;
842 int l, len = strlen(str);
843
844 while (*list)
845 {
846 sep = strchr(list, '|');
847 l = sep ? (long)sep - (long)list : (int)strlen(list);
848 if ((l==len) && !strncasecmp(str, list, len))
849 return true;
850 list += sep ? l + 1 : l;
851 }
852
853 return false;
854 }
855
856 static bool check_against_clause(long numeric, const char *str,
857 const struct tagcache_search_clause *clause)
858 {
859 if (clause->numeric)
860 {
861 switch (clause->type)
862 {
863 case clause_is:
864 return numeric == clause->numeric_data;
865 case clause_is_not:
866 return numeric != clause->numeric_data;
867 case clause_gt:
868 return numeric > clause->numeric_data;
869 case clause_gteq:
870 return numeric >= clause->numeric_data;
871 case clause_lt:
872 return numeric < clause->numeric_data;
873 case clause_lteq:
874 return numeric <= clause->numeric_data;
875 default:
876 logf("Incorrect numeric tag: %d", clause->type);
877 }
878 }
879 else
880 {
881 switch (clause->type)
882 {
883 case clause_is:
884 return !strcasecmp(clause->str, str);
885 case clause_is_not:
886 return strcasecmp(clause->str, str);
887 case clause_gt:
888 return 0>strcasecmp(clause->str, str);
889 case clause_gteq:
890 return 0>=strcasecmp(clause->str, str);
891 case clause_lt:
892 return 0<strcasecmp(clause->str, str);
893 case clause_lteq:
894 return 0<=strcasecmp(clause->str, str);
895 case clause_contains:
896 return (strcasestr(str, clause->str) != NULL);
897 case clause_not_contains:
898 return (strcasestr(str, clause->str) == NULL);
899 case clause_begins_with:
900 return (strcasestr(str, clause->str) == str);
901 case clause_not_begins_with:
902 return (strcasestr(str, clause->str) != str);
903 case clause_ends_with:
904 return str_ends_with(str, clause->str);
905 case clause_not_ends_with:
906 return !str_ends_with(str, clause->str);
907 case clause_oneof:
908 return str_oneof(str, clause->str);
909
910 default:
911 logf("Incorrect tag: %d", clause->type);
912 }
913 }
914
915 return false;
916 }
917
918 static bool check_clauses(struct tagcache_search *tcs,
919 struct index_entry *idx,
920 struct tagcache_search_clause **clause, int count)
921 {
922 int i;
923
924 #ifdef HAVE_TC_RAMCACHE
925 if (tcs->ramsearch)
926 {
927 /* Go through all conditional clauses. */
928 for (i = 0; i < count; i++)
929 {
930 struct tagfile_entry *tfe;
931 int seek;
932 char buf[256];
933 char *str = NULL;
934
935 seek = check_virtual_tags(clause[i]->tag, idx);
936
937 if (!tagcache_is_numeric_tag(clause[i]->tag))
938 {
939 if (clause[i]->tag == tag_filename)
940 {
941 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
942 str = buf;
943 }
944 else
945 {
946 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
947 str = tfe->tag_data;
948 }
949 }
950
951 if (!check_against_clause(seek, str, clause[i]))
952 return false;
953 }
954 }
955 else
956 #endif
957 {
958 /* Check for conditions. */
959 for (i = 0; i < count; i++)
960 {
961 struct tagfile_entry tfe;
962 int seek;
963 char str[256];
964
965 seek = check_virtual_tags(clause[i]->tag, idx);
966
967 memset(str, 0, sizeof str);
968 if (!tagcache_is_numeric_tag(clause[i]->tag))
969 {
970 int fd = tcs->idxfd[clause[i]->tag];
971 lseek(fd, seek, SEEK_SET);
972 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
973 if (tfe.tag_length >= (int)sizeof(str))
974 {
975 logf("Too long tag read!");
976 break ;
977 }
978
979 read(fd, str, tfe.tag_length);
980
981 /* Check if entry has been deleted. */
982 if (str[0] == '\0')
983 break;
984 }
985
986 if (!check_against_clause(seek, str, clause[i]))
987 return false;
988 }
989 }
990
991 return true;
992 }
993
994 bool tagcache_check_clauses(struct tagcache_search *tcs,
995 struct tagcache_search_clause **clause, int count)
996 {
997 struct index_entry idx;
998
999 if (count == 0)
1000 return true;
1001
1002 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1003 return false;
1004
1005 return check_clauses(tcs, &idx, clause, count);
1006 }
1007
1008 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1009 {
1010 int i;
1011
1012 /* If uniq buffer is not defined we must return true for search to work. */
1013 if (tcs->unique_list == NULL
1014 || (!tagcache_is_unique_tag(tcs->type)
1015 && !tagcache_is_numeric_tag(tcs->type)))
1016 {
1017 return true;
1018 }
1019
1020 for (i = 0; i < tcs->unique_list_count; i++)
1021 {
1022 /* Return false if entry is found. */
1023 if (tcs->unique_list[i] == id)
1024 return false;
1025 }
1026
1027 if (tcs->unique_list_count < tcs->unique_list_capacity)
1028 {
1029 tcs->unique_list[i] = id;
1030 tcs->unique_list_count++;
1031 }
1032
1033 return true;
1034 }
1035
1036 static bool build_lookup_list(struct tagcache_search *tcs)
1037 {
1038 struct index_entry entry;
1039 int i;
1040
1041 tcs->seek_list_count = 0;
1042
1043 #ifdef HAVE_TC_RAMCACHE
1044 if (tcs->ramsearch)
1045 {
1046 int j;
1047
1048 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1049 {
1050 struct index_entry *idx = &hdr->indices[i];
1051 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1052 break ;
1053
1054 /* Skip deleted files. */
1055 if (idx->flag & FLAG_DELETED)
1056 continue;
1057
1058 /* Go through all filters.. */
1059 for (j = 0; j < tcs->filter_count; j++)
1060 {
1061 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1062 {
1063 break ;
1064 }
1065 }
1066
1067 if (j < tcs->filter_count)
1068 continue ;
1069
1070 /* Check for conditions. */
1071 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1072 continue;
1073
1074 /* Add to the seek list if not already in uniq buffer. */
1075 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1076 continue;
1077
1078 /* Lets add it. */
1079 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1080 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1081 tcs->seek_list_count++;
1082 }
1083
1084 tcs->seek_pos = i;
1085
1086 return tcs->seek_list_count > 0;
1087 }
1088 #endif
1089
1090 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1091 sizeof(struct master_header), SEEK_SET);
1092
1093 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1094 == sizeof(struct index_entry))
1095 {
1096 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1097 break ;
1098
1099 tcs->seek_pos++;
1100
1101 /* Check if entry has been deleted. */
1102 if (entry.flag & FLAG_DELETED)
1103 continue;
1104
1105 /* Go through all filters.. */
1106 for (i = 0; i < tcs->filter_count; i++)
1107 {
1108 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1109 break ;
1110 }
1111
1112 if (i < tcs->filter_count)
1113 continue ;
1114
1115 /* Check for conditions. */
1116 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1117 continue;
1118
1119 /* Add to the seek list if not already in uniq buffer. */
1120 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1121 continue;
1122
1123 /* Lets add it. */
1124 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1125 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1126 tcs->seek_list_count++;
1127
1128 yield();
1129 }
1130
1131 return tcs->seek_list_count > 0;
1132 }
1133
1134
1135 static void remove_files(void)
1136 {
1137 int i;
1138 char buf[MAX_PATH];
1139
1140 tc_stat.ready = false;
1141 tc_stat.ramcache = false;
1142 tc_stat.econ = false;
1143 remove(TAGCACHE_FILE_MASTER);
1144 for (i = 0; i < TAG_COUNT; i++)
1145 {
1146 if (tagcache_is_numeric_tag(i))
1147 continue;
1148
1149 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1150 remove(buf);
1151 }
1152 }
1153
1154
1155 static bool check_all_headers(void)
1156 {
1157 struct master_header myhdr;
1158 struct tagcache_header tch;
1159 int tag;
1160 int fd;
1161
1162 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1163 return false;
1164
1165 close(fd);
1166 if (myhdr.dirty)
1167 {
1168 logf("tagcache is dirty!");
1169 return false;
1170 }
1171
1172 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1173
1174 for (tag = 0; tag < TAG_COUNT; tag++)
1175 {
1176 if (tagcache_is_numeric_tag(tag))
1177 continue;
1178
1179 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1180 return false;
1181
1182 close(fd);
1183 }
1184
1185 return true;
1186 }
1187
1188 bool tagcache_search(struct tagcache_search *tcs, int tag)
1189 {
1190 struct tagcache_header tag_hdr;
1191 struct master_header master_hdr;
1192 int i;
1193
1194 if (tcs->initialized)
1195 tagcache_search_finish(tcs);
1196
1197 while (read_lock)
1198 sleep(1);
1199
1200 memset(tcs, 0, sizeof(struct tagcache_search));
1201 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1202 return false;
1203
1204 tcs->position = sizeof(struct tagcache_header);
1205 tcs->type = tag;
1206 tcs->seek_pos = 0;
1207 tcs->seek_list_count = 0;
1208 tcs->filter_count = 0;
1209 tcs->masterfd = -1;
1210
1211 for (i = 0; i < TAG_COUNT; i++)
1212 tcs->idxfd[i] = -1;
1213
1214 #ifndef HAVE_TC_RAMCACHE
1215 tcs->ramsearch = false;
1216 #else
1217 tcs->ramsearch = tc_stat.ramcache;
1218 if (tcs->ramsearch)
1219 {
1220 tcs->entry_count = hdr->entry_count[tcs->type];
1221 }
1222 else
1223 #endif
1224 {
1225 if (!tagcache_is_numeric_tag(tcs->type))
1226 {
1227 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1228 if (tcs->idxfd[tcs->type] < 0)
1229 return false;
1230 }
1231
1232 /* Always open as R/W so we can pass tcs to functions that modify data also
1233 * without failing. */
1234 tcs->masterfd = open_master_fd(&master_hdr, true);
1235
1236 if (tcs->masterfd < 0)
1237 return false;
1238 }
1239
1240 tcs->valid = true;
1241 tcs->initialized = true;
1242 write_lock++;
1243
1244 return true;
1245 }
1246
1247 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1248 void *buffer, long length)
1249 {
1250 tcs->unique_list = (unsigned long *)buffer;
1251 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1252 tcs->unique_list_count = 0;
1253 }
1254
1255 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1256 int tag, int seek)
1257 {
1258 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1259 return false;
1260
1261 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1262 return false;
1263
1264 tcs->filter_tag[tcs->filter_count] = tag;
1265 tcs->filter_seek[tcs->filter_count] = seek;
1266 tcs->filter_count++;
1267
1268 return true;
1269 }
1270
1271 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1272 struct tagcache_search_clause *clause)
1273 {
1274 int i;
1275
1276 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1277 {
1278 logf("Too many clauses");
1279 return false;
1280 }
1281
1282 /* Check if there is already a similar filter in present (filters are
1283 * much faster than clauses).
1284 */
1285 for (i = 0; i < tcs->filter_count; i++)
1286 {
1287 if (tcs->filter_tag[i] == clause->tag)
1288 return true;
1289 }
1290
1291 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1292 {
1293 char buf[MAX_PATH];
1294
1295 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1296 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1297 }
1298
1299 tcs->clause[tcs->clause_count] = clause;
1300 tcs->clause_count++;
1301
1302 return true;
1303 }
1304
1305 /* TODO: Remove this mess. */
1306 #ifdef HAVE_DIRCACHE
1307 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1308 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1309 #else
1310 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1311 #endif
1312
1313 static bool get_next(struct tagcache_search *tcs)
1314 {
1315 static char buf[TAG_MAXLEN+32];
1316 struct tagfile_entry entry;
1317 long flag = 0;
1318
1319 if (!tcs->valid || !tc_stat.ready)
1320 return false;
1321
1322 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1323 #ifdef HAVE_TC_RAMCACHE
1324 && !tcs->ramsearch
1325 #endif
1326 )
1327 return false;
1328
1329 /* Relative fetch. */
1330 if (tcs->filter_count > 0 || tcs->clause_count > 0
1331 || tagcache_is_numeric_tag(tcs->type))
1332 {
1333 /* Check for end of list. */
1334 if (tcs->seek_list_count == 0)
1335 {
1336 /* Try to fetch more. */
1337 if (!build_lookup_list(tcs))
1338 {
1339 tcs->valid = false;
1340 return false;
1341 }
1342 }
1343
1344 tcs->seek_list_count--;
1345 flag = tcs->seek_flags[tcs->seek_list_count];
1346
1347 /* Seek stream to the correct position and continue to direct fetch. */
1348 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1349 && !tagcache_is_numeric_tag(tcs->type))
1350 {
1351 if (!open_files(tcs, tcs->type))
1352 return false;
1353
1354 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1355 }
1356 else
1357 tcs->position = tcs->seek_list[tcs->seek_list_count];
1358 }
1359
1360 if (tagcache_is_numeric_tag(tcs->type))
1361 {
1362 snprintf(buf, sizeof(buf), "%d", tcs->position);
1363 tcs->result_seek = tcs->position;
1364 tcs->result = buf;
1365 tcs->result_len = strlen(buf) + 1;
1366 return true;
1367 }
1368
1369 /* Direct fetch. */
1370 #ifdef HAVE_TC_RAMCACHE
1371 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1372 {
1373 struct tagfile_entry *ep;
1374
1375 if (tcs->entry_count == 0)
1376 {
1377 tcs->valid = false;
1378 return false;
1379 }
1380 tcs->entry_count--;
1381
1382 tcs->result_seek = tcs->position;
1383
1384 # ifdef HAVE_DIRCACHE
1385 if (tcs->type == tag_filename)
1386 {
1387 dircache_copy_path((struct dirent *)tcs->position,
1388 buf, sizeof buf);
1389 tcs->result = buf;
1390 tcs->result_len = strlen(buf) + 1;
1391 tcs->idx_id = FLAG_GET_ATTR(flag);
1392 tcs->ramresult = false;
1393
1394 return true;
1395 }
1396 # endif
1397
1398 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1399 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1400 tcs->result = ep->tag_data;
1401 tcs->result_len = strlen(tcs->result) + 1;
1402 tcs->idx_id = ep->idx_id;
1403 tcs->ramresult = true;
1404
1405 return true;
1406 }
1407 else
1408 #endif
1409 {
1410 if (!open_files(tcs, tcs->type))
1411 return false;
1412
1413 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1414 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1415 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1416 {
1417 /* End of data. */
1418 tcs->valid = false;
1419 return false;
1420 }
1421 }
1422
1423 if (entry.tag_length > (long)sizeof(buf))
1424 {
1425 tcs->valid = false;
1426 logf("too long tag #2");
1427 return false;
1428 }
1429
1430 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1431 {
1432 tcs->valid = false;
1433 logf("read error #4");
1434 return false;
1435 }
1436
1437 tcs->result = buf;
1438 tcs->result_len = strlen(tcs->result) + 1;
1439 tcs->idx_id = entry.idx_id;
1440 tcs->ramresult = false;
1441
1442 return true;
1443 }
1444
1445 bool tagcache_get_next(struct tagcache_search *tcs)
1446 {
1447 while (get_next(tcs))
1448 {
1449 if (tcs->result_len > 1)
1450 return true;
1451 }
1452
1453 return false;
1454 }
1455
1456 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1457 int tag, char *buf, long size)
1458 {
1459 struct index_entry idx;
1460
1461 *buf = '\0';
1462 if (!get_index(tcs->masterfd, idxid, &idx, true))
1463 return false;
1464
1465 return retrieve(tcs, &idx, tag, buf, size);
1466 }
1467
1468 static bool update_master_header(void)
1469 {
1470 struct master_header myhdr;
1471 int fd;
1472
1473 if (!tc_stat.ready)
1474 return false;
1475
1476 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1477 return false;
1478
1479 myhdr.serial = current_tcmh.serial;
1480 myhdr.commitid = current_tcmh.commitid;
1481
1482 /* Write it back */
1483 lseek(fd, 0, SEEK_SET);
1484 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1485 close(fd);
1486
1487 #ifdef HAVE_TC_RAMCACHE
1488 if (hdr)
1489 {
1490 hdr->h.serial = current_tcmh.serial;
1491 hdr->h.commitid = current_tcmh.commitid;
1492 }
1493 #endif
1494
1495 return true;
1496 }
1497
1498 #if 0
1499
1500 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1501 {
1502 struct tagentry *entry;
1503
1504 if (tcs->type != tag_title)
1505 return ;
1506
1507 /* We will need reserve buffer for this. */
1508 if (tcs->ramcache)
1509 {
1510 struct tagfile_entry *ep;
1511
1512 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1513 tcs->seek_list[tcs->seek_list_count];
1514 }
1515
1516 entry = find_entry_ram();
1517
1518 }
1519 #endif
1520
1521 void tagcache_search_finish(struct tagcache_search *tcs)
1522 {
1523 int i;
1524
1525 if (!tcs->initialized)
1526 return;
1527
1528 if (tcs->masterfd >= 0)
1529 {
1530 close(tcs->masterfd);
1531 tcs->masterfd = -1;
1532 }
1533
1534 for (i = 0; i < TAG_COUNT; i++)
1535 {
1536 if (tcs->idxfd[i] >= 0)
1537 {
1538 close(tcs->idxfd[i]);
1539 tcs->idxfd[i] = -1;
1540 }
1541 }
1542
1543 tcs->ramsearch = false;
1544 tcs->valid = false;
1545 tcs->initialized = 0;
1546 if (write_lock > 0)
1547 write_lock--;
1548 }
1549
1550 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1551 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1552 {
1553 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1554 }
1555
1556 static long get_tag_numeric(const struct index_entry *entry, int tag)
1557 {
1558 return check_virtual_tags(tag, entry);
1559 }
1560
1561 static char* get_tag_string(const struct index_entry *entry, int tag)
1562 {
1563 char* s = get_tag(entry, tag)->tag_data;
1564 return strcmp(s, UNTAGGED) ? s : NULL;
1565 }
1566
1567 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1568 {
1569 struct index_entry *entry;
1570 int idx_id;
1571
1572 if (!tc_stat.ready)
1573 return false;
1574
1575 /* Find the corresponding entry in tagcache. */
1576 idx_id = find_entry_ram(filename, NULL);
1577 if (idx_id < 0 || !tc_stat.ramcache)
1578 return false;
1579
1580 entry = &hdr->indices[idx_id];
1581
1582 id3->title = get_tag_string(entry, tag_title);
1583 id3->artist = get_tag_string(entry, tag_artist);
1584 id3->album = get_tag_string(entry, tag_album);
1585 id3->genre_string = get_tag_string(entry, tag_genre);
1586 id3->composer = get_tag_string(entry, tag_composer);
1587 id3->comment = get_tag_string(entry, tag_comment);
1588 id3->albumartist = get_tag_string(entry, tag_albumartist);
1589 id3->grouping = get_tag_string(entry, tag_grouping);
1590
1591 id3->playcount = get_tag_numeric(entry, tag_playcount);
1592 id3->rating = get_tag_numeric(entry, tag_rating);
1593 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1594 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1595 id3->year = get_tag_numeric(entry, tag_year);
1596
1597 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1598 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1599 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1600 if (id3->bitrate == 0)
1601 id3->bitrate = 1;
1602
1603 return true;
1604 }
1605 #endif
1606
1607 static inline void write_item(const char *item)
1608 {
1609 int len = strlen(item) + 1;
1610
1611 data_size += len;
1612 write(cachefd, item, len);
1613 }
1614
1615 static int check_if_empty(char **tag)
1616 {
1617 int length;
1618
1619 if (*tag == NULL || **tag == '\0')
1620 {
1621 *tag = UNTAGGED;
1622 return sizeof(UNTAGGED); /* Tag length */
1623 }
1624
1625 length = strlen(*tag);
1626 if (length > TAG_MAXLEN)
1627 {
1628 logf("over length tag: %s", *tag);
1629 length = TAG_MAXLEN;
1630 (*tag)[length] = '\0';
1631 }
1632
1633 return length + 1;
1634 }
1635
1636 #define ADD_TAG(entry,tag,data) \
1637 /* Adding tag */ \
1638 entry.tag_offset[tag] = offset; \
1639 entry.tag_length[tag] = check_if_empty(data); \
1640 offset += entry.tag_length[tag]
1641
1642 static void add_tagcache(char *path, unsigned long mtime
1643 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1644 ,const struct dirent *dc
1645 #endif
1646 )
1647 {
1648 struct mp3entry id3;
1649 struct temp_file_entry entry;
1650 bool ret;
1651 int fd;
1652 int idx_id = -1;
1653 char tracknumfix[3];
1654 int offset = 0;
1655 int path_length = strlen(path);
1656 bool has_albumartist;
1657 bool has_grouping;
1658
1659 if (cachefd < 0)
1660 return ;
1661
1662 /* Check for overlength file path. */
1663 if (path_length > TAG_MAXLEN)
1664 {
1665 /* Path can't be shortened. */
1666 logf("Too long path: %s", path);
1667 return ;
1668 }
1669
1670 /* Check if the file is supported. */
1671 if (probe_file_format(path) == AFMT_UNKNOWN)
1672 return ;
1673
1674 /* Check if the file is already cached. */
1675 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1676 if (tc_stat.ramcache && is_dircache_intact())
1677 {
1678 idx_id = find_entry_ram(path, dc);
1679 }
1680 else
1681 #endif
1682 {
1683 if (filenametag_fd >= 0)
1684 {
1685 idx_id = find_entry_disk(path);
1686 }
1687 }
1688
1689 /* Check if file has been modified. */
1690 if (idx_id >= 0)
1691 {
1692 struct index_entry idx;
1693
1694 if (!get_index(-1, idx_id, &idx, true))
1695 {
1696 logf("failed to retrieve index entry");
1697 return ;
1698 }
1699
1700 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1701 {
1702 /* No changes to file. */
1703 return ;
1704 }
1705
1706 /* Metadata might have been changed. Delete the entry. */
1707 logf("Re-adding: %s", path);
1708 if (!delete_entry(idx_id))
1709 {
1710 logf("delete_entry failed: %d", idx_id);
1711 return ;
1712 }
1713 }
1714
1715 fd = open(path, O_RDONLY);
1716 if (fd < 0)
1717 {
1718 logf("open fail: %s", path);
1719 return ;
1720 }
1721
1722 memset(&id3, 0, sizeof(struct mp3entry));
1723 memset(&entry, 0, sizeof(struct temp_file_entry));
1724 memset(&tracknumfix, 0, sizeof(tracknumfix));
1725 ret = get_metadata(&id3, fd, path);
1726 close(fd);
1727
1728 if (!ret)
1729 return ;
1730
1731 logf("-> %s", path);
1732
1733 /* Generate track number if missing. */
1734 if (id3.tracknum <= 0)
1735 {
1736 const char *p = strrchr(path, '.');
1737
1738 if (p == NULL)
1739 p = &path[strlen(path)-1];
1740
1741 while (*p != '/')
1742 {
1743 if (isdigit(*p) && isdigit(*(p-1)))
1744 {
1745 tracknumfix[1] = *p--;
1746 tracknumfix[0] = *p;
1747 break;
1748 }
1749 p--;
1750 }
1751
1752 if (tracknumfix[0] != '\0')
1753 {
1754 id3.tracknum = atoi(tracknumfix);
1755 /* Set a flag to indicate track number has been generated. */
1756 entry.flag |= FLAG_TRKNUMGEN;
1757 }
1758 else
1759 {
1760 /* Unable to generate track number. */
1761 id3.tracknum = -1;
1762 }
1763 }
1764
1765 /* Numeric tags */
1766 entry.tag_offset[tag_year] = id3.year;
1767 entry.tag_offset[tag_discnumber] = id3.discnum;
1768 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1769 entry.tag_offset[tag_length] = id3.length;
1770 entry.tag_offset[tag_bitrate] = id3.bitrate;
1771 entry.tag_offset[tag_mtime] = mtime;
1772
1773 /* String tags. */
1774 has_albumartist = id3.albumartist != NULL
1775 && strlen(id3.albumartist) > 0;
1776 has_grouping = id3.grouping != NULL
1777 && strlen(id3.grouping) > 0;
1778
1779 ADD_TAG(entry, tag_filename, &path);
1780 ADD_TAG(entry, tag_title, &id3.title);
1781 ADD_TAG(entry, tag_artist, &id3.artist);
1782 ADD_TAG(entry, tag_album, &id3.album);
1783 ADD_TAG(entry, tag_genre, &id3.genre_string);
1784 ADD_TAG(entry, tag_composer, &id3.composer);
1785 ADD_TAG(entry, tag_comment, &id3.comment);
1786 if (has_albumartist)
1787 {
1788 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1789 }
1790 else
1791 {
1792 ADD_TAG(entry, tag_albumartist, &id3.artist);
1793 }
1794 if (has_grouping)
1795 {
1796 ADD_TAG(entry, tag_grouping, &id3.grouping);
1797 }
1798 else
1799 {
1800 ADD_TAG(entry, tag_grouping, &id3.title);
1801 }
1802 entry.data_length = offset;
1803
1804 /* Write the header */
1805 write(cachefd, &entry, sizeof(struct temp_file_entry));
1806
1807 /* And tags also... Correct order is critical */
1808 write_item(path);
1809 write_item(id3.title);
1810 write_item(id3.artist);
1811 write_item(id3.album);
1812 write_item(id3.genre_string);
1813 write_item(id3.composer);
1814 write_item(id3.comment);
1815 if (has_albumartist)
1816 {
1817 write_item(id3.albumartist);
1818 }
1819 else
1820 {
1821 write_item(id3.artist);
1822 }
1823 if (has_grouping)
1824 {
1825 write_item(id3.grouping);
1826 }
1827 else
1828 {
1829 write_item(id3.title);
1830 }
1831 total_entry_count++;
1832 }
1833
1834 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1835 {
1836 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1837 int len = strlen(str)+1;
1838 int i;
1839 unsigned crc32;
1840 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1841 char buf[TAG_MAXLEN+32];
1842
1843 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1844 buf[i] = tolower(str[i]);
1845 buf[i] = '\0';
1846
1847 crc32 = crc_32(buf, i, 0xffffffff);
1848
1849 if (unique)
1850 {
1851 /* Check if the crc does not exist -> entry does not exist for sure. */
1852 for (i = 0; i < tempbufidx; i++)
1853 {
1854 if (crcbuf[-i] != crc32)
1855 continue;
1856
1857 if (!strcasecmp(str, index[i].str))
1858 {
1859 if (id < 0 || id >= lookup_buffer_depth)
1860 {
1861 logf("lookup buf overf.: %d", id);
1862 return false;
1863 }
1864
1865 lookup[id] = &index[i];
1866 return true;
1867 }
1868 }
1869 }
1870
1871 /* Insert to CRC buffer. */
1872 crcbuf[-tempbufidx] = crc32;
1873 tempbuf_left -= 4;
1874
1875 /* Insert it to the buffer. */
1876 tempbuf_left -= len;
1877 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1878 return false;
1879
1880 if (id >= lookup_buffer_depth)
1881 {
1882 logf("lookup buf overf. #2: %d", id);
1883 return false;
1884 }
1885
1886 if (id >= 0)
1887 {
1888 lookup[id] = &index[tempbufidx];
1889 index[tempbufidx].idlist.id = id;
1890 }
1891 else
1892 index[tempbufidx].idlist.id = -1;
1893
1894 index[tempbufidx].idlist.next = NULL;
1895 index[tempbufidx].idx_id = idx_id;
1896 index[tempbufidx].seek = -1;
1897 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1898 memcpy(index[tempbufidx].str, str, len);
1899 tempbuf_pos += len;
1900 tempbufidx++;
1901
1902 return true;
1903 }
1904
1905 static int compare(const void *p1, const void *p2)
1906 {
1907 do_timed_yield();
1908
1909 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1910 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1911
1912 if (strcmp(e1->str, UNTAGGED) == 0)
1913 {
1914 if (strcmp(e2->str, UNTAGGED) == 0)
1915 return 0;
1916 return -1;
1917 }
1918 else if (strcmp(e2->str, UNTAGGED) == 0)
1919 return 1;
1920
1921 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1922 }
1923
1924 static int tempbuf_sort(int fd)
1925 {
1926 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1927 struct tagfile_entry fe;
1928 int i;
1929 int length;
1930
1931 /* Generate reverse lookup entries. */
1932 for (i = 0; i < lookup_buffer_depth; i++)
1933 {
1934 struct tempbuf_id_list *idlist;
1935
1936 if (!lookup[i])
1937 continue;
1938
1939 if (lookup[i]->idlist.id == i)
1940 continue;
1941
1942 idlist = &lookup[i]->idlist;
1943 while (idlist->next != NULL)
1944 idlist = idlist->next;
1945
1946 tempbuf_left -= sizeof(struct tempbuf_id_list);
1947 if (tempbuf_left - 4 < 0)
1948 return -1;
1949
1950 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1951 if (tempbuf_pos & 0x03)
1952 {
1953 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1954 tempbuf_left -= 3;
1955 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1956 }
1957 tempbuf_pos += sizeof(struct tempbuf_id_list);
1958
1959 idlist = idlist->next;
1960 idlist->id = i;
1961 idlist->next = NULL;
1962
1963 do_timed_yield();
1964 }
1965
1966 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1967 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1968
1969 for (i = 0; i < tempbufidx; i++)
1970 {
1971 struct tempbuf_id_list *idlist = &index[i].idlist;
1972
1973 /* Fix the lookup list. */
1974 while (idlist != NULL)
1975 {
1976 if (idlist->id >= 0)
1977 lookup[idlist->id] = &index[i];
1978 idlist = idlist->next;
1979 }
1980
1981 index[i].seek = lseek(fd, 0, SEEK_CUR);
1982 length = strlen(index[i].str) + 1;
1983 fe.tag_length = length;
1984 fe.idx_id = index[i].idx_id;
1985
1986 /* Check the chunk alignment. */
1987 if ((fe.tag_length + sizeof(struct tagfile_entry))
1988 % TAGFILE_ENTRY_CHUNK_LENGTH)
1989 {
1990 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1991 ((fe.tag_length + sizeof(struct tagfile_entry))
1992 % TAGFILE_ENTRY_CHUNK_LENGTH);
1993 }
1994
1995 #ifdef TAGCACHE_STRICT_ALIGN
1996 /* Make sure the entry is long aligned. */
1997 if (index[i].seek & 0x03)
1998 {
1999 logf("tempbuf_sort: alignment error!");
2000 return -3;
2001 }
2002 #endif
2003
2004 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2005 sizeof(struct tagfile_entry))
2006 {
2007 logf("tempbuf_sort: write error #1");
2008 return -1;
2009 }
2010
2011 if (write(fd, index[i].str, length) != length)
2012 {
2013 logf("tempbuf_sort: write error #2");
2014 return -2;
2015 }
2016
2017 /* Write some padding. */
2018 if (fe.tag_length - length > 0)
2019 write(fd, "XXXXXXXX", fe.tag_length - length);
2020 }
2021
2022 return i;
2023 }
2024
2025 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2026 {
2027 if (id < 0 || id >= lookup_buffer_depth)
2028 return NULL;
2029
2030 return lookup[id];
2031 }
2032
2033
2034 inline static int tempbuf_find_location(int id)
2035 {
2036 struct tempbuf_searchidx *entry;
2037
2038 entry = tempbuf_locate(id);
2039 if (entry == NULL)
2040 return -1;
2041
2042 return entry->seek;
2043 }
2044
2045 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2046 {
2047 struct master_header tcmh;
2048 struct index_entry idx;
2049 int masterfd;
2050 int masterfd_pos;
2051 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2052 int max_entries;
2053 int entries_processed = 0;
2054 int i, j;
2055 char buf[TAG_MAXLEN];
2056
2057 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2058
2059 logf("Building numeric indices...");
2060 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2061
2062 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2063 return false;
2064
2065 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2066 SEEK_CUR);
2067 if (masterfd_pos == filesize(masterfd))
2068 {
2069 logf("we can't append!");
2070 close(masterfd);
2071 return false;
2072 }
2073
2074 while (entries_processed < h->entry_count)
2075 {
2076 int count = MIN(h->entry_count - entries_processed, max_entries);
2077
2078 /* Read in as many entries as possible. */
2079 for (i = 0; i < count; i++)
2080 {
2081 struct temp_file_entry *tfe = &entrybuf[i];
2082 int datastart;
2083
2084 /* Read in numeric data. */
2085 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2086 sizeof(struct temp_file_entry))
2087 {
2088 logf("read fail #1");
2089 close(masterfd);
2090 return false;
2091 }
2092
2093 datastart = lseek(tmpfd, 0, SEEK_CUR);
2094
2095 /**
2096 * Read string data from the following tags:
2097 * - tag_filename
2098 * - tag_artist
2099 * - tag_album
2100 * - tag_title
2101 *
2102 * A crc32 hash is calculated from the read data
2103 * and stored back to the data offset field kept in memory.
2104 */
2105 #define tmpdb_read_string_tag(tag) \
2106 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2107 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2108 { \
2109 logf("read fail: buffer overflow"); \
2110 close(masterfd); \
2111 return false; \
2112 } \
2113 \
2114 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2115 tfe->tag_length[tag]) \
2116 { \
2117 logf("read fail #2"); \
2118 close(masterfd); \
2119 return false; \
2120 } \
2121 \
2122 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2123 lseek(tmpfd, datastart, SEEK_SET)
2124
2125 tmpdb_read_string_tag(tag_filename);
2126 tmpdb_read_string_tag(tag_artist);
2127 tmpdb_read_string_tag(tag_album);
2128 tmpdb_read_string_tag(tag_title);
2129
2130 /* Seek to the end of the string data. */
2131 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2132 }
2133
2134 /* Backup the master index position. */
2135 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2136 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2137
2138 /* Check if we can resurrect some deleted runtime statistics data. */
2139 for (i = 0; i < tcmh.tch.entry_count; i++)
2140 {
2141 /* Read the index entry. */
2142 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2143 != sizeof(struct index_entry))
2144 {
2145 logf("read fail #3");
2146 close(masterfd);
2147 return false;
2148 }
2149
2150 /**
2151 * Skip unless the entry is marked as being deleted
2152 * or the data has already been resurrected.
2153 */
2154 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2155 continue;
2156
2157 /* Now try to match the entry. */
2158 /**
2159 * To succesfully match a song, the following conditions
2160 * must apply:
2161 *
2162 * For numeric fields: tag_length
2163 * - Full identical match is required
2164 *
2165 * If tag_filename matches, no further checking necessary.
2166 *
2167 * For string hashes: tag_artist, tag_album, tag_title
2168 * - Two of these must match
2169 */
2170 for (j = 0; j < count; j++)
2171 {
2172 struct temp_file_entry *tfe = &entrybuf[j];
2173
2174 /* Try to match numeric fields first. */
2175 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2176 continue;
2177
2178 /* Now it's time to do the hash matching. */
2179 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2180 {
2181 int match_count = 0;
2182
2183 /* No filename match, check if we can match two other tags. */
2184 #define tmpdb_match(tag) \
2185 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2186 match_count++
2187
2188 tmpdb_match(tag_artist);
2189 tmpdb_match(tag_album);
2190 tmpdb_match(tag_title);
2191
2192 if (match_count < 2)
2193 {
2194 /* Still no match found, give up. */
2195 continue;
2196 }
2197 }
2198
2199 /* A match found, now copy & resurrect the statistical data. */
2200 #define tmpdb_copy_tag(tag) \
2201 tfe->tag_offset[tag] = idx.tag_seek[tag]
2202
2203 tmpdb_copy_tag(tag_playcount);
2204 tmpdb_copy_tag(tag_rating);
2205 tmpdb_copy_tag(tag_playtime);
2206 tmpdb_copy_tag(tag_lastplayed);
2207 tmpdb_copy_tag(tag_commitid);
2208
2209 /* Avoid processing this entry again. */
2210 idx.flag |= FLAG_RESURRECTED;
2211
2212 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2213 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2214 != sizeof(struct index_entry))
2215 {
2216 logf("masterfd writeback fail #1");
2217 close(masterfd);
2218 return false;
2219 }
2220
2221 logf("Entry resurrected");
2222 }
2223 }
2224
2225
2226 /* Restore the master index position. */
2227 lseek(masterfd, masterfd_pos, SEEK_SET);
2228
2229 /* Commit the data to the index. */
2230 for (i = 0; i < count; i++)
2231 {
2232 int loc = lseek(masterfd, 0, SEEK_CUR);
2233
2234 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2235 != sizeof(struct index_entry))
2236 {
2237 logf("read fail #3");
2238 close(masterfd);
2239 return false;
2240 }
2241
2242 for (j = 0; j < TAG_COUNT; j++)
2243 {
2244 if (!tagcache_is_numeric_tag(j))
2245 continue;
2246
2247 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2248 }
2249 idx.flag = entrybuf[i].flag;
2250
2251 if (idx.tag_seek[tag_commitid])
2252 {
2253 /* Data has been resurrected. */
2254 idx.flag |= FLAG_DIRTYNUM;
2255 }
2256 else if (tc_stat.ready && current_tcmh.commitid > 0)
2257 {
2258 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2259 idx.flag |= FLAG_DIRTYNUM;
2260 }
2261
2262 /* Write back the updated index. */
2263 lseek(masterfd, loc, SEEK_SET);
2264 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2265 != sizeof(struct index_entry))
2266 {
2267 logf("write fail");
2268 close(masterfd);
2269 return false;
2270 }
2271 }
2272
2273 entries_processed += count;
2274 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2275 }
2276
2277 close(masterfd);
2278
2279 return true;
2280 }
2281
2282 /**
2283 * Return values:
2284 * > 0 success
2285 * == 0 temporary failure
2286 * < 0 fatal error
2287 */
2288 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2289 {
2290 int i;
2291 struct tagcache_header tch;
2292 struct master_header tcmh;
2293 struct index_entry idxbuf[IDX_BUF_DEPTH];
2294 int idxbuf_pos;
2295 char buf[TAG_MAXLEN+32];
2296 int fd = -1, masterfd;
2297 bool error = false;
2298 int init;
2299 int masterfd_pos;
2300
2301 logf("Building index: %d", index_type);
2302
2303 /* Check the number of entries we need to allocate ram for. */
2304 commit_entry_count = h->entry_count + 1;
2305
2306 masterfd = open_master_fd(&tcmh, false);
2307 if (masterfd >= 0)
2308 {
2309 commit_entry_count += tcmh.tch.entry_count;
2310 close(masterfd);
2311 }
2312 else
2313 remove_files(); /* Just to be sure we are clean. */
2314
2315 /* Open the index file, which contains the tag names. */
2316 fd = open_tag_fd(&tch, index_type, true);
2317 if (fd >= 0)
2318 {
2319 logf("tch.datasize=%ld", tch.datasize);
2320 lookup_buffer_depth = 1 +
2321 /* First part */ commit_entry_count +
2322 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2323 }
2324 else
2325 {
2326 lookup_buffer_depth = 1 +
2327 /* First part */ commit_entry_count +
2328 /* Second part */ 0;
2329 }
2330
2331 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2332 logf("commit_entry_count=%ld", commit_entry_count);
2333
2334 /* Allocate buffer for all index entries from both old and new
2335 * tag files. */
2336 tempbufidx = 0;
2337 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2338
2339 /* Allocate lookup buffer. The first portion of commit_entry_count
2340 * contains the new tags in the temporary file and the second
2341 * part for locating entries already in the db.
2342 *
2343 * New tags Old tags
2344 * +---------+---------------------------+
2345 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2346 * +---------+---------------------------+
2347 *
2348 * Old tags are inserted to a temporary buffer with position:
2349 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2350 * And new tags with index:
2351 * tempbuf_insert(idx, ...);
2352 *
2353 * The buffer is sorted and written into tag file:
2354 * tempbuf_sort(...);
2355 * leaving master index locations messed up.
2356 *
2357 * That is fixed using the lookup buffer for old tags:
2358 * new_seek = tempbuf_find_location(old_seek, ...);
2359 * and for new tags:
2360 * new_seek = tempbuf_find_location(idx);
2361 */
2362 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2363 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2364 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2365
2366 /* And calculate the remaining data space used mainly for storing
2367 * tag data (strings). */
2368 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2369 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2370 {
2371 logf("Buffer way too small!");
2372 return 0;
2373 }
2374
2375 if (fd >= 0)
2376 {
2377 /**
2378 * If tag file contains unique tags (sorted index), we will load
2379 * it entirely into memory so we can resort it later for use with
2380 * chunked browsing.
2381 */
2382 if (tagcache_is_sorted_tag(index_type))
2383 {
2384 logf("loading tags...");
2385 for (i = 0; i < tch.entry_count; i++)
2386 {
2387 struct tagfile_entry entry;
2388 int loc = lseek(fd, 0, SEEK_CUR);
2389 bool ret;
2390
2391 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2392 != sizeof(struct tagfile_entry))
2393 {
2394 logf("read error #7");
2395 close(fd);
2396 return -2;
2397 }
2398
2399 if (entry.tag_length >= (int)sizeof(buf))
2400 {
2401 logf("too long tag #3");
2402 close(fd);
2403 return -2;
2404 }
2405
2406 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2407 {
2408 logf("read error #8");
2409 close(fd);
2410 return -2;
2411 }
2412
2413 /* Skip deleted entries. */
2414 if (buf[0] == '\0')
2415 continue;
2416
2417 /**
2418 * Save the tag and tag id in the memory buffer. Tag id
2419 * is saved so we can later reindex the master lookup
2420 * table when the index gets resorted.
2421 */
2422 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2423 + commit_entry_count, entry.idx_id,
2424 tagcache_is_unique_tag(index_type));
2425 if (!ret)
2426 {
2427 close(fd);
2428 return -3;
2429 }
2430 do_timed_yield();
2431 }
2432 logf("done");
2433 }
2434 else
2435 tempbufidx = tch.entry_count;
2436 }
2437 else
2438 {
2439 /**
2440 * Creating new index file to store the tags. No need to preload
2441 * anything whether the index type is sorted or not.
2442 */
2443 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2444 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2445 if (fd < 0)
2446 {
2447 logf("%s open fail", buf);
2448 return -2;
2449 }
2450
2451 tch.magic = TAGCACHE_MAGIC;
2452 tch.entry_count = 0;
2453 tch.datasize = 0;
2454
2455 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2456 != sizeof(struct tagcache_header))
2457 {
2458 logf("header write failed");
2459 close(fd);
2460 return -2;
2461 }
2462 }
2463
2464 /* Loading the tag lookup file as "master file". */
2465 logf("Loading index file");
2466 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2467
2468 if (masterfd < 0)
2469 {
2470 logf("Creating new DB");
2471 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2472
2473 if (masterfd < 0)
2474 {
2475 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2476 close(fd);
2477 return -2;
2478 }
2479
2480 /* Write the header (write real values later). */
2481 memset(&tcmh, 0, sizeof(struct master_header));
2482 tcmh.tch = *h;
2483 tcmh.tch.entry_count = 0;
2484 tcmh.tch.datasize = 0;
2485 tcmh.dirty = true;
2486 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2487 init = true;
2488 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2489 }
2490 else
2491 {
2492 /**
2493 * Master file already exists so we need to process the current
2494 * file first.
2495 */
2496 init = false;
2497
2498 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2499 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2500 {
2501 logf("header error");
2502 close(fd);
2503 close(masterfd);
2504 return -2;
2505 }
2506
2507 /**
2508 * If we reach end of the master file, we need to expand it to
2509 * hold new tags. If the current index is not sorted, we can
2510 * simply append new data to end of the file.
2511 * However, if the index is sorted, we need to update all tag
2512 * pointers in the master file for the current index.
2513 */
2514 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2515 SEEK_CUR);
2516 if (masterfd_pos == filesize(masterfd))
2517 {
2518 logf("appending...");
2519 init = true;
2520 }
2521 }
2522
2523 /**
2524 * Load new unique tags in memory to be sorted later and added
2525 * to the master lookup file.
2526 */
2527 if (tagcache_is_sorted_tag(index_type))
2528 {
2529 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2530 /* h is the header of the temporary file containing new tags. */
2531 logf("inserting new tags...");
2532 for (i = 0; i < h->entry_count; i++)
2533 {
2534 struct temp_file_entry entry;
2535
2536 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2537 sizeof(struct temp_file_entry))
2538 {
2539 logf("read fail #3");
2540 error = true;
2541 goto error_exit;
2542 }
2543
2544 /* Read data. */
2545 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2546 {
2547 logf("too long entry!");
2548 error = true;
2549 goto error_exit;
2550 }
2551
2552 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2553 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2554 entry.tag_length[index_type])
2555 {
2556 logf("read fail #4");
2557 error = true;
2558 goto error_exit;
2559 }
2560
2561 if (tagcache_is_unique_tag(index_type))
2562 error = !tempbuf_insert(buf, i, -1, true);
2563 else
2564 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2565
2566 if (error)
2567 {
2568 logf("insert error");
2569 goto error_exit;
2570 }
2571
2572 /* Skip to next. */
2573 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2574 entry.tag_length[index_type], SEEK_CUR);
2575 do_timed_yield();
2576 }
2577 logf("done");
2578
2579 /* Sort the buffer data and write it to the index file. */
2580 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2581 i = tempbuf_sort(fd);
2582 if (i < 0)
2583 goto error_exit;
2584 logf("sorted %d tags", i);
2585
2586 /**
2587 * Now update all indexes in the master lookup file.
2588 */
2589 logf("updating indices...");
2590 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2591 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2592 {
2593 int j;
2594 int loc = lseek(masterfd, 0, SEEK_CUR);
2595
2596 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2597
2598 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2599 != (int)sizeof(struct index_entry)*idxbuf_pos)
2600 {
2601 logf("read fail #5");
2602 error = true;
2603 goto error_exit ;
2604 }
2605 lseek(masterfd, loc, SEEK_SET);
2606
2607 for (j = 0; j < idxbuf_pos; j++)
2608 {
2609 if (idxbuf[j].flag & FLAG_DELETED)
2610 {
2611 /* We can just ignore deleted entries. */
2612 // idxbuf[j].tag_seek[index_type] = 0;
2613 continue;
2614 }
2615
2616 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2617 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2618 + commit_entry_count);
2619
2620 if (idxbuf[j].tag_seek[index_type] < 0)
2621 {
2622 logf("update error: %d/%d/%ld",
2623 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2624 error = true;
2625 goto error_exit;
2626 }
2627
2628 do_timed_yield();
2629 }
2630
2631 /* Write back the updated index. */
2632 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2633 index_entry_ec, tc_stat.econ) !=
2634 (int)sizeof(struct index_entry)*idxbuf_pos)
2635 {
2636 logf("write fail");
2637 error = true;
2638 goto error_exit;
2639 }
2640 }
2641 logf("done");
2642 }
2643
2644 /**
2645 * Walk through the temporary file containing the new tags.
2646 */
2647 // build_normal_index(h, tmpfd, masterfd, idx);
2648 logf("updating new indices...");
2649 lseek(masterfd, masterfd_pos, SEEK_SET);
2650 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2651 lseek(fd, 0, SEEK_END);
2652 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2653 {
2654 int j;
2655
2656 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2657 if (init)
2658 {
2659 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2660 }
2661 else
2662 {
2663 int loc = lseek(masterfd, 0, SEEK_CUR);
2664
2665 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2666 != (int)sizeof(struct index_entry)*idxbuf_pos)
2667 {
2668 logf("read fail #6");
2669 error = true;
2670 break ;
2671 }
2672 lseek(masterfd, loc, SEEK_SET);
2673 }
2674
2675 /* Read entry headers. */
2676 for (j = 0; j < idxbuf_pos; j++)
2677 {
2678 if (!tagcache_is_sorted_tag(index_type))
2679 {
2680 struct temp_file_entry entry;
2681 struct tagfile_entry fe;
2682
2683 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2684 sizeof(struct temp_file_entry))
2685 {
2686 logf("read fail #7");
2687 error = true;
2688 break ;
2689 }
2690
2691 /* Read data. */
2692 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2693 {
2694 logf("too long entry!");
2695 logf("length=%d", entry.tag_length[index_type]);
2696 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2697 error = true;
2698 break ;
2699 }
2700
2701 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2702 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2703 entry.tag_length[index_type])
2704 {
2705 logf("read fail #8");
2706 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2707 logf("length=0x%02x", entry.tag_length[index_type]);
2708 error = true;
2709 break ;
2710 }
2711
2712 /* Write to index file. */
2713 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2714 fe.tag_length = entry.tag_length[index_type];
2715 fe.idx_id = tcmh.tch.entry_count + i + j;
2716 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2717 write(fd, buf, fe.tag_length);
2718 tempbufidx++;
2719
2720 /* Skip to next. */
2721 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2722 entry.tag_length[index_type], SEEK_CUR);
2723 }
2724 else
2725 {
2726 /* Locate the correct entry from the sorted array. */
2727 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2728 if (idxbuf[j].tag_seek[index_type] < 0)
2729 {
2730 logf("entry not found (%d)", j);
2731 error = true;
2732 break ;
2733 }
2734 }
2735 }
2736
2737 /* Write index. */
2738 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2739 index_entry_ec, tc_stat.econ) !=
2740 (int)sizeof(struct index_entry)*idxbuf_pos)
2741 {
2742 logf("tagcache: write fail #4");
2743 error = true;
2744 break ;
2745 }
2746
2747 do_timed_yield();
2748 }
2749 logf("done");
2750
2751 /* Finally write the header. */
2752 tch.magic = TAGCACHE_MAGIC;
2753 tch.entry_count = tempbufidx;
2754 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2755 lseek(fd, 0, SEEK_SET);
2756 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2757
2758 if (index_type != tag_filename)
2759 h->datasize += tch.datasize;
2760 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2761 error_exit:
2762
2763 close(fd);
2764 close(masterfd);
2765
2766 if (error)
2767 return -2;
2768
2769 return 1;
2770 }
2771
2772 static bool commit(void)
2773 {
2774 struct tagcache_header tch;
2775 struct master_header tcmh;
2776 int i, len, rc;
2777 int tmpfd;
2778 int masterfd;
2779 #ifdef HAVE_DIRCACHE
2780 bool dircache_buffer_stolen = false;
2781 #endif
2782 bool local_allocation = false;
2783
2784 logf("committing tagcache");
2785
2786 while (write_lock)
2787 sleep(1);
2788
2789 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2790 if (tmpfd < 0)
2791 {
2792 logf("nothing to commit");
2793 return true;
2794 }
2795
2796
2797 /* Load the header. */
2798 len = sizeof(struct tagcache_header);
2799 rc = read(tmpfd, &tch, len);
2800
2801 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2802 {
2803 logf("incorrect header");
2804 close(tmpfd);
2805 remove(TAGCACHE_FILE_TEMP);
2806 return false;
2807 }
2808
2809 if (tch.entry_count == 0)
2810 {
2811 logf("nothing to commit");
2812 close(tmpfd);
2813 remove(TAGCACHE_FILE_TEMP);
2814 return true;
2815 }
2816
2817 #ifdef HAVE_EEPROM_SETTINGS
2818 remove(TAGCACHE_STATEFILE);
2819 #endif
2820
2821 /* At first be sure to unload the ramcache! */
2822 #ifdef HAVE_TC_RAMCACHE
2823 tc_stat.ramcache = false;
2824 #endif
2825
2826 read_lock++;
2827
2828 /* Try to steal every buffer we can :) */
2829 if (tempbuf_size == 0)
2830 local_allocation = true;
2831
2832 #ifdef HAVE_DIRCACHE
2833 if (tempbuf_size == 0)
2834 {
2835 /* Try to steal the dircache buffer. */
2836 tempbuf = dircache_steal_buffer(&tempbuf_size);
2837 tempbuf_size &= ~0x03;
2838
2839 if (tempbuf_size > 0)
2840 {
2841 dircache_buffer_stolen = true;
2842 }
2843 }
2844 #endif
2845
2846 #ifdef HAVE_TC_RAMCACHE
2847 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2848 {
2849 tempbuf = (char *)(hdr + 1);
2850 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2851 tempbuf_size &= ~0x03;
2852 }
2853 #endif
2854
2855 /* And finally fail if there are no buffers available. */
2856 if (tempbuf_size == 0)
2857 {
2858 logf("delaying commit until next boot");
2859 tc_stat.commit_delayed = true;
2860 close(tmpfd);
2861 read_lock--;
2862 return false;
2863 }
2864
2865 logf("commit %ld entries...", tch.entry_count);
2866
2867 /* Mark DB dirty so it will stay disabled if commit fails. */
2868 current_tcmh.dirty = true;
2869 update_master_header();
2870
2871 /* Now create the index files. */
2872 tc_stat.commit_step = 0;
2873 tch.datasize = 0;
2874 tc_stat.commit_delayed = false;
2875
2876 for (i = 0; i < TAG_COUNT; i++)
2877 {
2878 int ret;
2879
2880 if (tagcache_is_numeric_tag(i))
2881 continue;
2882
2883 tc_stat.commit_step++;
2884 ret = build_index(i, &tch, tmpfd);
2885 if (ret <= 0)
2886 {
2887 close(tmpfd);
2888 logf("tagcache failed init");
2889 if (ret < 0)
2890 remove_files();
2891 else
2892 tc_stat.commit_delayed = true;
2893 tc_stat.commit_step = 0;
2894 read_lock--;
2895 return false;
2896 }
2897 }
2898
2899 if (!build_numeric_indices(&tch, tmpfd))
2900 {
2901 logf("Failure to commit numeric indices");
2902 close(tmpfd);
2903 remove_files();
2904 tc_stat.commit_step = 0;
2905 read_lock--;
2906 return false;
2907 }
2908
2909 close(tmpfd);
2910 tc_stat.commit_step = 0;
2911
2912 /* Update the master index headers. */
2913 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2914 {
2915 read_lock--;
2916 return false;
2917 }
2918
2919 tcmh.tch.entry_count += tch.entry_count;
2920 tcmh.tch.datasize = sizeof(struct master_header)
2921 + sizeof(struct index_entry) * tcmh.tch.entry_count
2922 + tch.datasize;
2923 tcmh.dirty = false;
2924 tcmh.commitid++;
2925
2926 lseek(masterfd, 0, SEEK_SET);
2927 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2928 close(masterfd);
2929
2930 logf("tagcache committed");
2931 remove(TAGCACHE_FILE_TEMP);
2932 tc_stat.ready = check_all_headers();
2933 tc_stat.readyvalid = true;
2934
2935 if (local_allocation)
2936 {
2937 tempbuf = NULL;
2938 tempbuf_size = 0;
2939 }
2940
2941 #ifdef HAVE_DIRCACHE
2942 /* Rebuild the dircache, if we stole the buffer. */
2943 if (dircache_buffer_stolen)
2944 dircache_build(0);
2945 #endif
2946
2947 #ifdef HAVE_TC_RAMCACHE
2948 /* Reload tagcache. */
2949 if (tc_stat.ramcache_allocated > 0)
2950 tagcache_start_scan();
2951 #endif
2952
2953 read_lock--;
2954
2955 return true;
2956 }
2957
2958 static void allocate_tempbuf(void)
2959 {
2960 /* Yeah, malloc would be really nice now :) */
2961 #ifdef __PCTOOL__
2962 tempbuf_size = 32*1024*1024;
2963 tempbuf = malloc(tempbuf_size);
2964 #else
2965 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2966 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2967 audiobuf += tempbuf_size;
2968 #endif
2969 }
2970
2971 static void free_tempbuf(void)
2972 {
2973 if (tempbuf_size == 0)
2974 return ;
2975
2976 #ifdef __PCTOOL__
2977 free(tempbuf);
2978 #else
2979 audiobuf -= tempbuf_size;
2980 #endif
2981 tempbuf = NULL;
2982 tempbuf_size = 0;
2983 }
2984
2985 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2986 {
2987 struct index_entry idx;
2988
2989 if (!tc_stat.ready)
2990 return false;
2991
2992 if (!tagcache_is_numeric_tag(tag))
2993 return false;
2994
2995 if (!get_index(masterfd, idx_id, &idx, false))
2996 return false;
2997
2998 idx.tag_seek[tag] = data;
2999 idx.flag |= FLAG_DIRTYNUM;
3000
3001 return write_index(masterfd, idx_id, &idx);
3002 }
3003
3004 #if 0
3005 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3006 int tag, long data)
3007 {
3008 struct master_header myhdr;
3009
3010 if (tcs->masterfd < 0)
3011 {
3012 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3013 return false;
3014 }
3015
3016 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3017 }
3018 #endif
3019
3020 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3021
3022 static bool command_queue_is_full(void)
3023 {
3024 int next;
3025
3026 next = command_queue_widx + 1;
3027 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3028 next = 0;
3029
3030 return (next == command_queue_ridx);
3031 }
3032 static bool command_queue_sync_callback(void)
3033 {
3034
3035 struct master_header myhdr;
3036 int masterfd;
3037
3038 mutex_lock(&command_queue_mutex);
3039
3040 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3041 return false;
3042
3043 while (command_queue_ridx != command_queue_widx)
3044 {
3045 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3046
3047 switch (ce->command)
3048 {
3049 case CMD_UPDATE_MASTER_HEADER:
3050 {
3051 close(masterfd);
3052 update_master_header();
3053
3054 /* Re-open the masterfd. */
3055 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3056 return true;
3057
3058 break;
3059 }
3060 case CMD_UPDATE_NUMERIC:
3061 {
3062 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3063 break;
3064 }
3065 }
3066
3067 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3068 command_queue_ridx = 0;
3069 }
3070
3071 close(masterfd);
3072
3073 tc_stat.queue_length = 0;
3074 mutex_unlock(&command_queue_mutex);
3075 return true;
3076 }
3077
3078 static void run_command_queue(bool force)
3079 {
3080 if (COMMAND_QUEUE_IS_EMPTY)
3081 return;
3082
3083 if (force || command_queue_is_full())
3084 command_queue_sync_callback();
3085 else
3086 register_ata_idle_func(command_queue_sync_callback);
3087 }
3088
3089 static void queue_command(int cmd, long idx_id, int tag, long data)
3090 {
3091 while (1)
3092 {
3093 int next;
3094
3095 mutex_lock(&command_queue_mutex);
3096 next = command_queue_widx + 1;
3097 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3098 next = 0;
3099
3100 /* Make sure queue is not full. */
3101 if (next != command_queue_ridx)
3102 {
3103 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3104
3105 ce->command = cmd;
3106 ce->idx_id = idx_id;
3107 ce->tag = tag;
3108 ce->data = data;
3109
3110 command_queue_widx = next;
3111
3112 tc_stat.queue_length++;
3113
3114 mutex_unlock(&command_queue_mutex);
3115 break;
3116 }
3117
3118 /* Queue is full, try again later... */
3119 mutex_unlock(&command_queue_mutex);
3120 sleep(1);
3121 }
3122 }
3123
3124 long tagcache_increase_serial(void)
3125 {
3126 long old;
3127
3128 if (!tc_stat.ready)
3129 return -2;
3130
3131 while (read_lock)
3132 sleep(1);
3133
3134 old = current_tcmh.serial++;
3135 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3136
3137 return old;
3138 }
3139
3140 void tagcache_update_numeric(int idx_id, int tag, long data)
3141 {
3142 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3143 }
3144
3145 long tagcache_get_serial(void)
3146 {
3147 return current_tcmh.serial;
3148 }
3149
3150 long tagcache_get_commitid(void)
3151 {
3152 return current_tcmh.commitid;
3153 }
3154
3155 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3156 {
3157 char buf[512];
3158 int i;
3159
3160 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3161 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3162 {
3163 if (*datastr == '\0')
3164 break;
3165
3166 if (*datastr == '"' || *datastr == '\\')
3167 buf[i++] = '\\';
3168
3169 buf[i] = *(datastr++);
3170 }
3171
3172 strcpy(&buf[i], "\" ");
3173
3174 write(fd, buf, i + 2);
3175
3176 return true;
3177 }
3178
3179 static bool read_tag(char *dest, long size,
3180 const char *src, const char *tagstr)
3181 {
3182 int pos;
3183 char current_tag[32];
3184
3185 while (*src != '\0')
3186 {
3187 /* Skip all whitespace */
3188 while (*src == ' ')
3189 src++;
3190
3191 if (*src == '\0')
3192 break;
3193
3194 pos = 0;
3195 /* Read in tag name */
3196 while (*src != '=' && *src != ' ')
3197 {
3198 current_tag[pos] = *src;
3199 src++;
3200 pos++;
3201
3202 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3203 return false;
3204 }
3205 current_tag[pos] = '\0';
3206
3207 /* Read in tag data */
3208
3209 /* Find the start. */
3210 while (*src != '"' && *src != '\0')
3211 src++;
3212
3213 if (*src == '\0' || *(++src) == '\0')
3214 return false;
3215
3216 /* Read the data. */
3217 for (pos = 0; pos < size; pos++)
3218 {
3219 if (*src == '\0')
3220 break;
3221
3222 if (*src == '\\')
3223 {
3224 dest[pos] = *(src+1);
3225 src += 2;
3226 continue;
3227 }
3228
3229 dest[pos] = *src;
3230
3231 if (*src == '"')
3232 {
3233 src++;
3234 break;
3235 }
3236
3237 if (*src == '\0')
3238 break;
3239
3240 src++;
3241 }
3242 dest[pos] = '\0';
3243
3244 if (!strcasecmp(tagstr, current_tag))
3245 return true;
3246 }
3247
3248 return false;
3249 }
3250
3251 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3252 {
3253 struct index_entry idx;
3254 char tag_data[TAG_MAXLEN+32];
3255 int idx_id;
3256 long masterfd = (long)parameters;
3257 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3258 tag_commitid };
3259 int i;
3260 (void)line_n;
3261
3262 if (*buf == '#')
3263 return 0;
3264
3265 logf("%d/%s", line_n, buf);
3266 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3267 {
3268 logf("filename missing");
3269 logf("-> %s", buf);
3270 return 0;
3271 }
3272
3273 idx_id = find_index(tag_data);
3274 if (idx_id < 0)
3275 {
3276 logf("entry not found");
3277 return 0;
3278 }
3279
3280 if (!get_index(masterfd, idx_id, &idx, false))
3281 {
3282 logf("failed to retrieve index entry");
3283 return 0;
3284 }
3285
3286 /* Stop if tag has already been modified. */
3287 if (idx.flag & FLAG_DIRTYNUM)
3288 return 0;
3289
3290 logf("import: %s", tag_data);
3291
3292 idx.flag |= FLAG_DIRTYNUM;
3293 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3294 {
3295 int data;
3296
3297 if (!read_tag(tag_data, sizeof tag_data, buf,
3298 tagcache_tag_to_str(import_tags[i])))
3299 {
3300 continue;
3301 }
3302
3303 data = atoi(tag_data);
3304 if (data < 0)
3305 continue;
3306
3307 idx.tag_seek[import_tags[i]] = data;
3308
3309 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3310 current_tcmh.serial = data;
3311 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3312 current_tcmh.commitid = data + 1;
3313 }
3314
3315 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3316 }
3317
3318 #ifndef __PCTOOL__
3319 bool tagcache_import_changelog(void)
3320 {
3321 struct master_header myhdr;
3322 struct tagcache_header tch;
3323 int clfd;
3324 long masterfd;
3325 char buf[2048];
3326
3327 if (!tc_stat.ready)
3328 return false;
3329
3330 while (read_lock)
3331 sleep(1);
3332
3333 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3334 if (clfd < 0)
3335 {
3336 logf("failure to open changelog");
3337 return false;
3338 }
3339
3340 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3341 {
3342 close(clfd);
3343 return false;
3344 }
3345
3346 write_lock++;
3347
3348 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3349
3350 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3351 parse_changelog_line);
3352
3353 close(clfd);
3354 close(masterfd);
3355
3356 if (filenametag_fd >= 0)
3357 close(filenametag_fd);
3358
3359 write_lock--;
3360
3361 update_master_header();
3362
3363 return true;
3364 }
3365 #endif
3366
3367 bool tagcache_create_changelog(struct tagcache_search *tcs)
3368 {
3369 struct master_header myhdr;
3370 struct index_entry idx;
3371 char buf[TAG_MAXLEN+32];
3372 char temp[32];
3373 int clfd;
3374 int i, j;
3375
3376 if (!tc_stat.ready)
3377 return false;
3378
3379 if (!tagcache_search(tcs, tag_filename))
3380 return false;
3381
3382 /* Initialize the changelog */
3383 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3384 if (clfd < 0)
3385 {
3386 logf("failure to open changelog");
3387 return false;
3388 }
3389
3390 if (tcs->masterfd < 0)
3391 {
3392 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3393 return false;
3394 }
3395 else
3396 {
3397 lseek(tcs->masterfd, 0, SEEK_SET);
3398 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3399 }
3400
3401 write(clfd, "## Changelog version 1\n", 23);
3402
3403 for (i = 0; i < myhdr.tch.entry_count; i++)
3404 {
3405 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3406 != sizeof(struct index_entry))
3407 {
3408 logf("read error #9");
3409 tagcache_search_finish(tcs);
3410 close(clfd);
3411 return false;
3412 }
3413
3414 /* Skip until the entry found has been modified. */
3415 if (! (idx.flag & FLAG_DIRTYNUM) )
3416 continue;
3417
3418 /* Skip deleted entries too. */
3419 if (idx.flag & FLAG_DELETED)
3420 continue;
3421
3422 /* Now retrieve all tags. */
3423 for (j = 0; j < TAG_COUNT; j++)
3424 {
3425 if (tagcache_is_numeric_tag(j))
3426 {
3427 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3428 write_tag(clfd, tagcache_tag_to_str(j), temp);
3429 continue;
3430 }
3431
3432 tcs->type = j;
3433 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3434 write_tag(clfd, tagcache_tag_to_str(j), buf);
3435 }
3436
3437 write(clfd, "\n", 1);
3438 do_timed_yield();
3439 }
3440
3441 close(clfd);
3442
3443 tagcache_search_finish(tcs);
3444
3445 return true;
3446 }
3447
3448 static bool delete_entry(long idx_id)
3449 {
3450 int fd = -1;
3451 int masterfd = -1;
3452 int tag, i;
3453 struct index_entry idx, myidx;
3454 struct master_header myhdr;
3455 char buf[TAG_MAXLEN+32];
3456 int in_use[TAG_COUNT];
3457
3458 logf("delete_entry(): %ld", idx_id);
3459
3460 #ifdef HAVE_TC_RAMCACHE
3461 /* At first mark the entry removed from ram cache. */
3462 if (tc_stat.ramcache)
3463 hdr->indices[idx_id].flag |= FLAG_DELETED;
3464 #endif
3465
3466 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3467 return false;
3468
3469 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3470 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3471 != sizeof(struct index_entry))
3472 {
3473 logf("delete_entry(): read error");
3474 goto cleanup;
3475 }
3476
3477 if (myidx.flag & FLAG_DELETED)
3478 {
3479 logf("delete_entry(): already deleted!");
3480 goto cleanup;
3481 }
3482
3483 myidx.flag |= FLAG_DELETED;
3484 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3485 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3486 != sizeof(struct index_entry))
3487 {
3488 logf("delete_entry(): write_error #1");
3489 goto cleanup;
3490 }
3491
3492 /* Now check which tags are no longer in use (if any) */
3493 for (tag = 0; tag < TAG_COUNT; tag++)
3494 in_use[tag] = 0;
3495
3496 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3497 for (i = 0; i < myhdr.tch.entry_count; i++)
3498 {
3499 struct index_entry *idxp;
3500
3501 #ifdef HAVE_TC_RAMCACHE
3502 /* Use RAM DB if available for greater speed */
3503 if (tc_stat.ramcache)
3504 idxp = &hdr->indices[i];
3505 else
3506 #endif
3507 {
3508 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3509 != sizeof(struct index_entry))
3510 {
3511 logf("delete_entry(): read error #2");
3512 goto cleanup;
3513 }
3514 idxp = &idx;
3515 }
3516
3517 if (idxp->flag & FLAG_DELETED)
3518 continue;
3519
3520 for (tag = 0; tag < TAG_COUNT; tag++)
3521 {
3522 if (tagcache_is_numeric_tag(tag))
3523 continue;
3524
3525 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3526 in_use[tag]++;
3527 }
3528 }
3529
3530 /* Now delete all tags no longer in use. */
3531 for (tag = 0; tag < TAG_COUNT; tag++)
3532 {
3533 struct tagcache_header tch;
3534 int oldseek = myidx.tag_seek[tag];
3535
3536 if (tagcache_is_numeric_tag(tag))
3537 continue;
3538
3539 /**
3540 * Replace tag seek with a hash value of the field string data.
3541 * That way runtime statistics of moved or altered files can be
3542 * resurrected.
3543 */
3544 #ifdef HAVE_TC_RAMCACHE
3545 if (tc_stat.ramcache && tag != tag_filename)
3546 {
3547 struct tagfile_entry *tfe;
3548 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3549
3550 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3551 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3552 myidx.tag_seek[tag] = *seek;
3553 }
3554 else
3555 #endif
3556 {
3557 struct tagfile_entry tfe;
3558
3559 /* Open the index file, which contains the tag names. */
3560 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3561 goto cleanup;
3562
3563 /* Skip the header block */
3564 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3565 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3566 != sizeof(struct tagfile_entry))
3567 {
3568 logf("delete_entry(): read error #3");
3569 goto cleanup;
3570 }
3571
3572 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3573 {
3574 logf("delete_entry(): read error #4");
3575 goto cleanup;
3576 }
3577
3578 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3579 }
3580
3581 if (in_use[tag])
3582 {
3583 logf("in use: %d/%d", tag, in_use[tag]);
3584 if (fd >= 0)
3585 {
3586 close(fd);
3587 fd = -1;
3588 }
3589 continue;
3590 }
3591
3592 #ifdef HAVE_TC_RAMCACHE
3593 /* Delete from ram. */
3594 if (tc_stat.ramcache && tag != tag_filename)
3595 {
3596 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3597 tagentry->tag_data[0] = '\0';
3598 }
3599 #endif
3600
3601 /* Open the index file, which contains the tag names. */
3602 if (fd < 0)
3603 {
3604 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3605 goto cleanup;
3606 }
3607
3608 /* Skip the header block */
3609 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3610
3611 /* Debug, print 10 first characters of the tag
3612 read(fd, buf, 10);
3613 buf[10]='\0';
3614 logf("TAG:%s", buf);
3615 lseek(fd, -10, SEEK_CUR);
3616 */
3617
3618 /* Write first data byte in tag as \0 */
3619 write(fd, "", 1);
3620
3621 /* Now tag data has been removed */
3622 close(fd);
3623 fd = -1;
3624 }
3625
3626 /* Write index entry back into master index. */
3627 lseek(masterfd, sizeof(struct master_header) +
3628 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3629 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3630 != sizeof(struct index_entry))
3631 {
3632 logf("delete_entry(): write_error #2");
3633 goto cleanup;
3634 }
3635
3636 close(masterfd);
3637
3638 return true;
3639
3640 cleanup:
3641 if (fd >= 0)
3642 close(fd);
3643 if (masterfd >= 0)
3644 close(masterfd);
3645
3646 return false;
3647 }
3648
3649 #ifndef __PCTOOL__
3650 /**
3651 * Returns true if there is an event waiting in the queue
3652 * that requires the current operation to be aborted.
3653 */
3654 static bool check_event_queue(void)
3655 {
3656 struct queue_event ev;
3657
3658 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3659 switch (ev.id)
3660 {
3661 case Q_STOP_SCAN:
3662 case SYS_POWEROFF:
3663 case SYS_USB_CONNECTED:
3664 /* Put the event back into the queue. */
3665 queue_post(&tagcache_queue, ev.id, ev.data);
3666 return true;
3667 }
3668
3669 return false;
3670 }
3671 #endif
3672
3673 #ifdef HAVE_TC_RAMCACHE
3674 static bool allocate_tagcache(void)
3675 {
3676 struct master_header tcmh;
3677 int fd;
3678
3679 /* Load the header. */
3680 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3681 {
3682 hdr = NULL;
3683 return false;
3684 }
3685
3686 close(fd);
3687
3688 /**
3689 * Now calculate the required cache size plus
3690 * some extra space for alignment fixes.
3691 */
3692 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3693 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3694 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3695 memset(hdr, 0, sizeof(struct ramcache_header));
3696 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3697 hdr->indices = (struct index_entry *)(hdr + 1);
3698 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3699
3700 return true;
3701 }
3702
3703 # ifdef HAVE_EEPROM_SETTINGS
3704 static bool tagcache_dumpload(void)
3705 {
3706 struct statefile_header shdr;
3707 int fd, rc;
3708 long offpos;
3709 int i;
3710
3711 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3712 if (fd < 0)
3713 {
3714 logf("no tagcache statedump");
3715 return false;
3716 }
3717
3718 /* Check the statefile memory placement */
3719 hdr = buffer_alloc(0);
3720 rc = read(fd, &shdr, sizeof(struct statefile_header));
3721 if (rc != sizeof(struct statefile_header)
3722 /* || (long)hdr != (long)shdr.hdr */)
3723 {
3724 logf("incorrect statefile");
3725 hdr = NULL;
3726 close(fd);
3727 return false;
3728 }
3729
3730 offpos = (long)hdr - (long)shdr.hdr;
3731
3732 /* Lets allocate real memory and load it */
3733 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3734 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3735 close(fd);
3736
3737 if (rc != shdr.tc_stat.ramcache_allocated)
3738 {
3739 logf("read failure!");
3740 hdr = NULL;
3741 return false;
3742 }
3743
3744 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3745
3746 /* Now fix the pointers */
3747 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3748 for (i = 0; i < TAG_COUNT; i++)
3749 hdr->tags[i] += offpos;
3750
3751 return true;
3752 }
3753
3754 static bool tagcache_dumpsave(void)
3755 {
3756 struct statefile_header shdr;
3757 int fd;
3758
3759 if (!tc_stat.ramcache)
3760 return false;
3761
3762 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3763 if (fd < 0)
3764 {
3765 logf("failed to create a statedump");
3766 return false;
3767 }
3768
3769 /* Create the header */
3770 shdr.hdr = hdr;
3771 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3772 write(fd, &shdr, sizeof(struct statefile_header));
3773
3774 /* And dump the data too */
3775 write(fd, hdr, tc_stat.ramcache_allocated);
3776 close(fd);
3777
3778 return true;
3779 }
3780 # endif
3781
3782 static bool load_tagcache(void)
3783 {
3784 struct tagcache_header *tch;
3785 long bytesleft = tc_stat.ramcache_allocated;
3786 struct index_entry *idx;
3787 int rc, fd;
3788 char *p;
3789 int i, tag;
3790
3791 # ifdef HAVE_DIRCACHE
3792 while (dircache_is_initializing())
3793 sleep(1);
3794
3795 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3796 # endif
3797
3798 logf("loading tagcache to ram...");
3799
3800 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3801 if (fd < 0)
3802 {
3803 logf("tagcache open failed");
3804 return false;
3805 }
3806
3807 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3808 != sizeof(struct master_header)
3809 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3810 {
3811 logf("incorrect header");
3812 return false;
3813 }
3814
3815 idx = hdr->indices;
3816
3817 /* Load the master index table. */
3818 for (i = 0; i < hdr->h.tch.entry_count; i++)
3819 {
3820 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3821 if (rc != sizeof(struct index_entry))
3822 {
3823 logf("read error #10");
3824 close(fd);
3825 return false;
3826 }
3827
3828 bytesleft -= sizeof(struct index_entry);
3829 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3830 {
3831 logf("too big tagcache.");
3832 close(fd);
3833 return false;
3834 }
3835
3836 idx++;
3837 }
3838
3839 close(fd);
3840
3841 /* Load the tags. */
3842 p = (char *)idx;
3843 for (tag = 0; tag < TAG_COUNT; tag++)
3844 {
3845 struct tagfile_entry *fe;
3846 char buf[TAG_MAXLEN+32];
3847
3848 if (tagcache_is_numeric_tag(tag))
3849 continue ;
3850
3851 //p = ((void *)p+1);
3852 p = (char *)((long)p & ~0x03) + 0x04;
3853 hdr->tags[tag] = p;
3854
3855 /* Check the header. */
3856 tch = (struct tagcache_header *)p;
3857 p += sizeof(struct tagcache_header);
3858
3859 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3860 return false;
3861
3862 for (hdr->entry_count[tag] = 0;
3863 hdr->entry_count[tag] < tch->entry_count;
3864 hdr->entry_count[tag]++)
3865 {
3866 long pos;
3867
3868 if (do_timed_yield())
3869 {
3870 /* Abort if we got a critical event in queue */
3871 if (check_event_queue())
3872 return false;
3873 }
3874
3875 fe = (struct tagfile_entry *)p;
3876 pos = lseek(fd, 0, SEEK_CUR);
3877 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3878 if (rc != sizeof(struct tagfile_entry))
3879 {
3880 /* End of lookup table. */
3881 logf("read error #11");
3882 close(fd);
3883 return false;
3884 }
3885
3886 /* We have a special handling for the filename tags. */
3887 if (tag == tag_filename)
3888 {
3889 # ifdef HAVE_DIRCACHE
3890 const struct dirent *dc;
3891 # endif
3892
3893 // FIXME: This is wrong!
3894 // idx = &hdr->indices[hdr->entry_count[i]];
3895 idx = &hdr->indices[fe->idx_id];
3896
3897 if (fe->tag_length >= (long)sizeof(buf)-1)
3898 {
3899 read(fd, buf, 10);
3900 buf[10] = '\0';
3901 logf("TAG:%s", buf);
3902 logf("too long filename");
3903 close(fd);
3904 return false;
3905 }
3906
3907 rc = read(fd, buf, fe->tag_length);
3908 if (rc != fe->tag_length)
3909 {
3910 logf("read error #12");
3911 close(fd);
3912 return false;
3913 }
3914
3915 /* Check if the entry has already been removed */
3916 if (idx->flag & FLAG_DELETED)
3917 continue;
3918
3919 /* This flag must not be used yet. */
3920 if (idx->flag & FLAG_DIRCACHE)
3921 {
3922 logf("internal error!");
3923 close(fd);
3924 return false;
3925 }
3926
3927 if (idx->tag_seek[tag] != pos)
3928 {
3929 logf("corrupt data structures!");
3930 close(fd);
3931 return false;
3932 }
3933
3934 # ifdef HAVE_DIRCACHE
3935 if (dircache_is_enabled())
3936 {
3937 dc = dircache_get_entry_ptr(buf);
3938 if (dc == NULL)
3939 {
3940 logf("Entry no longer valid.");
3941 logf("-> %s", buf);
3942 delete_entry(fe->idx_id);
3943 continue ;
3944 }
3945
3946 idx->flag |= FLAG_DIRCACHE;
3947 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3948 idx->tag_seek[tag_filename] = (long)dc;
3949 }
3950 else
3951 # endif
3952 {
3953 /* This will be very slow unless dircache is enabled
3954 or target is flash based, but do it anyway for
3955 consistency. */
3956 /* Check if entry has been removed. */
3957 if (global_settings.tagcache_autoupdate)
3958 {
3959 if (!file_exists(buf))
3960 {
3961 logf("Entry no longer valid.");
3962 logf("-> %s", buf);
3963 delete_entry(fe->idx_id);
3964 continue;
3965 }
3966 }
3967 }
3968
3969 continue ;
3970 }
3971
3972 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3973 if (bytesleft < 0)
3974 {
3975 logf("too big tagcache #2");
3976 logf("tl: %d", fe->tag_length);
3977 logf("bl: %ld", bytesleft);
3978 close(fd);
3979 return false;
3980 }
3981
3982 p = fe->tag_data;
3983 rc = read(fd, fe->tag_data, fe->tag_length);
3984 p += rc;
3985
3986 if (rc != fe->tag_length)
3987 {
3988 logf("read error #13");
3989 logf("rc=0x%04x", rc); // 0x431
3990 logf("len=0x%04x", fe->tag_length); // 0x4000
3991 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3992 logf("tag=0x%02x", tag); // 0x00
3993 close(fd);
3994 return false;
3995 }
3996 }
3997 close(fd);
3998 }
3999
4000 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4001 logf("tagcache loaded into ram!");
4002
4003 return true;
4004 }
4005 #endif /* HAVE_TC_RAMCACHE */
4006
4007 static bool check_deleted_files(void)
4008 {
4009 int fd;
4010 char buf[TAG_MAXLEN+32];
4011 struct tagfile_entry tfe;
4012
4013 logf("reverse scan...");
4014 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4015 fd = open(buf, O_RDONLY);
4016
4017 if (fd < 0)
4018 {
4019 logf("%s open fail", buf);
4020 return false;
4021 }
4022
4023 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4024 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4025 == sizeof(struct tagfile_entry)
4026 #ifndef __PCTOOL__
4027 && !check_event_queue()
4028 #endif
4029 )
4030 {
4031 if (tfe.tag_length >= (long)sizeof(buf)-1)
4032 {
4033 logf("too long tag");
4034 close(fd);
4035 return false;
4036 }
4037
4038 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4039 {
4040 logf("read error #14");
4041 close(fd);
4042 return false;
4043 }
4044
4045 /* Check if the file has already deleted from the db. */
4046 if (*buf == '\0')
4047 continue;
4048
4049 /* Now check if the file exists. */
4050 if (!file_exists(buf))
4051 {
4052 logf("Entry no longer valid.");
4053 logf("-> %s / %d", buf, tfe.tag_length);
4054 delete_entry(tfe.idx_id);
4055 }
4056 }
4057
4058 close(fd);
4059
4060 logf("done");
4061
4062 return true;
4063 }
4064
4065 static bool check_dir(const char *dirname, int add_files)
4066 {
4067 DIR *dir;
4068 int len;
4069 int success = false;
4070 int ignore, unignore;
4071 char newpath[MAX_PATH];
4072
4073 dir = opendir(dirname);
4074 if (!dir)
4075 {
4076 logf("tagcache: opendir() failed");
4077 return false;
4078 }
4079
4080 /* check for a database.ignore file */
4081 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4082 ignore = file_exists(newpath);
4083 /* check for a database.unignore file */
4084 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4085 unignore = file_exists(newpath);
4086
4087 /* don't do anything if both ignore and unignore are there */
4088 if (ignore != unignore)
4089 add_files = unignore;
4090
4091 /* Recursively scan the dir. */
4092 #ifdef __PCTOOL__
4093 while (1)
4094 #else
4095 while (!check_event_queue())
4096 #endif
4097 {
4098 struct dirent *entry;
4099
4100 entry = readdir(dir);
4101
4102 if (entry == NULL)
4103 {
4104 success = true;
4105 break ;
4106 }
4107
4108 if (!strcmp((char *)entry->d_name, ".") ||
4109 !strcmp((char *)entry->d_name, ".."))
4110 continue;
4111
4112 yield();
4113
4114 len = strlen(curpath);
4115 snprintf(&curpath[len], curpath_size - len, "/%s",
4116 entry->d_name);
4117
4118 processed_dir_count++;
4119 if (entry->attribute & ATTR_DIRECTORY)
4120 check_dir(curpath, add_files);
4121 else if (add_files)
4122 {
4123 tc_stat.curentry = curpath;
4124
4125 /* Add a new entry to the temporary db file. */
4126 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4127 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4128 , dir->internal_entry
4129 #endif
4130 );
4131
4132 /* Wait until current path for debug screen is read and unset. */
4133 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4134 yield();
4135
4136 tc_stat.curentry = NULL;
4137 }
4138
4139 curpath[len] = '\0';
4140 }
4141
4142 closedir(dir);
4143
4144 return success;
4145 }
4146
4147 void tagcache_screensync_event(void)
4148 {
4149 tc_stat.curentry = NULL;
4150 }
4151
4152 void tagcache_screensync_enable(bool state)
4153 {
4154 tc_stat.syncscreen = state;
4155 }
4156
4157 void tagcache_build(const char *path)
4158 {
4159 struct tagcache_header header;
4160 bool ret;
4161
4162 curpath[0] = '\0';
4163 data_size = 0;
4164 total_entry_count = 0;
4165 processed_dir_count = 0;
4166
4167 #ifdef HAVE_DIRCACHE
4168 while (dircache_is_initializing())
4169 sleep(1);
4170 #endif
4171
4172 logf("updating tagcache");
4173
4174 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4175 if (cachefd >= 0)
4176 {
4177 logf("skipping, cache already waiting for commit");
4178 close(cachefd);
4179 return ;
4180 }
4181
4182 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4183 if (cachefd < 0)
4184 {
4185 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4186 return ;
4187 }
4188
4189 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4190
4191 cpu_boost(true);
4192
4193 logf("Scanning files...");
4194 /* Scan for new files. */
4195 memset(&header, 0, sizeof(struct tagcache_header));
4196 write(cachefd, &header, sizeof(struct tagcache_header));
4197
4198 if (strcmp("/", path) != 0)
4199 strcpy(curpath, path);
4200 ret = check_dir(path, true);
4201
4202 /* Write the header. */
4203 header.magic = TAGCACHE_MAGIC;
4204 header.datasize = data_size;
4205 header.entry_count = total_entry_count;
4206 lseek(cachefd, 0, SEEK_SET);
4207 write(cachefd, &header, sizeof(struct tagcache_header));
4208 close(cachefd);
4209
4210 if (filenametag_fd >= 0)
4211 {
4212 close(filenametag_fd);
4213 filenametag_fd = -1;
4214 }
4215
4216 if (!ret)
4217 {
4218 logf("Aborted.");
4219 cpu_boost(false);
4220 return ;
4221 }
4222
4223 /* Commit changes to the database. */
4224 #ifdef __PCTOOL__
4225 allocate_tempbuf();
4226 #endif
4227 if (commit())
4228 {
4229 remove(TAGCACHE_FILE_TEMP);
4230 logf("tagcache built!");
4231 }
4232 #ifdef __PCTOOL__
4233 free_tempbuf();
4234 #endif
4235
4236 #ifdef HAVE_TC_RAMCACHE
4237 if (hdr)
4238 {
4239 /* Import runtime statistics if we just initialized the db. */
4240 if (hdr->h.serial == 0)
4241 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4242 }
4243 #endif
4244
4245 cpu_boost(false);
4246 }
4247
4248 #ifdef HAVE_TC_RAMCACHE
4249 static void load_ramcache(void)
4250 {
4251 if (!hdr)
4252 return ;
4253
4254 cpu_boost(true);
4255
4256 /* At first we should load the cache (if exists). */
4257 tc_stat.ramcache = load_tagcache();
4258
4259 if (!tc_stat.ramcache)
4260 {
4261 /* If loading failed, it must indicate some problem with the db
4262 * so disable it entirely to prevent further issues. */
4263 tc_stat.ready = false;
4264 hdr = NULL;
4265 }
4266
4267 cpu_boost(false);
4268 }
4269
4270 void tagcache_unload_ramcache(void)
4271 {
4272 tc_stat.ramcache = false;
4273 /* Just to make sure there is no statefile present. */
4274 // remove(TAGCACHE_STATEFILE);
4275 }
4276 #endif
4277
4278 #ifndef __PCTOOL__
4279 static void tagcache_thread(void)
4280 {
4281 struct queue_event ev;
4282 bool check_done = false;
4283
4284 /* If the previous cache build/update was interrupted, commit
4285 * the changes first in foreground. */
4286 cpu_boost(true);
4287 allocate_tempbuf();
4288 commit();
4289 free_tempbuf();
4290
4291 #ifdef HAVE_TC_RAMCACHE
4292 # ifdef HAVE_EEPROM_SETTINGS
4293 if (firmware_settings.initialized && firmware_settings.disk_clean)
4294 check_done = tagcache_dumpload();
4295
4296 remove(TAGCACHE_STATEFILE);
4297 # endif
4298
4299 /* Allocate space for the tagcache if found on disk. */
4300 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4301 allocate_tagcache();
4302 #endif
4303
4304 cpu_boost(false);
4305 tc_stat.initialized = true;
4306
4307 /* Don't delay bootup with the header check but do it on background. */
4308 sleep(HZ);
4309 tc_stat.ready = check_all_headers();
4310 tc_stat.readyvalid = true;
4311
4312 while (1)
4313 {
4314 run_command_queue(false);
4315
4316 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4317
4318 switch (ev.id)
4319 {
4320 case Q_IMPORT_CHANGELOG:
4321 tagcache_import_changelog();
4322 break;
4323
4324 case Q_REBUILD:
4325 remove_files();
4326 remove(TAGCACHE_FILE_TEMP);
4327 tagcache_build("/");
4328 break;
4329
4330 case Q_UPDATE:
4331 tagcache_build("/");
4332 #ifdef HAVE_TC_RAMCACHE
4333 load_ramcache();
4334 #endif
4335 check_deleted_files();
4336 break ;
4337
4338 case Q_START_SCAN:
4339 check_done = false;
4340 case SYS_TIMEOUT:
4341 if (check_done || !tc_stat.ready)
4342 break ;
4343
4344 #ifdef HAVE_TC_RAMCACHE
4345 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4346 {
4347 load_ramcache();
4348 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4349 tagcache_build("/");
4350 }
4351 else
4352 #endif
4353 if (global_settings.tagcache_autoupdate)
4354 {
4355 tagcache_build("/");
4356
4357 /* This will be very slow unless dircache is enabled
4358 or target is flash based, but do it anyway for
4359 consistency. */
4360 check_deleted_files();
4361 }
4362
4363 logf("tagcache check done");
4364
4365 check_done = true;
4366 break ;
4367
4368 case Q_STOP_SCAN:
4369 break ;
4370
4371 case SYS_POWEROFF:
4372 break ;
4373
4374 #ifndef SIMULATOR
4375 case SYS_USB_CONNECTED:
4376 logf("USB: TagCache");
4377 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4378 usb_wait_for_disconnect(&tagcache_queue);
4379 break ;
4380 #endif
4381 }
4382 }
4383 }
4384
4385 bool tagcache_prepare_shutdown(void)
4386 {
4387 if (tagcache_get_commit_step() > 0)
4388 return false;
4389
4390 tagcache_stop_scan();
4391 while (read_lock || write_lock)
4392 sleep(1);
4393
4394 return true;
4395 }
4396
4397 void tagcache_shutdown(void)
4398 {
4399 /* Flush the command queue. */
4400 run_command_queue(true);
4401
4402 #ifdef HAVE_EEPROM_SETTINGS
4403 if (tc_stat.ramcache)
4404 tagcache_dumpsave();
4405 #endif
4406 }
4407
4408 static int get_progress(void)
4409 {
4410 int total_count = -1;
4411
4412 #ifdef HAVE_DIRCACHE
4413 if (dircache_is_enabled())
4414 {
4415 total_count = dircache_get_entry_count();
4416 }
4417 else
4418 #endif
4419 #ifdef HAVE_TC_RAMCACHE
4420 {
4421 if (hdr && tc_stat.ramcache)
4422 total_count = hdr->h.tch.entry_count;
4423 }
4424 #endif
4425
4426 if (total_count < 0)
4427 return -1;
4428
4429 return processed_dir_count * 100 / total_count;
4430 }
4431
4432 struct tagcache_stat* tagcache_get_stat(void)
4433 {
4434 tc_stat.progress = get_progress();
4435 tc_stat.processed_entries = processed_dir_count;
4436
4437 return &tc_stat;
4438 }
4439
4440 void tagcache_start_scan(void)
4441 {
4442 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4443 }
4444
4445 bool tagcache_update(void)
4446 {
4447 if (!tc_stat.ready)
4448 return false;
4449
4450 queue_post(&tagcache_queue, Q_UPDATE, 0);
4451 return false;
4452 }
4453
4454 bool tagcache_rebuild()
4455 {
4456 queue_post(&tagcache_queue, Q_REBUILD, 0);
4457 return false;
4458 }
4459
4460 void tagcache_stop_scan(void)
4461 {
4462 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4463 }
4464
4465 #ifdef HAVE_TC_RAMCACHE
4466 bool tagcache_is_ramcache(void)
4467 {
4468 return tc_stat.ramcache;
4469 }
4470 #endif
4471
4472 #endif /* !__PCTOOL__ */
4473
4474
4475 void tagcache_init(void)
4476 {
4477 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4478 memset(&current_tcmh, 0, sizeof(struct master_header));
4479 filenametag_fd = -1;
4480 write_lock = read_lock = 0;
4481
4482 #ifndef __PCTOOL__
4483 mutex_init(&command_queue_mutex);
4484 queue_init(&tagcache_queue, true);
4485 create_thread(tagcache_thread, tagcache_stack,
4486 sizeof(tagcache_stack), 0, tagcache_thread_name
4487 IF_PRIO(, PRIORITY_BACKGROUND)
4488 IF_COP(, CPU));
4489 #else
4490 tc_stat.initialized = true;
4491 allocate_tempbuf();
4492 commit();
4493 free_tempbuf();
4494 tc_stat.ready = check_all_headers();
4495 #endif
4496 }
4497
4498 #ifdef __PCTOOL__
4499 void tagcache_reverse_scan(void)
4500 {
4501 logf("Checking for deleted files");
4502 check_deleted_files();
4503 }
4504 #endif
4505
4506 bool tagcache_is_initialized(void)
4507 {
4508 return tc_stat.initialized;
4509 }
4510 bool tagcache_is_usable(void)
4511 {
4512 return tc_stat.initialized && tc_stat.ready;
4513 }
4514 int tagcache_get_commit_step(void)
4515 {
4516 return tc_stat.commit_step;
4517 }
4518 int tagcache_get_max_commit_step(void)
4519 {
4520 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4521 }
4522
My public Rockbox git repo