?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Don't include a database.ignore file in a font-only package.
[kugel-rb.git] / apps / tagcache.c
blob9455b97327785665347424bc93ad2d433203e14a
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 #include <stdio.h>
59 #include <stdlib.h>
60 #include <ctype.h>
61 #include "config.h"
62 #include "ata_idle_notify.h"
63 #include "thread.h"
64 #include "kernel.h"
65 #include "system.h"
66 #include "logf.h"
67 #include "string.h"
68 #include "usb.h"
69 #include "metadata.h"
70 #include "id3.h"
71 #include "tagcache.h"
72 #include "buffer.h"
73 #include "crc32.h"
74 #include "misc.h"
75 #include "settings.h"
76 #include "dir.h"
77 #include "structec.h"
78 #include "tagcache.h"
79
80 #ifndef __PCTOOL__
81 #include "splash.h"
82 #include "lang.h"
83 #include "eeprom_settings.h"
84 #endif
85
86 #ifdef __PCTOOL__
87 #define yield() do { } while(0)
88 #define sim_sleep(timeout) do { } while(0)
89 #define do_timed_yield() do { } while(0)
90 #endif
91
92 #ifndef __PCTOOL__
93 /* Tag Cache thread. */
94 static struct event_queue tagcache_queue;
95 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
96 static const char tagcache_thread_name[] = "tagcache";
97 #endif
98
99 #define UNTAGGED "<Untagged>"
100
101 /* Previous path when scanning directory tree recursively. */
102 static char curpath[TAG_MAXLEN+32];
103 static long curpath_size = sizeof(curpath);
104
105 /* Used when removing duplicates. */
106 static char *tempbuf; /* Allocated when needed. */
107 static long tempbufidx; /* Current location in buffer. */
108 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
109 static long tempbuf_left; /* Buffer space left. */
110 static long tempbuf_pos;
111
112 /* Tags we want to get sorted (loaded to the tempbuf). */
113 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
114 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
115
116 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
117 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
118 tag_composer, tag_comment, tag_albumartist, tag_grouping };
119
120 /* Numeric tags (we can use these tags with conditional clauses). */
121 static const int numeric_tags[] = { tag_year, tag_discnumber,
122 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
123 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
124 tag_virt_length_min, tag_virt_length_sec,
125 tag_virt_playtime_min, tag_virt_playtime_sec,
126 tag_virt_entryage, tag_virt_autoscore };
127
128 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
129 static const char *tags_str[] = { "artist", "album", "genre", "title",
130 "filename", "composer", "comment", "albumartist", "grouping", "year",
131 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
132 "playtime", "lastplayed", "commitid", "mtime" };
133
134 /* Status information of the tagcache. */
135 static struct tagcache_stat tc_stat;
136
137 /* Queue commands. */
138 enum tagcache_queue {
139 Q_STOP_SCAN = 0,
140 Q_START_SCAN,
141 Q_IMPORT_CHANGELOG,
142 Q_UPDATE,
143 Q_REBUILD,
144
145 /* Internal tagcache command queue. */
146 CMD_UPDATE_MASTER_HEADER,
147 CMD_UPDATE_NUMERIC,
148 };
149
150 struct tagcache_command_entry {
151 int32_t command;
152 int32_t idx_id;
153 int32_t tag;
154 int32_t data;
155 };
156
157 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
158 static volatile int command_queue_widx = 0;
159 static volatile int command_queue_ridx = 0;
160 static struct mutex command_queue_mutex;
161
162 /* Tag database structures. */
163
164 /* Variable-length tag entry in tag files. */
165 struct tagfile_entry {
166 short tag_length; /* Length of the data in bytes including '\0' */
167 short idx_id; /* Corresponding entry location in index file of not unique tags */
168 char tag_data[0]; /* Begin of the tag data */
169 };
170
171 /* Fixed-size tag entry in master db index. */
172 struct index_entry {
173 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
174 int32_t flag; /* Status flags */
175 };
176
177 /* Header is the same in every file. */
178 struct tagcache_header {
179 int32_t magic; /* Header version number */
180 int32_t datasize; /* Data size in bytes */
181 int32_t entry_count; /* Number of entries in this file */
182 };
183
184 struct master_header {
185 struct tagcache_header tch;
186 int32_t serial; /* Increasing counting number */
187 int32_t commitid; /* Number of commits so far */
188 int32_t dirty;
189 };
190
191 /* For the endianess correction */
192 static const char *tagfile_entry_ec = "ss";
193 static const char *index_entry_ec = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
194 static const char *tagcache_header_ec = "lll";
195 static const char *master_header_ec = "llllll";
196
197 static struct master_header current_tcmh;
198
199 #ifdef HAVE_TC_RAMCACHE
200 /* Header is created when loading database to ram. */
201 struct ramcache_header {
202 struct master_header h; /* Header from the master index */
203 struct index_entry *indices; /* Master index file content */
204 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
205 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
206 };
207
208 # ifdef HAVE_EEPROM_SETTINGS
209 struct statefile_header {
210 struct ramcache_header *hdr;
211 struct tagcache_stat tc_stat;
212 };
213 # endif
214
215 /* Pointer to allocated ramcache_header */
216 static struct ramcache_header *hdr;
217 #endif
218
219 /**
220 * Full tag entries stored in a temporary file waiting
221 * for commit to the cache. */
222 struct temp_file_entry {
223 long tag_offset[TAG_COUNT];
224 short tag_length[TAG_COUNT];
225 long flag;
226
227 long data_length;
228 };
229
230 struct tempbuf_id_list {
231 long id;
232 struct tempbuf_id_list *next;
233 };
234
235 struct tempbuf_searchidx {
236 long idx_id;
237 char *str;
238 int seek;
239 struct tempbuf_id_list idlist;
240 };
241
242 /* Lookup buffer for fixing messed up index while after sorting. */
243 static long commit_entry_count;
244 static long lookup_buffer_depth;
245 static struct tempbuf_searchidx **lookup;
246
247 /* Used when building the temporary file. */
248 static int cachefd = -1, filenametag_fd;
249 static int total_entry_count = 0;
250 static int data_size = 0;
251 static int processed_dir_count;
252
253 /* Thread safe locking */
254 static volatile int write_lock;
255 static volatile int read_lock;
256
257 static bool delete_entry(long idx_id);
258
259 const char* tagcache_tag_to_str(int tag)
260 {
261 return tags_str[tag];
262 }
263
264 bool tagcache_is_numeric_tag(int type)
265 {
266 int i;
267
268 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
269 {
270 if (type == numeric_tags[i])
271 return true;
272 }
273
274 return false;
275 }
276
277 bool tagcache_is_unique_tag(int type)
278 {
279 int i;
280
281 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
282 {
283 if (type == unique_tags[i])
284 return true;
285 }
286
287 return false;
288 }
289
290 bool tagcache_is_sorted_tag(int type)
291 {
292 int i;
293
294 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
295 {
296 if (type == sorted_tags[i])
297 return true;
298 }
299
300 return false;
301 }
302
303 #ifdef HAVE_DIRCACHE
304 /**
305 * Returns true if specified flag is still present, i.e., dircache
306 * has not been reloaded.
307 */
308 static bool is_dircache_intact(void)
309 {
310 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
311 }
312 #endif
313
314 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
315 {
316 int fd;
317 char buf[MAX_PATH];
318 int rc;
319
320 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
321 return -1;
322
323 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
324
325 fd = open(buf, write ? O_RDWR : O_RDONLY);
326 if (fd < 0)
327 {
328 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
329 tc_stat.ready = false;
330 return fd;
331 }
332
333 /* Check the header. */
334 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
335 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
336 {
337 logf("header error");
338 tc_stat.ready = false;
339 close(fd);
340 return -2;
341 }
342
343 return fd;
344 }
345
346 static int open_master_fd(struct master_header *hdr, bool write)
347 {
348 int fd;
349 int rc;
350
351 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
352 if (fd < 0)
353 {
354 logf("master file open failed for R/W");
355 tc_stat.ready = false;
356 return fd;
357 }
358
359 tc_stat.econ = false;
360
361 /* Check the header. */
362 rc = read(fd, hdr, sizeof(struct master_header));
363 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
364 {
365 /* Success. */
366 return fd;
367 }
368
369 /* Trying to read again, this time with endianess correction enabled. */
370 lseek(fd, 0, SEEK_SET);
371
372 rc = ecread(fd, hdr, 1, master_header_ec, true);
373 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
374 {
375 logf("header error");
376 tc_stat.ready = false;
377 close(fd);
378 return -2;
379 }
380
381 tc_stat.econ = true;
382
383 return fd;
384 }
385
386 #ifndef __PCTOOL__
387 static bool do_timed_yield(void)
388 {
389 /* Sorting can lock up for quite a while, so yield occasionally */
390 static long wakeup_tick = 0;
391 if (current_tick >= wakeup_tick)
392 {
393 wakeup_tick = current_tick + (HZ/4);
394 yield();
395 return true;
396 }
397 return false;
398 }
399 #endif
400
401 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
402 static long find_entry_ram(const char *filename,
403 const struct dirent *dc)
404 {
405 static long last_pos = 0;
406 int i;
407
408 /* Check if we tagcache is loaded into ram. */
409 if (!tc_stat.ramcache)
410 return -1;
411
412 if (dc == NULL)
413 dc = dircache_get_entry_ptr(filename);
414
415 if (dc == NULL)
416 {
417 logf("tagcache: file not found.");
418 return -1;
419 }
420
421 try_again:
422
423 if (last_pos > 0)
424 i = last_pos;
425 else
426 i = 0;
427
428 for (; i < hdr->h.tch.entry_count; i++)
429 {
430 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
431 {
432 last_pos = MAX(0, i - 3);
433 return i;
434 }
435
436 do_timed_yield();
437 }
438
439 if (last_pos > 0)
440 {
441 last_pos = 0;
442 goto try_again;
443 }
444
445 return -1;
446 }
447 #endif
448
449 static long find_entry_disk(const char *filename)
450 {
451 struct tagcache_header tch;
452 static long last_pos = -1;
453 long pos_history[POS_HISTORY_COUNT];
454 long pos_history_idx = 0;
455 bool found = false;
456 struct tagfile_entry tfe;
457 int fd;
458 char buf[TAG_MAXLEN+32];
459 int i;
460 int pos = -1;
461
462 if (!tc_stat.ready)
463 return -2;
464
465 fd = filenametag_fd;
466 if (fd < 0)
467 {
468 last_pos = -1;
469 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
470 return -1;
471 }
472
473 check_again:
474
475 if (last_pos > 0)
476 lseek(fd, last_pos, SEEK_SET);
477 else
478 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
479
480 while (true)
481 {
482 pos = lseek(fd, 0, SEEK_CUR);
483 for (i = pos_history_idx-1; i >= 0; i--)
484 pos_history[i+1] = pos_history[i];
485 pos_history[0] = pos;
486
487 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
488 != sizeof(struct tagfile_entry))
489 {
490 break ;
491 }
492
493 if (tfe.tag_length >= (long)sizeof(buf))
494 {
495 logf("too long tag #1");
496 close(fd);
497 last_pos = -1;
498 return -2;
499 }
500
501 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
502 {
503 logf("read error #2");
504 close(fd);
505 last_pos = -1;
506 return -3;
507 }
508
509 if (!strcasecmp(filename, buf))
510 {
511 last_pos = pos_history[pos_history_idx];
512 found = true;
513 break ;
514 }
515
516 if (pos_history_idx < POS_HISTORY_COUNT - 1)
517 pos_history_idx++;
518 }
519
520 /* Not found? */
521 if (!found)
522 {
523 if (last_pos > 0)
524 {
525 last_pos = -1;
526 logf("seek again");
527 goto check_again;
528 }
529
530 if (fd != filenametag_fd)
531 close(fd);
532 return -4;
533 }
534
535 if (fd != filenametag_fd)
536 close(fd);
537
538 return tfe.idx_id;
539 }
540
541 static int find_index(const char *filename)
542 {
543 long idx_id = -1;
544
545 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
546 if (tc_stat.ramcache && is_dircache_intact())
547 idx_id = find_entry_ram(filename, NULL);
548 #endif
549
550 if (idx_id < 0)
551 idx_id = find_entry_disk(filename);
552
553 return idx_id;
554 }
555
556 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
557 {
558 int idx_id;
559
560 if (!tc_stat.ready)
561 return false;
562
563 idx_id = find_index(filename);
564 if (idx_id < 0)
565 return false;
566
567 if (!tagcache_search(tcs, tag_filename))
568 return false;
569
570 tcs->entry_count = 0;
571 tcs->idx_id = idx_id;
572
573 return true;
574 }
575
576 static bool get_index(int masterfd, int idxid,
577 struct index_entry *idx, bool use_ram)
578 {
579 bool localfd = false;
580
581 if (idxid < 0)
582 {
583 logf("Incorrect idxid: %d", idxid);
584 return false;
585 }
586
587 #ifdef HAVE_TC_RAMCACHE
588 if (tc_stat.ramcache && use_ram)
589 {
590 if (hdr->indices[idxid].flag & FLAG_DELETED)
591 return false;
592
593 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
594 return true;
595 }
596 #else
597 (void)use_ram;
598 #endif
599
600 if (masterfd < 0)
601 {
602 struct master_header tcmh;
603
604 localfd = true;
605 masterfd = open_master_fd(&tcmh, false);
606 if (masterfd < 0)
607 return false;
608 }
609
610 lseek(masterfd, idxid * sizeof(struct index_entry)
611 + sizeof(struct master_header), SEEK_SET);
612 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
613 != sizeof(struct index_entry))
614 {
615 logf("read error #3");
616 if (localfd)
617 close(masterfd);
618
619 return false;
620 }
621
622 if (localfd)
623 close(masterfd);
624
625 if (idx->flag & FLAG_DELETED)
626 return false;
627
628 return true;
629 }
630
631 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
632 {
633 /* We need to exclude all memory only flags & tags when writing to disk. */
634 if (idx->flag & FLAG_DIRCACHE)
635 {
636 logf("memory only flags!");
637 return false;
638 }
639
640 #ifdef HAVE_TC_RAMCACHE
641 /* Only update numeric data. Writing the whole index to RAM by memcpy
642 * destroys dircache pointers!
643 */
644 if (tc_stat.ramcache)
645 {
646 int tag;
647 struct index_entry *idx_ram = &hdr->indices[idxid];
648
649 for (tag = 0; tag < TAG_COUNT; tag++)
650 {
651 if (tagcache_is_numeric_tag(tag))
652 {
653 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
654 }
655 }
656
657 /* Don't touch the dircache flag. */
658 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
659 }
660 #endif
661
662 lseek(masterfd, idxid * sizeof(struct index_entry)
663 + sizeof(struct master_header), SEEK_SET);
664 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
665 != sizeof(struct index_entry))
666 {
667 logf("write error #3");
668 logf("idxid: %d", idxid);
669 return false;
670 }
671
672 return true;
673 }
674
675 static bool open_files(struct tagcache_search *tcs, int tag)
676 {
677 if (tcs->idxfd[tag] < 0)
678 {
679 char fn[MAX_PATH];
680
681 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
682 tcs->idxfd[tag] = open(fn, O_RDONLY);
683 }
684
685 if (tcs->idxfd[tag] < 0)
686 {
687 logf("File not open!");
688 return false;
689 }
690
691 return true;
692 }
693
694 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
695 int tag, char *buf, long size)
696 {
697 struct tagfile_entry tfe;
698 long seek;
699
700 *buf = '\0';
701
702 if (tagcache_is_numeric_tag(tag))
703 return false;
704
705 seek = idx->tag_seek[tag];
706 if (seek < 0)
707 {
708 logf("Retrieve failed");
709 return false;
710 }
711
712 #ifdef HAVE_TC_RAMCACHE
713 if (tcs->ramsearch)
714 {
715 struct tagfile_entry *ep;
716
717 # ifdef HAVE_DIRCACHE
718 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
719 && is_dircache_intact())
720 {
721 dircache_copy_path((struct dirent *)seek,
722 buf, size);
723 return true;
724 }
725 else
726 # endif
727 if (tag != tag_filename)
728 {
729 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
730 strncpy(buf, ep->tag_data, size-1);
731
732 return true;
733 }
734 }
735 #endif
736
737 if (!open_files(tcs, tag))
738 return false;
739
740 lseek(tcs->idxfd[tag], seek, SEEK_SET);
741 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
742 != sizeof(struct tagfile_entry))
743 {
744 logf("read error #5");
745 return false;
746 }
747
748 if (tfe.tag_length >= size)
749 {
750 logf("too small buffer");
751 return false;
752 }
753
754 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
755 tfe.tag_length)
756 {
757 logf("read error #6");
758 return false;
759 }
760
761 buf[tfe.tag_length] = '\0';
762
763 return true;
764 }
765
766 static long check_virtual_tags(int tag, const struct index_entry *idx)
767 {
768 long data = 0;
769
770 switch (tag)
771 {
772 case tag_virt_length_sec:
773 data = (idx->tag_seek[tag_length]/1000) % 60;
774 break;
775
776 case tag_virt_length_min:
777 data = (idx->tag_seek[tag_length]/1000) / 60;
778 break;
779
780 case tag_virt_playtime_sec:
781 data = (idx->tag_seek[tag_playtime]/1000) % 60;
782 break;
783
784 case tag_virt_playtime_min:
785 data = (idx->tag_seek[tag_playtime]/1000) / 60;
786 break;
787
788 case tag_virt_autoscore:
789 if (idx->tag_seek[tag_length] == 0
790 || idx->tag_seek[tag_playcount] == 0)
791 {
792 data = 0;
793 }
794 else
795 {
796 data = 100 * idx->tag_seek[tag_playtime]
797 / idx->tag_seek[tag_length]
798 / idx->tag_seek[tag_playcount];
799 }
800 break;
801
802 /* How many commits before the file has been added to the DB. */
803 case tag_virt_entryage:
804 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
805 break;
806
807 default:
808 data = idx->tag_seek[tag];
809 }
810
811 return data;
812 }
813
814 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
815 {
816 struct index_entry idx;
817
818 if (!tc_stat.ready)
819 return false;
820
821 if (!tagcache_is_numeric_tag(tag))
822 return -1;
823
824 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
825 return -2;
826
827 return check_virtual_tags(tag, &idx);
828 }
829
830 inline static bool str_ends_with(const char *str1, const char *str2)
831 {
832 int str_len = strlen(str1);
833 int clause_len = strlen(str2);
834
835 if (clause_len > str_len)
836 return false;
837
838 return !strcasecmp(&str1[str_len - clause_len], str2);
839 }
840
841 inline static bool str_oneof(const char *str, const char *list)
842 {
843 const char *sep;
844 int l, len = strlen(str);
845
846 while (*list)
847 {
848 sep = strchr(list, '|');
849 l = sep ? (long)sep - (long)list : (int)strlen(list);
850 if ((l==len) && !strncasecmp(str, list, len))
851 return true;
852 list += sep ? l + 1 : l;
853 }
854
855 return false;
856 }
857
858 static bool check_against_clause(long numeric, const char *str,
859 const struct tagcache_search_clause *clause)
860 {
861 if (clause->numeric)
862 {
863 switch (clause->type)
864 {
865 case clause_is:
866 return numeric == clause->numeric_data;
867 case clause_is_not:
868 return numeric != clause->numeric_data;
869 case clause_gt:
870 return numeric > clause->numeric_data;
871 case clause_gteq:
872 return numeric >= clause->numeric_data;
873 case clause_lt:
874 return numeric < clause->numeric_data;
875 case clause_lteq:
876 return numeric <= clause->numeric_data;
877 default:
878 logf("Incorrect numeric tag: %d", clause->type);
879 }
880 }
881 else
882 {
883 switch (clause->type)
884 {
885 case clause_is:
886 return !strcasecmp(clause->str, str);
887 case clause_is_not:
888 return strcasecmp(clause->str, str);
889 case clause_gt:
890 return 0>strcasecmp(clause->str, str);
891 case clause_gteq:
892 return 0>=strcasecmp(clause->str, str);
893 case clause_lt:
894 return 0<strcasecmp(clause->str, str);
895 case clause_lteq:
896 return 0<=strcasecmp(clause->str, str);
897 case clause_contains:
898 return (strcasestr(str, clause->str) != NULL);
899 case clause_not_contains:
900 return (strcasestr(str, clause->str) == NULL);
901 case clause_begins_with:
902 return (strcasestr(str, clause->str) == str);
903 case clause_not_begins_with:
904 return (strcasestr(str, clause->str) != str);
905 case clause_ends_with:
906 return str_ends_with(str, clause->str);
907 case clause_not_ends_with:
908 return !str_ends_with(str, clause->str);
909 case clause_oneof:
910 return str_oneof(str, clause->str);
911
912 default:
913 logf("Incorrect tag: %d", clause->type);
914 }
915 }
916
917 return false;
918 }
919
920 static bool check_clauses(struct tagcache_search *tcs,
921 struct index_entry *idx,
922 struct tagcache_search_clause **clause, int count)
923 {
924 int i;
925
926 #ifdef HAVE_TC_RAMCACHE
927 if (tcs->ramsearch)
928 {
929 /* Go through all conditional clauses. */
930 for (i = 0; i < count; i++)
931 {
932 struct tagfile_entry *tfe;
933 int seek;
934 char buf[256];
935 char *str = NULL;
936
937 seek = check_virtual_tags(clause[i]->tag, idx);
938
939 if (!tagcache_is_numeric_tag(clause[i]->tag))
940 {
941 if (clause[i]->tag == tag_filename)
942 {
943 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
944 str = buf;
945 }
946 else
947 {
948 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
949 str = tfe->tag_data;
950 }
951 }
952
953 if (!check_against_clause(seek, str, clause[i]))
954 return false;
955 }
956 }
957 else
958 #endif
959 {
960 /* Check for conditions. */
961 for (i = 0; i < count; i++)
962 {
963 struct tagfile_entry tfe;
964 int seek;
965 char str[256];
966
967 seek = check_virtual_tags(clause[i]->tag, idx);
968
969 memset(str, 0, sizeof str);
970 if (!tagcache_is_numeric_tag(clause[i]->tag))
971 {
972 int fd = tcs->idxfd[clause[i]->tag];
973 lseek(fd, seek, SEEK_SET);
974 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
975 if (tfe.tag_length >= (int)sizeof(str))
976 {
977 logf("Too long tag read!");
978 break ;
979 }
980
981 read(fd, str, tfe.tag_length);
982
983 /* Check if entry has been deleted. */
984 if (str[0] == '\0')
985 break;
986 }
987
988 if (!check_against_clause(seek, str, clause[i]))
989 return false;
990 }
991 }
992
993 return true;
994 }
995
996 bool tagcache_check_clauses(struct tagcache_search *tcs,
997 struct tagcache_search_clause **clause, int count)
998 {
999 struct index_entry idx;
1000
1001 if (count == 0)
1002 return true;
1003
1004 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1005 return false;
1006
1007 return check_clauses(tcs, &idx, clause, count);
1008 }
1009
1010 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1011 {
1012 int i;
1013
1014 /* If uniq buffer is not defined we must return true for search to work. */
1015 if (tcs->unique_list == NULL
1016 || (!tagcache_is_unique_tag(tcs->type)
1017 && !tagcache_is_numeric_tag(tcs->type)))
1018 {
1019 return true;
1020 }
1021
1022 for (i = 0; i < tcs->unique_list_count; i++)
1023 {
1024 /* Return false if entry is found. */
1025 if (tcs->unique_list[i] == id)
1026 return false;
1027 }
1028
1029 if (tcs->unique_list_count < tcs->unique_list_capacity)
1030 {
1031 tcs->unique_list[i] = id;
1032 tcs->unique_list_count++;
1033 }
1034
1035 return true;
1036 }
1037
1038 static bool build_lookup_list(struct tagcache_search *tcs)
1039 {
1040 struct index_entry entry;
1041 int i;
1042
1043 tcs->seek_list_count = 0;
1044
1045 #ifdef HAVE_TC_RAMCACHE
1046 if (tcs->ramsearch)
1047 {
1048 int j;
1049
1050 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1051 {
1052 struct index_entry *idx = &hdr->indices[i];
1053 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1054 break ;
1055
1056 /* Skip deleted files. */
1057 if (idx->flag & FLAG_DELETED)
1058 continue;
1059
1060 /* Go through all filters.. */
1061 for (j = 0; j < tcs->filter_count; j++)
1062 {
1063 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1064 {
1065 break ;
1066 }
1067 }
1068
1069 if (j < tcs->filter_count)
1070 continue ;
1071
1072 /* Check for conditions. */
1073 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1074 continue;
1075
1076 /* Add to the seek list if not already in uniq buffer. */
1077 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1078 continue;
1079
1080 /* Lets add it. */
1081 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1082 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1083 tcs->seek_list_count++;
1084 }
1085
1086 tcs->seek_pos = i;
1087
1088 return tcs->seek_list_count > 0;
1089 }
1090 #endif
1091
1092 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1093 sizeof(struct master_header), SEEK_SET);
1094
1095 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1096 == sizeof(struct index_entry))
1097 {
1098 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1099 break ;
1100
1101 tcs->seek_pos++;
1102
1103 /* Check if entry has been deleted. */
1104 if (entry.flag & FLAG_DELETED)
1105 continue;
1106
1107 /* Go through all filters.. */
1108 for (i = 0; i < tcs->filter_count; i++)
1109 {
1110 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1111 break ;
1112 }
1113
1114 if (i < tcs->filter_count)
1115 continue ;
1116
1117 /* Check for conditions. */
1118 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1119 continue;
1120
1121 /* Add to the seek list if not already in uniq buffer. */
1122 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1123 continue;
1124
1125 /* Lets add it. */
1126 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1127 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1128 tcs->seek_list_count++;
1129
1130 yield();
1131 }
1132
1133 return tcs->seek_list_count > 0;
1134 }
1135
1136
1137 static void remove_files(void)
1138 {
1139 int i;
1140 char buf[MAX_PATH];
1141
1142 tc_stat.ready = false;
1143 tc_stat.ramcache = false;
1144 tc_stat.econ = false;
1145 remove(TAGCACHE_FILE_MASTER);
1146 for (i = 0; i < TAG_COUNT; i++)
1147 {
1148 if (tagcache_is_numeric_tag(i))
1149 continue;
1150
1151 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1152 remove(buf);
1153 }
1154 }
1155
1156
1157 static bool check_all_headers(void)
1158 {
1159 struct master_header myhdr;
1160 struct tagcache_header tch;
1161 int tag;
1162 int fd;
1163
1164 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1165 return false;
1166
1167 close(fd);
1168 if (myhdr.dirty)
1169 {
1170 logf("tagcache is dirty!");
1171 return false;
1172 }
1173
1174 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1175
1176 for (tag = 0; tag < TAG_COUNT; tag++)
1177 {
1178 if (tagcache_is_numeric_tag(tag))
1179 continue;
1180
1181 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1182 return false;
1183
1184 close(fd);
1185 }
1186
1187 return true;
1188 }
1189
1190 bool tagcache_search(struct tagcache_search *tcs, int tag)
1191 {
1192 struct tagcache_header tag_hdr;
1193 struct master_header master_hdr;
1194 int i;
1195
1196 if (tcs->initialized)
1197 tagcache_search_finish(tcs);
1198
1199 while (read_lock)
1200 sleep(1);
1201
1202 memset(tcs, 0, sizeof(struct tagcache_search));
1203 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1204 return false;
1205
1206 tcs->position = sizeof(struct tagcache_header);
1207 tcs->type = tag;
1208 tcs->seek_pos = 0;
1209 tcs->seek_list_count = 0;
1210 tcs->filter_count = 0;
1211 tcs->masterfd = -1;
1212
1213 for (i = 0; i < TAG_COUNT; i++)
1214 tcs->idxfd[i] = -1;
1215
1216 #ifndef HAVE_TC_RAMCACHE
1217 tcs->ramsearch = false;
1218 #else
1219 tcs->ramsearch = tc_stat.ramcache;
1220 if (tcs->ramsearch)
1221 {
1222 tcs->entry_count = hdr->entry_count[tcs->type];
1223 }
1224 else
1225 #endif
1226 {
1227 if (!tagcache_is_numeric_tag(tcs->type))
1228 {
1229 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1230 if (tcs->idxfd[tcs->type] < 0)
1231 return false;
1232 }
1233
1234 /* Always open as R/W so we can pass tcs to functions that modify data also
1235 * without failing. */
1236 tcs->masterfd = open_master_fd(&master_hdr, true);
1237
1238 if (tcs->masterfd < 0)
1239 return false;
1240 }
1241
1242 tcs->valid = true;
1243 tcs->initialized = true;
1244 write_lock++;
1245
1246 return true;
1247 }
1248
1249 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1250 void *buffer, long length)
1251 {
1252 tcs->unique_list = (unsigned long *)buffer;
1253 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1254 tcs->unique_list_count = 0;
1255 }
1256
1257 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1258 int tag, int seek)
1259 {
1260 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1261 return false;
1262
1263 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1264 return false;
1265
1266 tcs->filter_tag[tcs->filter_count] = tag;
1267 tcs->filter_seek[tcs->filter_count] = seek;
1268 tcs->filter_count++;
1269
1270 return true;
1271 }
1272
1273 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1274 struct tagcache_search_clause *clause)
1275 {
1276 int i;
1277
1278 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1279 {
1280 logf("Too many clauses");
1281 return false;
1282 }
1283
1284 /* Check if there is already a similar filter in present (filters are
1285 * much faster than clauses).
1286 */
1287 for (i = 0; i < tcs->filter_count; i++)
1288 {
1289 if (tcs->filter_tag[i] == clause->tag)
1290 return true;
1291 }
1292
1293 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1294 {
1295 char buf[MAX_PATH];
1296
1297 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1298 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1299 }
1300
1301 tcs->clause[tcs->clause_count] = clause;
1302 tcs->clause_count++;
1303
1304 return true;
1305 }
1306
1307 /* TODO: Remove this mess. */
1308 #ifdef HAVE_DIRCACHE
1309 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1310 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1311 #else
1312 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1313 #endif
1314
1315 static bool get_next(struct tagcache_search *tcs)
1316 {
1317 static char buf[TAG_MAXLEN+32];
1318 struct tagfile_entry entry;
1319 long flag = 0;
1320
1321 if (!tcs->valid || !tc_stat.ready)
1322 return false;
1323
1324 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1325 #ifdef HAVE_TC_RAMCACHE
1326 && !tcs->ramsearch
1327 #endif
1328 )
1329 return false;
1330
1331 /* Relative fetch. */
1332 if (tcs->filter_count > 0 || tcs->clause_count > 0
1333 || tagcache_is_numeric_tag(tcs->type))
1334 {
1335 /* Check for end of list. */
1336 if (tcs->seek_list_count == 0)
1337 {
1338 /* Try to fetch more. */
1339 if (!build_lookup_list(tcs))
1340 {
1341 tcs->valid = false;
1342 return false;
1343 }
1344 }
1345
1346 tcs->seek_list_count--;
1347 flag = tcs->seek_flags[tcs->seek_list_count];
1348
1349 /* Seek stream to the correct position and continue to direct fetch. */
1350 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1351 && !tagcache_is_numeric_tag(tcs->type))
1352 {
1353 if (!open_files(tcs, tcs->type))
1354 return false;
1355
1356 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1357 }
1358 else
1359 tcs->position = tcs->seek_list[tcs->seek_list_count];
1360 }
1361
1362 if (tagcache_is_numeric_tag(tcs->type))
1363 {
1364 snprintf(buf, sizeof(buf), "%d", tcs->position);
1365 tcs->result_seek = tcs->position;
1366 tcs->result = buf;
1367 tcs->result_len = strlen(buf) + 1;
1368 return true;
1369 }
1370
1371 /* Direct fetch. */
1372 #ifdef HAVE_TC_RAMCACHE
1373 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1374 {
1375 struct tagfile_entry *ep;
1376
1377 if (tcs->entry_count == 0)
1378 {
1379 tcs->valid = false;
1380 return false;
1381 }
1382 tcs->entry_count--;
1383
1384 tcs->result_seek = tcs->position;
1385
1386 # ifdef HAVE_DIRCACHE
1387 if (tcs->type == tag_filename)
1388 {
1389 dircache_copy_path((struct dirent *)tcs->position,
1390 buf, sizeof buf);
1391 tcs->result = buf;
1392 tcs->result_len = strlen(buf) + 1;
1393 tcs->idx_id = FLAG_GET_ATTR(flag);
1394 tcs->ramresult = false;
1395
1396 return true;
1397 }
1398 # endif
1399
1400 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1401 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1402 tcs->result = ep->tag_data;
1403 tcs->result_len = strlen(tcs->result) + 1;
1404 tcs->idx_id = ep->idx_id;
1405 tcs->ramresult = true;
1406
1407 return true;
1408 }
1409 else
1410 #endif
1411 {
1412 if (!open_files(tcs, tcs->type))
1413 return false;
1414
1415 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1416 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1417 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1418 {
1419 /* End of data. */
1420 tcs->valid = false;
1421 return false;
1422 }
1423 }
1424
1425 if (entry.tag_length > (long)sizeof(buf))
1426 {
1427 tcs->valid = false;
1428 logf("too long tag #2");
1429 return false;
1430 }
1431
1432 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1433 {
1434 tcs->valid = false;
1435 logf("read error #4");
1436 return false;
1437 }
1438
1439 tcs->result = buf;
1440 tcs->result_len = strlen(tcs->result) + 1;
1441 tcs->idx_id = entry.idx_id;
1442 tcs->ramresult = false;
1443
1444 return true;
1445 }
1446
1447 bool tagcache_get_next(struct tagcache_search *tcs)
1448 {
1449 while (get_next(tcs))
1450 {
1451 if (tcs->result_len > 1)
1452 return true;
1453 }
1454
1455 return false;
1456 }
1457
1458 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1459 int tag, char *buf, long size)
1460 {
1461 struct index_entry idx;
1462
1463 *buf = '\0';
1464 if (!get_index(tcs->masterfd, idxid, &idx, true))
1465 return false;
1466
1467 return retrieve(tcs, &idx, tag, buf, size);
1468 }
1469
1470 static bool update_master_header(void)
1471 {
1472 struct master_header myhdr;
1473 int fd;
1474
1475 if (!tc_stat.ready)
1476 return false;
1477
1478 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1479 return false;
1480
1481 myhdr.serial = current_tcmh.serial;
1482 myhdr.commitid = current_tcmh.commitid;
1483
1484 /* Write it back */
1485 lseek(fd, 0, SEEK_SET);
1486 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1487 close(fd);
1488
1489 #ifdef HAVE_TC_RAMCACHE
1490 if (hdr)
1491 {
1492 hdr->h.serial = current_tcmh.serial;
1493 hdr->h.commitid = current_tcmh.commitid;
1494 }
1495 #endif
1496
1497 return true;
1498 }
1499
1500 #if 0
1501
1502 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1503 {
1504 struct tagentry *entry;
1505
1506 if (tcs->type != tag_title)
1507 return ;
1508
1509 /* We will need reserve buffer for this. */
1510 if (tcs->ramcache)
1511 {
1512 struct tagfile_entry *ep;
1513
1514 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1515 tcs->seek_list[tcs->seek_list_count];
1516 }
1517
1518 entry = find_entry_ram();
1519
1520 }
1521 #endif
1522
1523 void tagcache_search_finish(struct tagcache_search *tcs)
1524 {
1525 int i;
1526
1527 if (!tcs->initialized)
1528 return;
1529
1530 if (tcs->masterfd >= 0)
1531 {
1532 close(tcs->masterfd);
1533 tcs->masterfd = -1;
1534 }
1535
1536 for (i = 0; i < TAG_COUNT; i++)
1537 {
1538 if (tcs->idxfd[i] >= 0)
1539 {
1540 close(tcs->idxfd[i]);
1541 tcs->idxfd[i] = -1;
1542 }
1543 }
1544
1545 tcs->ramsearch = false;
1546 tcs->valid = false;
1547 tcs->initialized = 0;
1548 if (write_lock > 0)
1549 write_lock--;
1550 }
1551
1552 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1553 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1554 {
1555 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1556 }
1557
1558 static long get_tag_numeric(const struct index_entry *entry, int tag)
1559 {
1560 return check_virtual_tags(tag, entry);
1561 }
1562
1563 static char* get_tag_string(const struct index_entry *entry, int tag)
1564 {
1565 char* s = get_tag(entry, tag)->tag_data;
1566 return strcmp(s, UNTAGGED) ? s : NULL;
1567 }
1568
1569 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1570 {
1571 struct index_entry *entry;
1572 int idx_id;
1573
1574 if (!tc_stat.ready)
1575 return false;
1576
1577 /* Find the corresponding entry in tagcache. */
1578 idx_id = find_entry_ram(filename, NULL);
1579 if (idx_id < 0 || !tc_stat.ramcache)
1580 return false;
1581
1582 entry = &hdr->indices[idx_id];
1583
1584 id3->title = get_tag_string(entry, tag_title);
1585 id3->artist = get_tag_string(entry, tag_artist);
1586 id3->album = get_tag_string(entry, tag_album);
1587 id3->genre_string = get_tag_string(entry, tag_genre);
1588 id3->composer = get_tag_string(entry, tag_composer);
1589 id3->comment = get_tag_string(entry, tag_comment);
1590 id3->albumartist = get_tag_string(entry, tag_albumartist);
1591 id3->grouping = get_tag_string(entry, tag_grouping);
1592
1593 id3->playcount = get_tag_numeric(entry, tag_playcount);
1594 id3->rating = get_tag_numeric(entry, tag_rating);
1595 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1596 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1597 id3->year = get_tag_numeric(entry, tag_year);
1598
1599 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1600 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1601 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1602 if (id3->bitrate == 0)
1603 id3->bitrate = 1;
1604
1605 return true;
1606 }
1607 #endif
1608
1609 static inline void write_item(const char *item)
1610 {
1611 int len = strlen(item) + 1;
1612
1613 data_size += len;
1614 write(cachefd, item, len);
1615 }
1616
1617 static int check_if_empty(char **tag)
1618 {
1619 int length;
1620
1621 if (*tag == NULL || **tag == '\0')
1622 {
1623 *tag = UNTAGGED;
1624 return sizeof(UNTAGGED); /* Tag length */
1625 }
1626
1627 length = strlen(*tag);
1628 if (length > TAG_MAXLEN)
1629 {
1630 logf("over length tag: %s", *tag);
1631 length = TAG_MAXLEN;
1632 (*tag)[length] = '\0';
1633 }
1634
1635 return length + 1;
1636 }
1637
1638 #define ADD_TAG(entry,tag,data) \
1639 /* Adding tag */ \
1640 entry.tag_offset[tag] = offset; \
1641 entry.tag_length[tag] = check_if_empty(data); \
1642 offset += entry.tag_length[tag]
1643
1644 static void add_tagcache(char *path, unsigned long mtime
1645 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1646 ,const struct dirent *dc
1647 #endif
1648 )
1649 {
1650 struct mp3entry id3;
1651 struct temp_file_entry entry;
1652 bool ret;
1653 int fd;
1654 int idx_id = -1;
1655 char tracknumfix[3];
1656 int offset = 0;
1657 int path_length = strlen(path);
1658 bool has_albumartist;
1659 bool has_grouping;
1660
1661 if (cachefd < 0)
1662 return ;
1663
1664 /* Check for overlength file path. */
1665 if (path_length > TAG_MAXLEN)
1666 {
1667 /* Path can't be shortened. */
1668 logf("Too long path: %s", path);
1669 return ;
1670 }
1671
1672 /* Check if the file is supported. */
1673 if (probe_file_format(path) == AFMT_UNKNOWN)
1674 return ;
1675
1676 /* Check if the file is already cached. */
1677 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1678 if (tc_stat.ramcache && is_dircache_intact())
1679 {
1680 idx_id = find_entry_ram(path, dc);
1681 }
1682 else
1683 #endif
1684 {
1685 if (filenametag_fd >= 0)
1686 {
1687 idx_id = find_entry_disk(path);
1688 }
1689 }
1690
1691 /* Check if file has been modified. */
1692 if (idx_id >= 0)
1693 {
1694 struct index_entry idx;
1695
1696 if (!get_index(-1, idx_id, &idx, true))
1697 {
1698 logf("failed to retrieve index entry");
1699 return ;
1700 }
1701
1702 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1703 {
1704 /* No changes to file. */
1705 return ;
1706 }
1707
1708 /* Metadata might have been changed. Delete the entry. */
1709 logf("Re-adding: %s", path);
1710 if (!delete_entry(idx_id))
1711 {
1712 logf("delete_entry failed: %d", idx_id);
1713 return ;
1714 }
1715 }
1716
1717 fd = open(path, O_RDONLY);
1718 if (fd < 0)
1719 {
1720 logf("open fail: %s", path);
1721 return ;
1722 }
1723
1724 memset(&id3, 0, sizeof(struct mp3entry));
1725 memset(&entry, 0, sizeof(struct temp_file_entry));
1726 memset(&tracknumfix, 0, sizeof(tracknumfix));
1727 ret = get_metadata(&id3, fd, path);
1728 close(fd);
1729
1730 if (!ret)
1731 return ;
1732
1733 logf("-> %s", path);
1734
1735 /* Generate track number if missing. */
1736 if (id3.tracknum <= 0)
1737 {
1738 const char *p = strrchr(path, '.');
1739
1740 if (p == NULL)
1741 p = &path[strlen(path)-1];
1742
1743 while (*p != '/')
1744 {
1745 if (isdigit(*p) && isdigit(*(p-1)))
1746 {
1747 tracknumfix[1] = *p--;
1748 tracknumfix[0] = *p;
1749 break;
1750 }
1751 p--;
1752 }
1753
1754 if (tracknumfix[0] != '\0')
1755 {
1756 id3.tracknum = atoi(tracknumfix);
1757 /* Set a flag to indicate track number has been generated. */
1758 entry.flag |= FLAG_TRKNUMGEN;
1759 }
1760 else
1761 {
1762 /* Unable to generate track number. */
1763 id3.tracknum = -1;
1764 }
1765 }
1766
1767 /* Numeric tags */
1768 entry.tag_offset[tag_year] = id3.year;
1769 entry.tag_offset[tag_discnumber] = id3.discnum;
1770 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1771 entry.tag_offset[tag_length] = id3.length;
1772 entry.tag_offset[tag_bitrate] = id3.bitrate;
1773 entry.tag_offset[tag_mtime] = mtime;
1774
1775 /* String tags. */
1776 has_albumartist = id3.albumartist != NULL
1777 && strlen(id3.albumartist) > 0;
1778 has_grouping = id3.grouping != NULL
1779 && strlen(id3.grouping) > 0;
1780
1781 ADD_TAG(entry, tag_filename, &path);
1782 ADD_TAG(entry, tag_title, &id3.title);
1783 ADD_TAG(entry, tag_artist, &id3.artist);
1784 ADD_TAG(entry, tag_album, &id3.album);
1785 ADD_TAG(entry, tag_genre, &id3.genre_string);
1786 ADD_TAG(entry, tag_composer, &id3.composer);
1787 ADD_TAG(entry, tag_comment, &id3.comment);
1788 if (has_albumartist)
1789 {
1790 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1791 }
1792 else
1793 {
1794 ADD_TAG(entry, tag_albumartist, &id3.artist);
1795 }
1796 if (has_grouping)
1797 {
1798 ADD_TAG(entry, tag_grouping, &id3.grouping);
1799 }
1800 else
1801 {
1802 ADD_TAG(entry, tag_grouping, &id3.title);
1803 }
1804 entry.data_length = offset;
1805
1806 /* Write the header */
1807 write(cachefd, &entry, sizeof(struct temp_file_entry));
1808
1809 /* And tags also... Correct order is critical */
1810 write_item(path);
1811 write_item(id3.title);
1812 write_item(id3.artist);
1813 write_item(id3.album);
1814 write_item(id3.genre_string);
1815 write_item(id3.composer);
1816 write_item(id3.comment);
1817 if (has_albumartist)
1818 {
1819 write_item(id3.albumartist);
1820 }
1821 else
1822 {
1823 write_item(id3.artist);
1824 }
1825 if (has_grouping)
1826 {
1827 write_item(id3.grouping);
1828 }
1829 else
1830 {
1831 write_item(id3.title);
1832 }
1833 total_entry_count++;
1834 }
1835
1836 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1837 {
1838 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1839 int len = strlen(str)+1;
1840 int i;
1841 unsigned crc32;
1842 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1843 char buf[TAG_MAXLEN+32];
1844
1845 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1846 buf[i] = tolower(str[i]);
1847 buf[i] = '\0';
1848
1849 crc32 = crc_32(buf, i, 0xffffffff);
1850
1851 if (unique)
1852 {
1853 /* Check if the crc does not exist -> entry does not exist for sure. */
1854 for (i = 0; i < tempbufidx; i++)
1855 {
1856 if (crcbuf[-i] != crc32)
1857 continue;
1858
1859 if (!strcasecmp(str, index[i].str))
1860 {
1861 if (id < 0 || id >= lookup_buffer_depth)
1862 {
1863 logf("lookup buf overf.: %d", id);
1864 return false;
1865 }
1866
1867 lookup[id] = &index[i];
1868 return true;
1869 }
1870 }
1871 }
1872
1873 /* Insert to CRC buffer. */
1874 crcbuf[-tempbufidx] = crc32;
1875 tempbuf_left -= 4;
1876
1877 /* Insert it to the buffer. */
1878 tempbuf_left -= len;
1879 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1880 return false;
1881
1882 if (id >= lookup_buffer_depth)
1883 {
1884 logf("lookup buf overf. #2: %d", id);
1885 return false;
1886 }
1887
1888 if (id >= 0)
1889 {
1890 lookup[id] = &index[tempbufidx];
1891 index[tempbufidx].idlist.id = id;
1892 }
1893 else
1894 index[tempbufidx].idlist.id = -1;
1895
1896 index[tempbufidx].idlist.next = NULL;
1897 index[tempbufidx].idx_id = idx_id;
1898 index[tempbufidx].seek = -1;
1899 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1900 memcpy(index[tempbufidx].str, str, len);
1901 tempbuf_pos += len;
1902 tempbufidx++;
1903
1904 return true;
1905 }
1906
1907 static int compare(const void *p1, const void *p2)
1908 {
1909 do_timed_yield();
1910
1911 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1912 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1913
1914 if (strcmp(e1->str, UNTAGGED) == 0)
1915 {
1916 if (strcmp(e2->str, UNTAGGED) == 0)
1917 return 0;
1918 return -1;
1919 }
1920 else if (strcmp(e2->str, UNTAGGED) == 0)
1921 return 1;
1922
1923 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1924 }
1925
1926 static int tempbuf_sort(int fd)
1927 {
1928 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1929 struct tagfile_entry fe;
1930 int i;
1931 int length;
1932
1933 /* Generate reverse lookup entries. */
1934 for (i = 0; i < lookup_buffer_depth; i++)
1935 {
1936 struct tempbuf_id_list *idlist;
1937
1938 if (!lookup[i])
1939 continue;
1940
1941 if (lookup[i]->idlist.id == i)
1942 continue;
1943
1944 idlist = &lookup[i]->idlist;
1945 while (idlist->next != NULL)
1946 idlist = idlist->next;
1947
1948 tempbuf_left -= sizeof(struct tempbuf_id_list);
1949 if (tempbuf_left - 4 < 0)
1950 return -1;
1951
1952 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1953 if (tempbuf_pos & 0x03)
1954 {
1955 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1956 tempbuf_left -= 3;
1957 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1958 }
1959 tempbuf_pos += sizeof(struct tempbuf_id_list);
1960
1961 idlist = idlist->next;
1962 idlist->id = i;
1963 idlist->next = NULL;
1964
1965 do_timed_yield();
1966 }
1967
1968 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1969 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1970
1971 for (i = 0; i < tempbufidx; i++)
1972 {
1973 struct tempbuf_id_list *idlist = &index[i].idlist;
1974
1975 /* Fix the lookup list. */
1976 while (idlist != NULL)
1977 {
1978 if (idlist->id >= 0)
1979 lookup[idlist->id] = &index[i];
1980 idlist = idlist->next;
1981 }
1982
1983 index[i].seek = lseek(fd, 0, SEEK_CUR);
1984 length = strlen(index[i].str) + 1;
1985 fe.tag_length = length;
1986 fe.idx_id = index[i].idx_id;
1987
1988 /* Check the chunk alignment. */
1989 if ((fe.tag_length + sizeof(struct tagfile_entry))
1990 % TAGFILE_ENTRY_CHUNK_LENGTH)
1991 {
1992 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1993 ((fe.tag_length + sizeof(struct tagfile_entry))
1994 % TAGFILE_ENTRY_CHUNK_LENGTH);
1995 }
1996
1997 #ifdef TAGCACHE_STRICT_ALIGN
1998 /* Make sure the entry is long aligned. */
1999 if (index[i].seek & 0x03)
2000 {
2001 logf("tempbuf_sort: alignment error!");
2002 return -3;
2003 }
2004 #endif
2005
2006 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2007 sizeof(struct tagfile_entry))
2008 {
2009 logf("tempbuf_sort: write error #1");
2010 return -1;
2011 }
2012
2013 if (write(fd, index[i].str, length) != length)
2014 {
2015 logf("tempbuf_sort: write error #2");
2016 return -2;
2017 }
2018
2019 /* Write some padding. */
2020 if (fe.tag_length - length > 0)
2021 write(fd, "XXXXXXXX", fe.tag_length - length);
2022 }
2023
2024 return i;
2025 }
2026
2027 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2028 {
2029 if (id < 0 || id >= lookup_buffer_depth)
2030 return NULL;
2031
2032 return lookup[id];
2033 }
2034
2035
2036 inline static int tempbuf_find_location(int id)
2037 {
2038 struct tempbuf_searchidx *entry;
2039
2040 entry = tempbuf_locate(id);
2041 if (entry == NULL)
2042 return -1;
2043
2044 return entry->seek;
2045 }
2046
2047 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2048 {
2049 struct master_header tcmh;
2050 struct index_entry idx;
2051 int masterfd;
2052 int masterfd_pos;
2053 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2054 int max_entries;
2055 int entries_processed = 0;
2056 int i, j;
2057 char buf[TAG_MAXLEN];
2058
2059 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2060
2061 logf("Building numeric indices...");
2062 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2063
2064 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2065 return false;
2066
2067 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2068 SEEK_CUR);
2069 if (masterfd_pos == filesize(masterfd))
2070 {
2071 logf("we can't append!");
2072 close(masterfd);
2073 return false;
2074 }
2075
2076 while (entries_processed < h->entry_count)
2077 {
2078 int count = MIN(h->entry_count - entries_processed, max_entries);
2079
2080 /* Read in as many entries as possible. */
2081 for (i = 0; i < count; i++)
2082 {
2083 struct temp_file_entry *tfe = &entrybuf[i];
2084 int datastart;
2085
2086 /* Read in numeric data. */
2087 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2088 sizeof(struct temp_file_entry))
2089 {
2090 logf("read fail #1");
2091 close(masterfd);
2092 return false;
2093 }
2094
2095 datastart = lseek(tmpfd, 0, SEEK_CUR);
2096
2097 /**
2098 * Read string data from the following tags:
2099 * - tag_filename
2100 * - tag_artist
2101 * - tag_album
2102 * - tag_title
2103 *
2104 * A crc32 hash is calculated from the read data
2105 * and stored back to the data offset field kept in memory.
2106 */
2107 #define tmpdb_read_string_tag(tag) \
2108 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2109 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2110 { \
2111 logf("read fail: buffer overflow"); \
2112 close(masterfd); \
2113 return false; \
2114 } \
2115 \
2116 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2117 tfe->tag_length[tag]) \
2118 { \
2119 logf("read fail #2"); \
2120 close(masterfd); \
2121 return false; \
2122 } \
2123 \
2124 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2125 lseek(tmpfd, datastart, SEEK_SET)
2126
2127 tmpdb_read_string_tag(tag_filename);
2128 tmpdb_read_string_tag(tag_artist);
2129 tmpdb_read_string_tag(tag_album);
2130 tmpdb_read_string_tag(tag_title);
2131
2132 /* Seek to the end of the string data. */
2133 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2134 }
2135
2136 /* Backup the master index position. */
2137 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2138 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2139
2140 /* Check if we can resurrect some deleted runtime statistics data. */
2141 for (i = 0; i < tcmh.tch.entry_count; i++)
2142 {
2143 /* Read the index entry. */
2144 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2145 != sizeof(struct index_entry))
2146 {
2147 logf("read fail #3");
2148 close(masterfd);
2149 return false;
2150 }
2151
2152 /**
2153 * Skip unless the entry is marked as being deleted
2154 * or the data has already been resurrected.
2155 */
2156 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2157 continue;
2158
2159 /* Now try to match the entry. */
2160 /**
2161 * To succesfully match a song, the following conditions
2162 * must apply:
2163 *
2164 * For numeric fields: tag_length
2165 * - Full identical match is required
2166 *
2167 * If tag_filename matches, no further checking necessary.
2168 *
2169 * For string hashes: tag_artist, tag_album, tag_title
2170 * - Two of these must match
2171 */
2172 for (j = 0; j < count; j++)
2173 {
2174 struct temp_file_entry *tfe = &entrybuf[j];
2175
2176 /* Try to match numeric fields first. */
2177 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2178 continue;
2179
2180 /* Now it's time to do the hash matching. */
2181 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2182 {
2183 int match_count = 0;
2184
2185 /* No filename match, check if we can match two other tags. */
2186 #define tmpdb_match(tag) \
2187 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2188 match_count++
2189
2190 tmpdb_match(tag_artist);
2191 tmpdb_match(tag_album);
2192 tmpdb_match(tag_title);
2193
2194 if (match_count < 2)
2195 {
2196 /* Still no match found, give up. */
2197 continue;
2198 }
2199 }
2200
2201 /* A match found, now copy & resurrect the statistical data. */
2202 #define tmpdb_copy_tag(tag) \
2203 tfe->tag_offset[tag] = idx.tag_seek[tag]
2204
2205 tmpdb_copy_tag(tag_playcount);
2206 tmpdb_copy_tag(tag_rating);
2207 tmpdb_copy_tag(tag_playtime);
2208 tmpdb_copy_tag(tag_lastplayed);
2209 tmpdb_copy_tag(tag_commitid);
2210
2211 /* Avoid processing this entry again. */
2212 idx.flag |= FLAG_RESURRECTED;
2213
2214 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2215 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2216 != sizeof(struct index_entry))
2217 {
2218 logf("masterfd writeback fail #1");
2219 close(masterfd);
2220 return false;
2221 }
2222
2223 logf("Entry resurrected");
2224 }
2225 }
2226
2227
2228 /* Restore the master index position. */
2229 lseek(masterfd, masterfd_pos, SEEK_SET);
2230
2231 /* Commit the data to the index. */
2232 for (i = 0; i < count; i++)
2233 {
2234 int loc = lseek(masterfd, 0, SEEK_CUR);
2235
2236 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2237 != sizeof(struct index_entry))
2238 {
2239 logf("read fail #3");
2240 close(masterfd);
2241 return false;
2242 }
2243
2244 for (j = 0; j < TAG_COUNT; j++)
2245 {
2246 if (!tagcache_is_numeric_tag(j))
2247 continue;
2248
2249 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2250 }
2251 idx.flag = entrybuf[i].flag;
2252
2253 if (idx.tag_seek[tag_commitid])
2254 {
2255 /* Data has been resurrected. */
2256 idx.flag |= FLAG_DIRTYNUM;
2257 }
2258 else if (tc_stat.ready && current_tcmh.commitid > 0)
2259 {
2260 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2261 idx.flag |= FLAG_DIRTYNUM;
2262 }
2263
2264 /* Write back the updated index. */
2265 lseek(masterfd, loc, SEEK_SET);
2266 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2267 != sizeof(struct index_entry))
2268 {
2269 logf("write fail");
2270 close(masterfd);
2271 return false;
2272 }
2273 }
2274
2275 entries_processed += count;
2276 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2277 }
2278
2279 close(masterfd);
2280
2281 return true;
2282 }
2283
2284 /**
2285 * Return values:
2286 * > 0 success
2287 * == 0 temporary failure
2288 * < 0 fatal error
2289 */
2290 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2291 {
2292 int i;
2293 struct tagcache_header tch;
2294 struct master_header tcmh;
2295 struct index_entry idxbuf[IDX_BUF_DEPTH];
2296 int idxbuf_pos;
2297 char buf[TAG_MAXLEN+32];
2298 int fd = -1, masterfd;
2299 bool error = false;
2300 int init;
2301 int masterfd_pos;
2302
2303 logf("Building index: %d", index_type);
2304
2305 /* Check the number of entries we need to allocate ram for. */
2306 commit_entry_count = h->entry_count + 1;
2307
2308 masterfd = open_master_fd(&tcmh, false);
2309 if (masterfd >= 0)
2310 {
2311 commit_entry_count += tcmh.tch.entry_count;
2312 close(masterfd);
2313 }
2314 else
2315 remove_files(); /* Just to be sure we are clean. */
2316
2317 /* Open the index file, which contains the tag names. */
2318 fd = open_tag_fd(&tch, index_type, true);
2319 if (fd >= 0)
2320 {
2321 logf("tch.datasize=%ld", tch.datasize);
2322 lookup_buffer_depth = 1 +
2323 /* First part */ commit_entry_count +
2324 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2325 }
2326 else
2327 {
2328 lookup_buffer_depth = 1 +
2329 /* First part */ commit_entry_count +
2330 /* Second part */ 0;
2331 }
2332
2333 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2334 logf("commit_entry_count=%ld", commit_entry_count);
2335
2336 /* Allocate buffer for all index entries from both old and new
2337 * tag files. */
2338 tempbufidx = 0;
2339 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2340
2341 /* Allocate lookup buffer. The first portion of commit_entry_count
2342 * contains the new tags in the temporary file and the second
2343 * part for locating entries already in the db.
2344 *
2345 * New tags Old tags
2346 * +---------+---------------------------+
2347 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2348 * +---------+---------------------------+
2349 *
2350 * Old tags are inserted to a temporary buffer with position:
2351 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2352 * And new tags with index:
2353 * tempbuf_insert(idx, ...);
2354 *
2355 * The buffer is sorted and written into tag file:
2356 * tempbuf_sort(...);
2357 * leaving master index locations messed up.
2358 *
2359 * That is fixed using the lookup buffer for old tags:
2360 * new_seek = tempbuf_find_location(old_seek, ...);
2361 * and for new tags:
2362 * new_seek = tempbuf_find_location(idx);
2363 */
2364 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2365 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2366 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2367
2368 /* And calculate the remaining data space used mainly for storing
2369 * tag data (strings). */
2370 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2371 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2372 {
2373 logf("Buffer way too small!");
2374 return 0;
2375 }
2376
2377 if (fd >= 0)
2378 {
2379 /**
2380 * If tag file contains unique tags (sorted index), we will load
2381 * it entirely into memory so we can resort it later for use with
2382 * chunked browsing.
2383 */
2384 if (tagcache_is_sorted_tag(index_type))
2385 {
2386 logf("loading tags...");
2387 for (i = 0; i < tch.entry_count; i++)
2388 {
2389 struct tagfile_entry entry;
2390 int loc = lseek(fd, 0, SEEK_CUR);
2391 bool ret;
2392
2393 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2394 != sizeof(struct tagfile_entry))
2395 {
2396 logf("read error #7");
2397 close(fd);
2398 return -2;
2399 }
2400
2401 if (entry.tag_length >= (int)sizeof(buf))
2402 {
2403 logf("too long tag #3");
2404 close(fd);
2405 return -2;
2406 }
2407
2408 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2409 {
2410 logf("read error #8");
2411 close(fd);
2412 return -2;
2413 }
2414
2415 /* Skip deleted entries. */
2416 if (buf[0] == '\0')
2417 continue;
2418
2419 /**
2420 * Save the tag and tag id in the memory buffer. Tag id
2421 * is saved so we can later reindex the master lookup
2422 * table when the index gets resorted.
2423 */
2424 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2425 + commit_entry_count, entry.idx_id,
2426 tagcache_is_unique_tag(index_type));
2427 if (!ret)
2428 {
2429 close(fd);
2430 return -3;
2431 }
2432 do_timed_yield();
2433 }
2434 logf("done");
2435 }
2436 else
2437 tempbufidx = tch.entry_count;
2438 }
2439 else
2440 {
2441 /**
2442 * Creating new index file to store the tags. No need to preload
2443 * anything whether the index type is sorted or not.
2444 */
2445 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2446 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2447 if (fd < 0)
2448 {
2449 logf("%s open fail", buf);
2450 return -2;
2451 }
2452
2453 tch.magic = TAGCACHE_MAGIC;
2454 tch.entry_count = 0;
2455 tch.datasize = 0;
2456
2457 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2458 != sizeof(struct tagcache_header))
2459 {
2460 logf("header write failed");
2461 close(fd);
2462 return -2;
2463 }
2464 }
2465
2466 /* Loading the tag lookup file as "master file". */
2467 logf("Loading index file");
2468 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2469
2470 if (masterfd < 0)
2471 {
2472 logf("Creating new DB");
2473 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2474
2475 if (masterfd < 0)
2476 {
2477 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2478 close(fd);
2479 return -2;
2480 }
2481
2482 /* Write the header (write real values later). */
2483 memset(&tcmh, 0, sizeof(struct master_header));
2484 tcmh.tch = *h;
2485 tcmh.tch.entry_count = 0;
2486 tcmh.tch.datasize = 0;
2487 tcmh.dirty = true;
2488 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2489 init = true;
2490 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2491 }
2492 else
2493 {
2494 /**
2495 * Master file already exists so we need to process the current
2496 * file first.
2497 */
2498 init = false;
2499
2500 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2501 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2502 {
2503 logf("header error");
2504 close(fd);
2505 close(masterfd);
2506 return -2;
2507 }
2508
2509 /**
2510 * If we reach end of the master file, we need to expand it to
2511 * hold new tags. If the current index is not sorted, we can
2512 * simply append new data to end of the file.
2513 * However, if the index is sorted, we need to update all tag
2514 * pointers in the master file for the current index.
2515 */
2516 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2517 SEEK_CUR);
2518 if (masterfd_pos == filesize(masterfd))
2519 {
2520 logf("appending...");
2521 init = true;
2522 }
2523 }
2524
2525 /**
2526 * Load new unique tags in memory to be sorted later and added
2527 * to the master lookup file.
2528 */
2529 if (tagcache_is_sorted_tag(index_type))
2530 {
2531 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2532 /* h is the header of the temporary file containing new tags. */
2533 logf("inserting new tags...");
2534 for (i = 0; i < h->entry_count; i++)
2535 {
2536 struct temp_file_entry entry;
2537
2538 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2539 sizeof(struct temp_file_entry))
2540 {
2541 logf("read fail #3");
2542 error = true;
2543 goto error_exit;
2544 }
2545
2546 /* Read data. */
2547 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2548 {
2549 logf("too long entry!");
2550 error = true;
2551 goto error_exit;
2552 }
2553
2554 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2555 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2556 entry.tag_length[index_type])
2557 {
2558 logf("read fail #4");
2559 error = true;
2560 goto error_exit;
2561 }
2562
2563 if (tagcache_is_unique_tag(index_type))
2564 error = !tempbuf_insert(buf, i, -1, true);
2565 else
2566 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2567
2568 if (error)
2569 {
2570 logf("insert error");
2571 goto error_exit;
2572 }
2573
2574 /* Skip to next. */
2575 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2576 entry.tag_length[index_type], SEEK_CUR);
2577 do_timed_yield();
2578 }
2579 logf("done");
2580
2581 /* Sort the buffer data and write it to the index file. */
2582 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2583 i = tempbuf_sort(fd);
2584 if (i < 0)
2585 goto error_exit;
2586 logf("sorted %d tags", i);
2587
2588 /**
2589 * Now update all indexes in the master lookup file.
2590 */
2591 logf("updating indices...");
2592 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2593 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2594 {
2595 int j;
2596 int loc = lseek(masterfd, 0, SEEK_CUR);
2597
2598 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2599
2600 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2601 != (int)sizeof(struct index_entry)*idxbuf_pos)
2602 {
2603 logf("read fail #5");
2604 error = true;
2605 goto error_exit ;
2606 }
2607 lseek(masterfd, loc, SEEK_SET);
2608
2609 for (j = 0; j < idxbuf_pos; j++)
2610 {
2611 if (idxbuf[j].flag & FLAG_DELETED)
2612 {
2613 /* We can just ignore deleted entries. */
2614 // idxbuf[j].tag_seek[index_type] = 0;
2615 continue;
2616 }
2617
2618 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2619 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2620 + commit_entry_count);
2621
2622 if (idxbuf[j].tag_seek[index_type] < 0)
2623 {
2624 logf("update error: %d/%d/%ld",
2625 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2626 error = true;
2627 goto error_exit;
2628 }
2629
2630 do_timed_yield();
2631 }
2632
2633 /* Write back the updated index. */
2634 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2635 index_entry_ec, tc_stat.econ) !=
2636 (int)sizeof(struct index_entry)*idxbuf_pos)
2637 {
2638 logf("write fail");
2639 error = true;
2640 goto error_exit;
2641 }
2642 }
2643 logf("done");
2644 }
2645
2646 /**
2647 * Walk through the temporary file containing the new tags.
2648 */
2649 // build_normal_index(h, tmpfd, masterfd, idx);
2650 logf("updating new indices...");
2651 lseek(masterfd, masterfd_pos, SEEK_SET);
2652 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2653 lseek(fd, 0, SEEK_END);
2654 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2655 {
2656 int j;
2657
2658 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2659 if (init)
2660 {
2661 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2662 }
2663 else
2664 {
2665 int loc = lseek(masterfd, 0, SEEK_CUR);
2666
2667 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2668 != (int)sizeof(struct index_entry)*idxbuf_pos)
2669 {
2670 logf("read fail #6");
2671 error = true;
2672 break ;
2673 }
2674 lseek(masterfd, loc, SEEK_SET);
2675 }
2676
2677 /* Read entry headers. */
2678 for (j = 0; j < idxbuf_pos; j++)
2679 {
2680 if (!tagcache_is_sorted_tag(index_type))
2681 {
2682 struct temp_file_entry entry;
2683 struct tagfile_entry fe;
2684
2685 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2686 sizeof(struct temp_file_entry))
2687 {
2688 logf("read fail #7");
2689 error = true;
2690 break ;
2691 }
2692
2693 /* Read data. */
2694 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2695 {
2696 logf("too long entry!");
2697 logf("length=%d", entry.tag_length[index_type]);
2698 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2699 error = true;
2700 break ;
2701 }
2702
2703 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2704 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2705 entry.tag_length[index_type])
2706 {
2707 logf("read fail #8");
2708 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2709 logf("length=0x%02x", entry.tag_length[index_type]);
2710 error = true;
2711 break ;
2712 }
2713
2714 /* Write to index file. */
2715 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2716 fe.tag_length = entry.tag_length[index_type];
2717 fe.idx_id = tcmh.tch.entry_count + i + j;
2718 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2719 write(fd, buf, fe.tag_length);
2720 tempbufidx++;
2721
2722 /* Skip to next. */
2723 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2724 entry.tag_length[index_type], SEEK_CUR);
2725 }
2726 else
2727 {
2728 /* Locate the correct entry from the sorted array. */
2729 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2730 if (idxbuf[j].tag_seek[index_type] < 0)
2731 {
2732 logf("entry not found (%d)", j);
2733 error = true;
2734 break ;
2735 }
2736 }
2737 }
2738
2739 /* Write index. */
2740 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2741 index_entry_ec, tc_stat.econ) !=
2742 (int)sizeof(struct index_entry)*idxbuf_pos)
2743 {
2744 logf("tagcache: write fail #4");
2745 error = true;
2746 break ;
2747 }
2748
2749 do_timed_yield();
2750 }
2751 logf("done");
2752
2753 /* Finally write the header. */
2754 tch.magic = TAGCACHE_MAGIC;
2755 tch.entry_count = tempbufidx;
2756 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2757 lseek(fd, 0, SEEK_SET);
2758 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2759
2760 if (index_type != tag_filename)
2761 h->datasize += tch.datasize;
2762 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2763 error_exit:
2764
2765 close(fd);
2766 close(masterfd);
2767
2768 if (error)
2769 return -2;
2770
2771 return 1;
2772 }
2773
2774 static bool commit(void)
2775 {
2776 struct tagcache_header tch;
2777 struct master_header tcmh;
2778 int i, len, rc;
2779 int tmpfd;
2780 int masterfd;
2781 #ifdef HAVE_DIRCACHE
2782 bool dircache_buffer_stolen = false;
2783 #endif
2784 bool local_allocation = false;
2785
2786 logf("committing tagcache");
2787
2788 while (write_lock)
2789 sleep(1);
2790
2791 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2792 if (tmpfd < 0)
2793 {
2794 logf("nothing to commit");
2795 return true;
2796 }
2797
2798
2799 /* Load the header. */
2800 len = sizeof(struct tagcache_header);
2801 rc = read(tmpfd, &tch, len);
2802
2803 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2804 {
2805 logf("incorrect header");
2806 close(tmpfd);
2807 remove(TAGCACHE_FILE_TEMP);
2808 return false;
2809 }
2810
2811 if (tch.entry_count == 0)
2812 {
2813 logf("nothing to commit");
2814 close(tmpfd);
2815 remove(TAGCACHE_FILE_TEMP);
2816 return true;
2817 }
2818
2819 #ifdef HAVE_EEPROM_SETTINGS
2820 remove(TAGCACHE_STATEFILE);
2821 #endif
2822
2823 /* At first be sure to unload the ramcache! */
2824 #ifdef HAVE_TC_RAMCACHE
2825 tc_stat.ramcache = false;
2826 #endif
2827
2828 read_lock++;
2829
2830 /* Try to steal every buffer we can :) */
2831 if (tempbuf_size == 0)
2832 local_allocation = true;
2833
2834 #ifdef HAVE_DIRCACHE
2835 if (tempbuf_size == 0)
2836 {
2837 /* Try to steal the dircache buffer. */
2838 tempbuf = dircache_steal_buffer(&tempbuf_size);
2839 tempbuf_size &= ~0x03;
2840
2841 if (tempbuf_size > 0)
2842 {
2843 dircache_buffer_stolen = true;
2844 }
2845 }
2846 #endif
2847
2848 #ifdef HAVE_TC_RAMCACHE
2849 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2850 {
2851 tempbuf = (char *)(hdr + 1);
2852 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2853 tempbuf_size &= ~0x03;
2854 }
2855 #endif
2856
2857 /* And finally fail if there are no buffers available. */
2858 if (tempbuf_size == 0)
2859 {
2860 logf("delaying commit until next boot");
2861 tc_stat.commit_delayed = true;
2862 close(tmpfd);
2863 read_lock--;
2864 return false;
2865 }
2866
2867 logf("commit %ld entries...", tch.entry_count);
2868
2869 /* Mark DB dirty so it will stay disabled if commit fails. */
2870 current_tcmh.dirty = true;
2871 update_master_header();
2872
2873 /* Now create the index files. */
2874 tc_stat.commit_step = 0;
2875 tch.datasize = 0;
2876 tc_stat.commit_delayed = false;
2877
2878 for (i = 0; i < TAG_COUNT; i++)
2879 {
2880 int ret;
2881
2882 if (tagcache_is_numeric_tag(i))
2883 continue;
2884
2885 tc_stat.commit_step++;
2886 ret = build_index(i, &tch, tmpfd);
2887 if (ret <= 0)
2888 {
2889 close(tmpfd);
2890 logf("tagcache failed init");
2891 if (ret < 0)
2892 remove_files();
2893 else
2894 tc_stat.commit_delayed = true;
2895 tc_stat.commit_step = 0;
2896 read_lock--;
2897 return false;
2898 }
2899 }
2900
2901 if (!build_numeric_indices(&tch, tmpfd))
2902 {
2903 logf("Failure to commit numeric indices");
2904 close(tmpfd);
2905 remove_files();
2906 tc_stat.commit_step = 0;
2907 read_lock--;
2908 return false;
2909 }
2910
2911 close(tmpfd);
2912 tc_stat.commit_step = 0;
2913
2914 /* Update the master index headers. */
2915 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2916 {
2917 read_lock--;
2918 return false;
2919 }
2920
2921 tcmh.tch.entry_count += tch.entry_count;
2922 tcmh.tch.datasize = sizeof(struct master_header)
2923 + sizeof(struct index_entry) * tcmh.tch.entry_count
2924 + tch.datasize;
2925 tcmh.dirty = false;
2926 tcmh.commitid++;
2927
2928 lseek(masterfd, 0, SEEK_SET);
2929 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2930 close(masterfd);
2931
2932 logf("tagcache committed");
2933 remove(TAGCACHE_FILE_TEMP);
2934 tc_stat.ready = check_all_headers();
2935 tc_stat.readyvalid = true;
2936
2937 if (local_allocation)
2938 {
2939 tempbuf = NULL;
2940 tempbuf_size = 0;
2941 }
2942
2943 #ifdef HAVE_DIRCACHE
2944 /* Rebuild the dircache, if we stole the buffer. */
2945 if (dircache_buffer_stolen)
2946 dircache_build(0);
2947 #endif
2948
2949 #ifdef HAVE_TC_RAMCACHE
2950 /* Reload tagcache. */
2951 if (tc_stat.ramcache_allocated > 0)
2952 tagcache_start_scan();
2953 #endif
2954
2955 read_lock--;
2956
2957 return true;
2958 }
2959
2960 static void allocate_tempbuf(void)
2961 {
2962 /* Yeah, malloc would be really nice now :) */
2963 #ifdef __PCTOOL__
2964 tempbuf_size = 32*1024*1024;
2965 tempbuf = malloc(tempbuf_size);
2966 #else
2967 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2968 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2969 audiobuf += tempbuf_size;
2970 #endif
2971 }
2972
2973 static void free_tempbuf(void)
2974 {
2975 if (tempbuf_size == 0)
2976 return ;
2977
2978 #ifdef __PCTOOL__
2979 free(tempbuf);
2980 #else
2981 audiobuf -= tempbuf_size;
2982 #endif
2983 tempbuf = NULL;
2984 tempbuf_size = 0;
2985 }
2986
2987 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2988 {
2989 struct index_entry idx;
2990
2991 if (!tc_stat.ready)
2992 return false;
2993
2994 if (!tagcache_is_numeric_tag(tag))
2995 return false;
2996
2997 if (!get_index(masterfd, idx_id, &idx, false))
2998 return false;
2999
3000 idx.tag_seek[tag] = data;
3001 idx.flag |= FLAG_DIRTYNUM;
3002
3003 return write_index(masterfd, idx_id, &idx);
3004 }
3005
3006 #if 0
3007 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3008 int tag, long data)
3009 {
3010 struct master_header myhdr;
3011
3012 if (tcs->masterfd < 0)
3013 {
3014 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3015 return false;
3016 }
3017
3018 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3019 }
3020 #endif
3021
3022 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3023
3024 static bool command_queue_is_full(void)
3025 {
3026 int next;
3027
3028 next = command_queue_widx + 1;
3029 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3030 next = 0;
3031
3032 return (next == command_queue_ridx);
3033 }
3034 static bool command_queue_sync_callback(void)
3035 {
3036
3037 struct master_header myhdr;
3038 int masterfd;
3039
3040 mutex_lock(&command_queue_mutex);
3041
3042 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3043 return false;
3044
3045 while (command_queue_ridx != command_queue_widx)
3046 {
3047 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3048
3049 switch (ce->command)
3050 {
3051 case CMD_UPDATE_MASTER_HEADER:
3052 {
3053 close(masterfd);
3054 update_master_header();
3055
3056 /* Re-open the masterfd. */
3057 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3058 return true;
3059
3060 break;
3061 }
3062 case CMD_UPDATE_NUMERIC:
3063 {
3064 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3065 break;
3066 }
3067 }
3068
3069 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3070 command_queue_ridx = 0;
3071 }
3072
3073 close(masterfd);
3074
3075 tc_stat.queue_length = 0;
3076 mutex_unlock(&command_queue_mutex);
3077 return true;
3078 }
3079
3080 static void run_command_queue(bool force)
3081 {
3082 if (COMMAND_QUEUE_IS_EMPTY)
3083 return;
3084
3085 if (force || command_queue_is_full())
3086 command_queue_sync_callback();
3087 else
3088 register_ata_idle_func(command_queue_sync_callback);
3089 }
3090
3091 static void queue_command(int cmd, long idx_id, int tag, long data)
3092 {
3093 while (1)
3094 {
3095 int next;
3096
3097 mutex_lock(&command_queue_mutex);
3098 next = command_queue_widx + 1;
3099 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3100 next = 0;
3101
3102 /* Make sure queue is not full. */
3103 if (next != command_queue_ridx)
3104 {
3105 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3106
3107 ce->command = cmd;
3108 ce->idx_id = idx_id;
3109 ce->tag = tag;
3110 ce->data = data;
3111
3112 command_queue_widx = next;
3113
3114 tc_stat.queue_length++;
3115
3116 mutex_unlock(&command_queue_mutex);
3117 break;
3118 }
3119
3120 /* Queue is full, try again later... */
3121 mutex_unlock(&command_queue_mutex);
3122 sleep(1);
3123 }
3124 }
3125
3126 long tagcache_increase_serial(void)
3127 {
3128 long old;
3129
3130 if (!tc_stat.ready)
3131 return -2;
3132
3133 while (read_lock)
3134 sleep(1);
3135
3136 old = current_tcmh.serial++;
3137 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3138
3139 return old;
3140 }
3141
3142 void tagcache_update_numeric(int idx_id, int tag, long data)
3143 {
3144 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3145 }
3146
3147 long tagcache_get_serial(void)
3148 {
3149 return current_tcmh.serial;
3150 }
3151
3152 long tagcache_get_commitid(void)
3153 {
3154 return current_tcmh.commitid;
3155 }
3156
3157 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3158 {
3159 char buf[512];
3160 int i;
3161
3162 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3163 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3164 {
3165 if (*datastr == '\0')
3166 break;
3167
3168 if (*datastr == '"' || *datastr == '\\')
3169 buf[i++] = '\\';
3170
3171 buf[i] = *(datastr++);
3172 }
3173
3174 strcpy(&buf[i], "\" ");
3175
3176 write(fd, buf, i + 2);
3177
3178 return true;
3179 }
3180
3181 static bool read_tag(char *dest, long size,
3182 const char *src, const char *tagstr)
3183 {
3184 int pos;
3185 char current_tag[32];
3186
3187 while (*src != '\0')
3188 {
3189 /* Skip all whitespace */
3190 while (*src == ' ')
3191 src++;
3192
3193 if (*src == '\0')
3194 break;
3195
3196 pos = 0;
3197 /* Read in tag name */
3198 while (*src != '=' && *src != ' ')
3199 {
3200 current_tag[pos] = *src;
3201 src++;
3202 pos++;
3203
3204 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3205 return false;
3206 }
3207 current_tag[pos] = '\0';
3208
3209 /* Read in tag data */
3210
3211 /* Find the start. */
3212 while (*src != '"' && *src != '\0')
3213 src++;
3214
3215 if (*src == '\0' || *(++src) == '\0')
3216 return false;
3217
3218 /* Read the data. */
3219 for (pos = 0; pos < size; pos++)
3220 {
3221 if (*src == '\0')
3222 break;
3223
3224 if (*src == '\\')
3225 {
3226 dest[pos] = *(src+1);
3227 src += 2;
3228 continue;
3229 }
3230
3231 dest[pos] = *src;
3232
3233 if (*src == '"')
3234 {
3235 src++;
3236 break;
3237 }
3238
3239 if (*src == '\0')
3240 break;
3241
3242 src++;
3243 }
3244 dest[pos] = '\0';
3245
3246 if (!strcasecmp(tagstr, current_tag))
3247 return true;
3248 }
3249
3250 return false;
3251 }
3252
3253 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3254 {
3255 struct index_entry idx;
3256 char tag_data[TAG_MAXLEN+32];
3257 int idx_id;
3258 long masterfd = (long)parameters;
3259 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3260 tag_commitid };
3261 int i;
3262 (void)line_n;
3263
3264 if (*buf == '#')
3265 return 0;
3266
3267 logf("%d/%s", line_n, buf);
3268 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3269 {
3270 logf("filename missing");
3271 logf("-> %s", buf);
3272 return 0;
3273 }
3274
3275 idx_id = find_index(tag_data);
3276 if (idx_id < 0)
3277 {
3278 logf("entry not found");
3279 return 0;
3280 }
3281
3282 if (!get_index(masterfd, idx_id, &idx, false))
3283 {
3284 logf("failed to retrieve index entry");
3285 return 0;
3286 }
3287
3288 /* Stop if tag has already been modified. */
3289 if (idx.flag & FLAG_DIRTYNUM)
3290 return 0;
3291
3292 logf("import: %s", tag_data);
3293
3294 idx.flag |= FLAG_DIRTYNUM;
3295 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3296 {
3297 int data;
3298
3299 if (!read_tag(tag_data, sizeof tag_data, buf,
3300 tagcache_tag_to_str(import_tags[i])))
3301 {
3302 continue;
3303 }
3304
3305 data = atoi(tag_data);
3306 if (data < 0)
3307 continue;
3308
3309 idx.tag_seek[import_tags[i]] = data;
3310
3311 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3312 current_tcmh.serial = data;
3313 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3314 current_tcmh.commitid = data + 1;
3315 }
3316
3317 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3318 }
3319
3320 #ifndef __PCTOOL__
3321 bool tagcache_import_changelog(void)
3322 {
3323 struct master_header myhdr;
3324 struct tagcache_header tch;
3325 int clfd;
3326 long masterfd;
3327 char buf[2048];
3328
3329 if (!tc_stat.ready)
3330 return false;
3331
3332 while (read_lock)
3333 sleep(1);
3334
3335 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3336 if (clfd < 0)
3337 {
3338 logf("failure to open changelog");
3339 return false;
3340 }
3341
3342 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3343 {
3344 close(clfd);
3345 return false;
3346 }
3347
3348 write_lock++;
3349
3350 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3351
3352 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3353 parse_changelog_line);
3354
3355 close(clfd);
3356 close(masterfd);
3357
3358 if (filenametag_fd >= 0)
3359 close(filenametag_fd);
3360
3361 write_lock--;
3362
3363 update_master_header();
3364
3365 return true;
3366 }
3367 #endif
3368
3369 bool tagcache_create_changelog(struct tagcache_search *tcs)
3370 {
3371 struct master_header myhdr;
3372 struct index_entry idx;
3373 char buf[TAG_MAXLEN+32];
3374 char temp[32];
3375 int clfd;
3376 int i, j;
3377
3378 if (!tc_stat.ready)
3379 return false;
3380
3381 if (!tagcache_search(tcs, tag_filename))
3382 return false;
3383
3384 /* Initialize the changelog */
3385 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3386 if (clfd < 0)
3387 {
3388 logf("failure to open changelog");
3389 return false;
3390 }
3391
3392 if (tcs->masterfd < 0)
3393 {
3394 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3395 return false;
3396 }
3397 else
3398 {
3399 lseek(tcs->masterfd, 0, SEEK_SET);
3400 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3401 }
3402
3403 write(clfd, "## Changelog version 1\n", 23);
3404
3405 for (i = 0; i < myhdr.tch.entry_count; i++)
3406 {
3407 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3408 != sizeof(struct index_entry))
3409 {
3410 logf("read error #9");
3411 tagcache_search_finish(tcs);
3412 close(clfd);
3413 return false;
3414 }
3415
3416 /* Skip until the entry found has been modified. */
3417 if (! (idx.flag & FLAG_DIRTYNUM) )
3418 continue;
3419
3420 /* Skip deleted entries too. */
3421 if (idx.flag & FLAG_DELETED)
3422 continue;
3423
3424 /* Now retrieve all tags. */
3425 for (j = 0; j < TAG_COUNT; j++)
3426 {
3427 if (tagcache_is_numeric_tag(j))
3428 {
3429 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3430 write_tag(clfd, tagcache_tag_to_str(j), temp);
3431 continue;
3432 }
3433
3434 tcs->type = j;
3435 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3436 write_tag(clfd, tagcache_tag_to_str(j), buf);
3437 }
3438
3439 write(clfd, "\n", 1);
3440 do_timed_yield();
3441 }
3442
3443 close(clfd);
3444
3445 tagcache_search_finish(tcs);
3446
3447 return true;
3448 }
3449
3450 static bool delete_entry(long idx_id)
3451 {
3452 int fd = -1;
3453 int masterfd = -1;
3454 int tag, i;
3455 struct index_entry idx, myidx;
3456 struct master_header myhdr;
3457 char buf[TAG_MAXLEN+32];
3458 int in_use[TAG_COUNT];
3459
3460 logf("delete_entry(): %ld", idx_id);
3461
3462 #ifdef HAVE_TC_RAMCACHE
3463 /* At first mark the entry removed from ram cache. */
3464 if (tc_stat.ramcache)
3465 hdr->indices[idx_id].flag |= FLAG_DELETED;
3466 #endif
3467
3468 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3469 return false;
3470
3471 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3472 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3473 != sizeof(struct index_entry))
3474 {
3475 logf("delete_entry(): read error");
3476 goto cleanup;
3477 }
3478
3479 if (myidx.flag & FLAG_DELETED)
3480 {
3481 logf("delete_entry(): already deleted!");
3482 goto cleanup;
3483 }
3484
3485 myidx.flag |= FLAG_DELETED;
3486 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3487 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3488 != sizeof(struct index_entry))
3489 {
3490 logf("delete_entry(): write_error #1");
3491 goto cleanup;
3492 }
3493
3494 /* Now check which tags are no longer in use (if any) */
3495 for (tag = 0; tag < TAG_COUNT; tag++)
3496 in_use[tag] = 0;
3497
3498 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3499 for (i = 0; i < myhdr.tch.entry_count; i++)
3500 {
3501 struct index_entry *idxp;
3502
3503 #ifdef HAVE_TC_RAMCACHE
3504 /* Use RAM DB if available for greater speed */
3505 if (tc_stat.ramcache)
3506 idxp = &hdr->indices[i];
3507 else
3508 #endif
3509 {
3510 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3511 != sizeof(struct index_entry))
3512 {
3513 logf("delete_entry(): read error #2");
3514 goto cleanup;
3515 }
3516 idxp = &idx;
3517 }
3518
3519 if (idxp->flag & FLAG_DELETED)
3520 continue;
3521
3522 for (tag = 0; tag < TAG_COUNT; tag++)
3523 {
3524 if (tagcache_is_numeric_tag(tag))
3525 continue;
3526
3527 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3528 in_use[tag]++;
3529 }
3530 }
3531
3532 /* Now delete all tags no longer in use. */
3533 for (tag = 0; tag < TAG_COUNT; tag++)
3534 {
3535 struct tagcache_header tch;
3536 int oldseek = myidx.tag_seek[tag];
3537
3538 if (tagcache_is_numeric_tag(tag))
3539 continue;
3540
3541 /**
3542 * Replace tag seek with a hash value of the field string data.
3543 * That way runtime statistics of moved or altered files can be
3544 * resurrected.
3545 */
3546 #ifdef HAVE_TC_RAMCACHE
3547 if (tc_stat.ramcache && tag != tag_filename)
3548 {
3549 struct tagfile_entry *tfe;
3550 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3551
3552 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3553 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3554 myidx.tag_seek[tag] = *seek;
3555 }
3556 else
3557 #endif
3558 {
3559 struct tagfile_entry tfe;
3560
3561 /* Open the index file, which contains the tag names. */
3562 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3563 goto cleanup;
3564
3565 /* Skip the header block */
3566 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3567 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3568 != sizeof(struct tagfile_entry))
3569 {
3570 logf("delete_entry(): read error #3");
3571 goto cleanup;
3572 }
3573
3574 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3575 {
3576 logf("delete_entry(): read error #4");
3577 goto cleanup;
3578 }
3579
3580 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3581 }
3582
3583 if (in_use[tag])
3584 {
3585 logf("in use: %d/%d", tag, in_use[tag]);
3586 if (fd >= 0)
3587 {
3588 close(fd);
3589 fd = -1;
3590 }
3591 continue;
3592 }
3593
3594 #ifdef HAVE_TC_RAMCACHE
3595 /* Delete from ram. */
3596 if (tc_stat.ramcache && tag != tag_filename)
3597 {
3598 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3599 tagentry->tag_data[0] = '\0';
3600 }
3601 #endif
3602
3603 /* Open the index file, which contains the tag names. */
3604 if (fd < 0)
3605 {
3606 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3607 goto cleanup;
3608 }
3609
3610 /* Skip the header block */
3611 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3612
3613 /* Debug, print 10 first characters of the tag
3614 read(fd, buf, 10);
3615 buf[10]='\0';
3616 logf("TAG:%s", buf);
3617 lseek(fd, -10, SEEK_CUR);
3618 */
3619
3620 /* Write first data byte in tag as \0 */
3621 write(fd, "", 1);
3622
3623 /* Now tag data has been removed */
3624 close(fd);
3625 fd = -1;
3626 }
3627
3628 /* Write index entry back into master index. */
3629 lseek(masterfd, sizeof(struct master_header) +
3630 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3631 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3632 != sizeof(struct index_entry))
3633 {
3634 logf("delete_entry(): write_error #2");
3635 goto cleanup;
3636 }
3637
3638 close(masterfd);
3639
3640 return true;
3641
3642 cleanup:
3643 if (fd >= 0)
3644 close(fd);
3645 if (masterfd >= 0)
3646 close(masterfd);
3647
3648 return false;
3649 }
3650
3651 #ifndef __PCTOOL__
3652 /**
3653 * Returns true if there is an event waiting in the queue
3654 * that requires the current operation to be aborted.
3655 */
3656 static bool check_event_queue(void)
3657 {
3658 struct queue_event ev;
3659
3660 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3661 switch (ev.id)
3662 {
3663 case Q_STOP_SCAN:
3664 case SYS_POWEROFF:
3665 case SYS_USB_CONNECTED:
3666 /* Put the event back into the queue. */
3667 queue_post(&tagcache_queue, ev.id, ev.data);
3668 return true;
3669 }
3670
3671 return false;
3672 }
3673 #endif
3674
3675 #ifdef HAVE_TC_RAMCACHE
3676 static bool allocate_tagcache(void)
3677 {
3678 struct master_header tcmh;
3679 int fd;
3680
3681 /* Load the header. */
3682 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3683 {
3684 hdr = NULL;
3685 return false;
3686 }
3687
3688 close(fd);
3689
3690 /**
3691 * Now calculate the required cache size plus
3692 * some extra space for alignment fixes.
3693 */
3694 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3695 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3696 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3697 memset(hdr, 0, sizeof(struct ramcache_header));
3698 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3699 hdr->indices = (struct index_entry *)(hdr + 1);
3700 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3701
3702 return true;
3703 }
3704
3705 # ifdef HAVE_EEPROM_SETTINGS
3706 static bool tagcache_dumpload(void)
3707 {
3708 struct statefile_header shdr;
3709 int fd, rc;
3710 long offpos;
3711 int i;
3712
3713 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3714 if (fd < 0)
3715 {
3716 logf("no tagcache statedump");
3717 return false;
3718 }
3719
3720 /* Check the statefile memory placement */
3721 hdr = buffer_alloc(0);
3722 rc = read(fd, &shdr, sizeof(struct statefile_header));
3723 if (rc != sizeof(struct statefile_header)
3724 /* || (long)hdr != (long)shdr.hdr */)
3725 {
3726 logf("incorrect statefile");
3727 hdr = NULL;
3728 close(fd);
3729 return false;
3730 }
3731
3732 offpos = (long)hdr - (long)shdr.hdr;
3733
3734 /* Lets allocate real memory and load it */
3735 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3736 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3737 close(fd);
3738
3739 if (rc != shdr.tc_stat.ramcache_allocated)
3740 {
3741 logf("read failure!");
3742 hdr = NULL;
3743 return false;
3744 }
3745
3746 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3747
3748 /* Now fix the pointers */
3749 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3750 for (i = 0; i < TAG_COUNT; i++)
3751 hdr->tags[i] += offpos;
3752
3753 return true;
3754 }
3755
3756 static bool tagcache_dumpsave(void)
3757 {
3758 struct statefile_header shdr;
3759 int fd;
3760
3761 if (!tc_stat.ramcache)
3762 return false;
3763
3764 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3765 if (fd < 0)
3766 {
3767 logf("failed to create a statedump");
3768 return false;
3769 }
3770
3771 /* Create the header */
3772 shdr.hdr = hdr;
3773 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3774 write(fd, &shdr, sizeof(struct statefile_header));
3775
3776 /* And dump the data too */
3777 write(fd, hdr, tc_stat.ramcache_allocated);
3778 close(fd);
3779
3780 return true;
3781 }
3782 # endif
3783
3784 static bool load_tagcache(void)
3785 {
3786 struct tagcache_header *tch;
3787 long bytesleft = tc_stat.ramcache_allocated;
3788 struct index_entry *idx;
3789 int rc, fd;
3790 char *p;
3791 int i, tag;
3792
3793 # ifdef HAVE_DIRCACHE
3794 while (dircache_is_initializing())
3795 sleep(1);
3796
3797 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3798 # endif
3799
3800 logf("loading tagcache to ram...");
3801
3802 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3803 if (fd < 0)
3804 {
3805 logf("tagcache open failed");
3806 return false;
3807 }
3808
3809 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3810 != sizeof(struct master_header)
3811 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3812 {
3813 logf("incorrect header");
3814 return false;
3815 }
3816
3817 idx = hdr->indices;
3818
3819 /* Load the master index table. */
3820 for (i = 0; i < hdr->h.tch.entry_count; i++)
3821 {
3822 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3823 if (rc != sizeof(struct index_entry))
3824 {
3825 logf("read error #10");
3826 close(fd);
3827 return false;
3828 }
3829
3830 bytesleft -= sizeof(struct index_entry);
3831 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3832 {
3833 logf("too big tagcache.");
3834 close(fd);
3835 return false;
3836 }
3837
3838 idx++;
3839 }
3840
3841 close(fd);
3842
3843 /* Load the tags. */
3844 p = (char *)idx;
3845 for (tag = 0; tag < TAG_COUNT; tag++)
3846 {
3847 struct tagfile_entry *fe;
3848 char buf[TAG_MAXLEN+32];
3849
3850 if (tagcache_is_numeric_tag(tag))
3851 continue ;
3852
3853 //p = ((void *)p+1);
3854 p = (char *)((long)p & ~0x03) + 0x04;
3855 hdr->tags[tag] = p;
3856
3857 /* Check the header. */
3858 tch = (struct tagcache_header *)p;
3859 p += sizeof(struct tagcache_header);
3860
3861 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3862 return false;
3863
3864 for (hdr->entry_count[tag] = 0;
3865 hdr->entry_count[tag] < tch->entry_count;
3866 hdr->entry_count[tag]++)
3867 {
3868 long pos;
3869
3870 if (do_timed_yield())
3871 {
3872 /* Abort if we got a critical event in queue */
3873 if (check_event_queue())
3874 return false;
3875 }
3876
3877 fe = (struct tagfile_entry *)p;
3878 pos = lseek(fd, 0, SEEK_CUR);
3879 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3880 if (rc != sizeof(struct tagfile_entry))
3881 {
3882 /* End of lookup table. */
3883 logf("read error #11");
3884 close(fd);
3885 return false;
3886 }
3887
3888 /* We have a special handling for the filename tags. */
3889 if (tag == tag_filename)
3890 {
3891 # ifdef HAVE_DIRCACHE
3892 const struct dirent *dc;
3893 # endif
3894
3895 // FIXME: This is wrong!
3896 // idx = &hdr->indices[hdr->entry_count[i]];
3897 idx = &hdr->indices[fe->idx_id];
3898
3899 if (fe->tag_length >= (long)sizeof(buf)-1)
3900 {
3901 read(fd, buf, 10);
3902 buf[10] = '\0';
3903 logf("TAG:%s", buf);
3904 logf("too long filename");
3905 close(fd);
3906 return false;
3907 }
3908
3909 rc = read(fd, buf, fe->tag_length);
3910 if (rc != fe->tag_length)
3911 {
3912 logf("read error #12");
3913 close(fd);
3914 return false;
3915 }
3916
3917 /* Check if the entry has already been removed */
3918 if (idx->flag & FLAG_DELETED)
3919 continue;
3920
3921 /* This flag must not be used yet. */
3922 if (idx->flag & FLAG_DIRCACHE)
3923 {
3924 logf("internal error!");
3925 close(fd);
3926 return false;
3927 }
3928
3929 if (idx->tag_seek[tag] != pos)
3930 {
3931 logf("corrupt data structures!");
3932 close(fd);
3933 return false;
3934 }
3935
3936 # ifdef HAVE_DIRCACHE
3937 if (dircache_is_enabled())
3938 {
3939 dc = dircache_get_entry_ptr(buf);
3940 if (dc == NULL)
3941 {
3942 logf("Entry no longer valid.");
3943 logf("-> %s", buf);
3944 delete_entry(fe->idx_id);
3945 continue ;
3946 }
3947
3948 idx->flag |= FLAG_DIRCACHE;
3949 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3950 idx->tag_seek[tag_filename] = (long)dc;
3951 }
3952 else
3953 # endif
3954 {
3955 /* This will be very slow unless dircache is enabled
3956 or target is flash based, but do it anyway for
3957 consistency. */
3958 /* Check if entry has been removed. */
3959 if (global_settings.tagcache_autoupdate)
3960 {
3961 if (!file_exists(buf))
3962 {
3963 logf("Entry no longer valid.");
3964 logf("-> %s", buf);
3965 delete_entry(fe->idx_id);
3966 continue;
3967 }
3968 }
3969 }
3970
3971 continue ;
3972 }
3973
3974 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3975 if (bytesleft < 0)
3976 {
3977 logf("too big tagcache #2");
3978 logf("tl: %d", fe->tag_length);
3979 logf("bl: %ld", bytesleft);
3980 close(fd);
3981 return false;
3982 }
3983
3984 p = fe->tag_data;
3985 rc = read(fd, fe->tag_data, fe->tag_length);
3986 p += rc;
3987
3988 if (rc != fe->tag_length)
3989 {
3990 logf("read error #13");
3991 logf("rc=0x%04x", rc); // 0x431
3992 logf("len=0x%04x", fe->tag_length); // 0x4000
3993 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3994 logf("tag=0x%02x", tag); // 0x00
3995 close(fd);
3996 return false;
3997 }
3998 }
3999 close(fd);
4000 }
4001
4002 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4003 logf("tagcache loaded into ram!");
4004
4005 return true;
4006 }
4007 #endif /* HAVE_TC_RAMCACHE */
4008
4009 static bool check_deleted_files(void)
4010 {
4011 int fd;
4012 char buf[TAG_MAXLEN+32];
4013 struct tagfile_entry tfe;
4014
4015 logf("reverse scan...");
4016 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4017 fd = open(buf, O_RDONLY);
4018
4019 if (fd < 0)
4020 {
4021 logf("%s open fail", buf);
4022 return false;
4023 }
4024
4025 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4026 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4027 == sizeof(struct tagfile_entry)
4028 #ifndef __PCTOOL__
4029 && !check_event_queue()
4030 #endif
4031 )
4032 {
4033 if (tfe.tag_length >= (long)sizeof(buf)-1)
4034 {
4035 logf("too long tag");
4036 close(fd);
4037 return false;
4038 }
4039
4040 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4041 {
4042 logf("read error #14");
4043 close(fd);
4044 return false;
4045 }
4046
4047 /* Check if the file has already deleted from the db. */
4048 if (*buf == '\0')
4049 continue;
4050
4051 /* Now check if the file exists. */
4052 if (!file_exists(buf))
4053 {
4054 logf("Entry no longer valid.");
4055 logf("-> %s / %d", buf, tfe.tag_length);
4056 delete_entry(tfe.idx_id);
4057 }
4058 }
4059
4060 close(fd);
4061
4062 logf("done");
4063
4064 return true;
4065 }
4066
4067 static bool check_dir(const char *dirname, int add_files)
4068 {
4069 DIR *dir;
4070 int len;
4071 int success = false;
4072 int ignore, unignore;
4073 char newpath[MAX_PATH];
4074
4075 dir = opendir(dirname);
4076 if (!dir)
4077 {
4078 logf("tagcache: opendir() failed");
4079 return false;
4080 }
4081
4082 /* check for a database.ignore file */
4083 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4084 ignore = file_exists(newpath);
4085 /* check for a database.unignore file */
4086 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4087 unignore = file_exists(newpath);
4088
4089 /* don't do anything if both ignore and unignore are there */
4090 if (ignore != unignore)
4091 add_files = unignore;
4092
4093 /* Recursively scan the dir. */
4094 #ifdef __PCTOOL__
4095 while (1)
4096 #else
4097 while (!check_event_queue())
4098 #endif
4099 {
4100 struct dirent *entry;
4101
4102 entry = readdir(dir);
4103
4104 if (entry == NULL)
4105 {
4106 success = true;
4107 break ;
4108 }
4109
4110 if (!strcmp((char *)entry->d_name, ".") ||
4111 !strcmp((char *)entry->d_name, ".."))
4112 continue;
4113
4114 yield();
4115
4116 len = strlen(curpath);
4117 snprintf(&curpath[len], curpath_size - len, "/%s",
4118 entry->d_name);
4119
4120 processed_dir_count++;
4121 if (entry->attribute & ATTR_DIRECTORY)
4122 check_dir(curpath, add_files);
4123 else if (add_files)
4124 {
4125 tc_stat.curentry = curpath;
4126
4127 /* Add a new entry to the temporary db file. */
4128 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4129 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4130 , dir->internal_entry
4131 #endif
4132 );
4133
4134 /* Wait until current path for debug screen is read and unset. */
4135 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4136 yield();
4137
4138 tc_stat.curentry = NULL;
4139 }
4140
4141 curpath[len] = '\0';
4142 }
4143
4144 closedir(dir);
4145
4146 return success;
4147 }
4148
4149 void tagcache_screensync_event(void)
4150 {
4151 tc_stat.curentry = NULL;
4152 }
4153
4154 void tagcache_screensync_enable(bool state)
4155 {
4156 tc_stat.syncscreen = state;
4157 }
4158
4159 void tagcache_build(const char *path)
4160 {
4161 struct tagcache_header header;
4162 bool ret;
4163
4164 curpath[0] = '\0';
4165 data_size = 0;
4166 total_entry_count = 0;
4167 processed_dir_count = 0;
4168
4169 #ifdef HAVE_DIRCACHE
4170 while (dircache_is_initializing())
4171 sleep(1);
4172 #endif
4173
4174 logf("updating tagcache");
4175
4176 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4177 if (cachefd >= 0)
4178 {
4179 logf("skipping, cache already waiting for commit");
4180 close(cachefd);
4181 return ;
4182 }
4183
4184 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4185 if (cachefd < 0)
4186 {
4187 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4188 return ;
4189 }
4190
4191 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4192
4193 cpu_boost(true);
4194
4195 logf("Scanning files...");
4196 /* Scan for new files. */
4197 memset(&header, 0, sizeof(struct tagcache_header));
4198 write(cachefd, &header, sizeof(struct tagcache_header));
4199
4200 if (strcmp("/", path) != 0)
4201 strcpy(curpath, path);
4202 ret = check_dir(path, true);
4203
4204 /* Write the header. */
4205 header.magic = TAGCACHE_MAGIC;
4206 header.datasize = data_size;
4207 header.entry_count = total_entry_count;
4208 lseek(cachefd, 0, SEEK_SET);
4209 write(cachefd, &header, sizeof(struct tagcache_header));
4210 close(cachefd);
4211
4212 if (filenametag_fd >= 0)
4213 {
4214 close(filenametag_fd);
4215 filenametag_fd = -1;
4216 }
4217
4218 if (!ret)
4219 {
4220 logf("Aborted.");
4221 cpu_boost(false);
4222 return ;
4223 }
4224
4225 /* Commit changes to the database. */
4226 #ifdef __PCTOOL__
4227 allocate_tempbuf();
4228 #endif
4229 if (commit())
4230 {
4231 remove(TAGCACHE_FILE_TEMP);
4232 logf("tagcache built!");
4233 }
4234 #ifdef __PCTOOL__
4235 free_tempbuf();
4236 #endif
4237
4238 #ifdef HAVE_TC_RAMCACHE
4239 if (hdr)
4240 {
4241 /* Import runtime statistics if we just initialized the db. */
4242 if (hdr->h.serial == 0)
4243 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4244 }
4245 #endif
4246
4247 cpu_boost(false);
4248 }
4249
4250 #ifdef HAVE_TC_RAMCACHE
4251 static void load_ramcache(void)
4252 {
4253 if (!hdr)
4254 return ;
4255
4256 cpu_boost(true);
4257
4258 /* At first we should load the cache (if exists). */
4259 tc_stat.ramcache = load_tagcache();
4260
4261 if (!tc_stat.ramcache)
4262 {
4263 /* If loading failed, it must indicate some problem with the db
4264 * so disable it entirely to prevent further issues. */
4265 tc_stat.ready = false;
4266 hdr = NULL;
4267 }
4268
4269 cpu_boost(false);
4270 }
4271
4272 void tagcache_unload_ramcache(void)
4273 {
4274 tc_stat.ramcache = false;
4275 /* Just to make sure there is no statefile present. */
4276 // remove(TAGCACHE_STATEFILE);
4277 }
4278 #endif
4279
4280 #ifndef __PCTOOL__
4281 static void tagcache_thread(void)
4282 {
4283 struct queue_event ev;
4284 bool check_done = false;
4285
4286 /* If the previous cache build/update was interrupted, commit
4287 * the changes first in foreground. */
4288 cpu_boost(true);
4289 allocate_tempbuf();
4290 commit();
4291 free_tempbuf();
4292
4293 #ifdef HAVE_TC_RAMCACHE
4294 # ifdef HAVE_EEPROM_SETTINGS
4295 if (firmware_settings.initialized && firmware_settings.disk_clean)
4296 check_done = tagcache_dumpload();
4297
4298 remove(TAGCACHE_STATEFILE);
4299 # endif
4300
4301 /* Allocate space for the tagcache if found on disk. */
4302 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4303 allocate_tagcache();
4304 #endif
4305
4306 cpu_boost(false);
4307 tc_stat.initialized = true;
4308
4309 /* Don't delay bootup with the header check but do it on background. */
4310 sleep(HZ);
4311 tc_stat.ready = check_all_headers();
4312 tc_stat.readyvalid = true;
4313
4314 while (1)
4315 {
4316 run_command_queue(false);
4317
4318 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4319
4320 switch (ev.id)
4321 {
4322 case Q_IMPORT_CHANGELOG:
4323 tagcache_import_changelog();
4324 break;
4325
4326 case Q_REBUILD:
4327 remove_files();
4328 remove(TAGCACHE_FILE_TEMP);
4329 tagcache_build("/");
4330 break;
4331
4332 case Q_UPDATE:
4333 tagcache_build("/");
4334 #ifdef HAVE_TC_RAMCACHE
4335 load_ramcache();
4336 #endif
4337 check_deleted_files();
4338 break ;
4339
4340 case Q_START_SCAN:
4341 check_done = false;
4342 case SYS_TIMEOUT:
4343 if (check_done || !tc_stat.ready)
4344 break ;
4345
4346 #ifdef HAVE_TC_RAMCACHE
4347 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4348 {
4349 load_ramcache();
4350 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4351 tagcache_build("/");
4352 }
4353 else
4354 #endif
4355 if (global_settings.tagcache_autoupdate)
4356 {
4357 tagcache_build("/");
4358
4359 /* This will be very slow unless dircache is enabled
4360 or target is flash based, but do it anyway for
4361 consistency. */
4362 check_deleted_files();
4363 }
4364
4365 logf("tagcache check done");
4366
4367 check_done = true;
4368 break ;
4369
4370 case Q_STOP_SCAN:
4371 break ;
4372
4373 case SYS_POWEROFF:
4374 break ;
4375
4376 #ifndef SIMULATOR
4377 case SYS_USB_CONNECTED:
4378 logf("USB: TagCache");
4379 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4380 usb_wait_for_disconnect(&tagcache_queue);
4381 break ;
4382 #endif
4383 }
4384 }
4385 }
4386
4387 bool tagcache_prepare_shutdown(void)
4388 {
4389 if (tagcache_get_commit_step() > 0)
4390 return false;
4391
4392 tagcache_stop_scan();
4393 while (read_lock || write_lock)
4394 sleep(1);
4395
4396 return true;
4397 }
4398
4399 void tagcache_shutdown(void)
4400 {
4401 /* Flush the command queue. */
4402 run_command_queue(true);
4403
4404 #ifdef HAVE_EEPROM_SETTINGS
4405 if (tc_stat.ramcache)
4406 tagcache_dumpsave();
4407 #endif
4408 }
4409
4410 static int get_progress(void)
4411 {
4412 int total_count = -1;
4413
4414 #ifdef HAVE_DIRCACHE
4415 if (dircache_is_enabled())
4416 {
4417 total_count = dircache_get_entry_count();
4418 }
4419 else
4420 #endif
4421 #ifdef HAVE_TC_RAMCACHE
4422 {
4423 if (hdr && tc_stat.ramcache)
4424 total_count = hdr->h.tch.entry_count;
4425 }
4426 #endif
4427
4428 if (total_count < 0)
4429 return -1;
4430
4431 return processed_dir_count * 100 / total_count;
4432 }
4433
4434 struct tagcache_stat* tagcache_get_stat(void)
4435 {
4436 tc_stat.progress = get_progress();
4437 tc_stat.processed_entries = processed_dir_count;
4438
4439 return &tc_stat;
4440 }
4441
4442 void tagcache_start_scan(void)
4443 {
4444 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4445 }
4446
4447 bool tagcache_update(void)
4448 {
4449 if (!tc_stat.ready)
4450 return false;
4451
4452 queue_post(&tagcache_queue, Q_UPDATE, 0);
4453 return false;
4454 }
4455
4456 bool tagcache_rebuild()
4457 {
4458 queue_post(&tagcache_queue, Q_REBUILD, 0);
4459 return false;
4460 }
4461
4462 void tagcache_stop_scan(void)
4463 {
4464 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4465 }
4466
4467 #ifdef HAVE_TC_RAMCACHE
4468 bool tagcache_is_ramcache(void)
4469 {
4470 return tc_stat.ramcache;
4471 }
4472 #endif
4473
4474 #endif /* !__PCTOOL__ */
4475
4476
4477 void tagcache_init(void)
4478 {
4479 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4480 memset(&current_tcmh, 0, sizeof(struct master_header));
4481 filenametag_fd = -1;
4482 write_lock = read_lock = 0;
4483
4484 #ifndef __PCTOOL__
4485 mutex_init(&command_queue_mutex);
4486 queue_init(&tagcache_queue, true);
4487 create_thread(tagcache_thread, tagcache_stack,
4488 sizeof(tagcache_stack), 0, tagcache_thread_name
4489 IF_PRIO(, PRIORITY_BACKGROUND)
4490 IF_COP(, CPU));
4491 #else
4492 tc_stat.initialized = true;
4493 allocate_tempbuf();
4494 commit();
4495 free_tempbuf();
4496 tc_stat.ready = check_all_headers();
4497 #endif
4498 }
4499
4500 #ifdef __PCTOOL__
4501 void tagcache_reverse_scan(void)
4502 {
4503 logf("Checking for deleted files");
4504 check_deleted_files();
4505 }
4506 #endif
4507
4508 bool tagcache_is_initialized(void)
4509 {
4510 return tc_stat.initialized;
4511 }
4512 bool tagcache_is_usable(void)
4513 {
4514 return tc_stat.initialized && tc_stat.ready;
4515 }
4516 int tagcache_get_commit_step(void)
4517 {
4518 return tc_stat.commit_step;
4519 }
4520 int tagcache_get_max_commit_step(void)
4521 {
4522 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4523 }
4524
My public Rockbox git repo