?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Set the font to FONT_UI after exiting a plugin (FS#10132). This makes the core Rockbo...
[kugel-rb/myfork.git] / apps / tagcache.c
blob84e8875968f046340ebb551a38b53daf12fa04b2
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 /* #define LOGF_ENABLE */
59
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #include "config.h"
64 #include "ata_idle_notify.h"
65 #include "thread.h"
66 #include "kernel.h"
67 #include "system.h"
68 #include "logf.h"
69 #include "string.h"
70 #include "usb.h"
71 #include "metadata.h"
72 #include "metadata.h"
73 #include "tagcache.h"
74 #include "buffer.h"
75 #include "crc32.h"
76 #include "misc.h"
77 #include "settings.h"
78 #include "dir.h"
79 #include "structec.h"
80 #include "tagcache.h"
81
82 #ifndef __PCTOOL__
83 #include "lang.h"
84 #include "eeprom_settings.h"
85 #endif
86
87 #ifdef __PCTOOL__
88 #define yield() do { } while(0)
89 #define sim_sleep(timeout) do { } while(0)
90 #define do_timed_yield() do { } while(0)
91 #endif
92
93 #ifndef __PCTOOL__
94 /* Tag Cache thread. */
95 static struct event_queue tagcache_queue;
96 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
97 static const char tagcache_thread_name[] = "tagcache";
98 #endif
99
100 #define UNTAGGED "<Untagged>"
101
102 /* Previous path when scanning directory tree recursively. */
103 static char curpath[TAG_MAXLEN+32];
104
105 /* Used when removing duplicates. */
106 static char *tempbuf; /* Allocated when needed. */
107 static long tempbufidx; /* Current location in buffer. */
108 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
109 static long tempbuf_left; /* Buffer space left. */
110 static long tempbuf_pos;
111
112 /* Tags we want to get sorted (loaded to the tempbuf). */
113 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
114 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
115
116 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
117 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
118 tag_composer, tag_comment, tag_albumartist, tag_grouping };
119
120 /* Numeric tags (we can use these tags with conditional clauses). */
121 static const int numeric_tags[] = { tag_year, tag_discnumber,
122 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
123 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
124 tag_virt_length_min, tag_virt_length_sec,
125 tag_virt_playtime_min, tag_virt_playtime_sec,
126 tag_virt_entryage, tag_virt_autoscore };
127
128 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
129 static const char *tags_str[] = { "artist", "album", "genre", "title",
130 "filename", "composer", "comment", "albumartist", "grouping", "year",
131 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
132 "playtime", "lastplayed", "commitid", "mtime" };
133
134 /* Status information of the tagcache. */
135 static struct tagcache_stat tc_stat;
136
137 /* Queue commands. */
138 enum tagcache_queue {
139 Q_STOP_SCAN = 0,
140 Q_START_SCAN,
141 Q_IMPORT_CHANGELOG,
142 Q_UPDATE,
143 Q_REBUILD,
144
145 /* Internal tagcache command queue. */
146 CMD_UPDATE_MASTER_HEADER,
147 CMD_UPDATE_NUMERIC,
148 };
149
150 struct tagcache_command_entry {
151 int32_t command;
152 int32_t idx_id;
153 int32_t tag;
154 int32_t data;
155 };
156
157 #ifndef __PCTOOL__
158 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
159 static volatile int command_queue_widx = 0;
160 static volatile int command_queue_ridx = 0;
161 static struct mutex command_queue_mutex;
162 #endif
163
164 /* Tag database structures. */
165
166 /* Variable-length tag entry in tag files. */
167 struct tagfile_entry {
168 int32_t tag_length; /* Length of the data in bytes including '\0' */
169 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
170 char tag_data[0]; /* Begin of the tag data */
171 };
172
173 /* Fixed-size tag entry in master db index. */
174 struct index_entry {
175 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
176 int32_t flag; /* Status flags */
177 };
178
179 /* Header is the same in every file. */
180 struct tagcache_header {
181 int32_t magic; /* Header version number */
182 int32_t datasize; /* Data size in bytes */
183 int32_t entry_count; /* Number of entries in this file */
184 };
185
186 struct master_header {
187 struct tagcache_header tch;
188 int32_t serial; /* Increasing counting number */
189 int32_t commitid; /* Number of commits so far */
190 int32_t dirty;
191 };
192
193 /* For the endianess correction */
194 static const char *tagfile_entry_ec = "ll";
195 /**
196 Note: This should be (1 + TAG_COUNT) amount of l's.
197 */
198 static const char *index_entry_ec = "lllllllllllllllllllll";
199
200 static const char *tagcache_header_ec = "lll";
201 static const char *master_header_ec = "llllll";
202
203 static struct master_header current_tcmh;
204
205 #ifdef HAVE_TC_RAMCACHE
206 /* Header is created when loading database to ram. */
207 struct ramcache_header {
208 struct master_header h; /* Header from the master index */
209 struct index_entry *indices; /* Master index file content */
210 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
211 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
212 };
213
214 # ifdef HAVE_EEPROM_SETTINGS
215 struct statefile_header {
216 struct ramcache_header *hdr;
217 struct tagcache_stat tc_stat;
218 };
219 # endif
220
221 /* Pointer to allocated ramcache_header */
222 static struct ramcache_header *hdr;
223 #endif
224
225 /**
226 * Full tag entries stored in a temporary file waiting
227 * for commit to the cache. */
228 struct temp_file_entry {
229 long tag_offset[TAG_COUNT];
230 short tag_length[TAG_COUNT];
231 long flag;
232
233 long data_length;
234 };
235
236 struct tempbuf_id_list {
237 long id;
238 struct tempbuf_id_list *next;
239 };
240
241 struct tempbuf_searchidx {
242 long idx_id;
243 char *str;
244 int seek;
245 struct tempbuf_id_list idlist;
246 };
247
248 /* Lookup buffer for fixing messed up index while after sorting. */
249 static long commit_entry_count;
250 static long lookup_buffer_depth;
251 static struct tempbuf_searchidx **lookup;
252
253 /* Used when building the temporary file. */
254 static int cachefd = -1, filenametag_fd;
255 static int total_entry_count = 0;
256 static int data_size = 0;
257 static int processed_dir_count;
258
259 /* Thread safe locking */
260 static volatile int write_lock;
261 static volatile int read_lock;
262
263 static bool delete_entry(long idx_id);
264
265 const char* tagcache_tag_to_str(int tag)
266 {
267 return tags_str[tag];
268 }
269
270 bool tagcache_is_numeric_tag(int type)
271 {
272 int i;
273
274 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
275 {
276 if (type == numeric_tags[i])
277 return true;
278 }
279
280 return false;
281 }
282
283 bool tagcache_is_unique_tag(int type)
284 {
285 int i;
286
287 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
288 {
289 if (type == unique_tags[i])
290 return true;
291 }
292
293 return false;
294 }
295
296 bool tagcache_is_sorted_tag(int type)
297 {
298 int i;
299
300 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
301 {
302 if (type == sorted_tags[i])
303 return true;
304 }
305
306 return false;
307 }
308
309 #ifdef HAVE_DIRCACHE
310 /**
311 * Returns true if specified flag is still present, i.e., dircache
312 * has not been reloaded.
313 */
314 static bool is_dircache_intact(void)
315 {
316 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
317 }
318 #endif
319
320 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
321 {
322 int fd;
323 char buf[MAX_PATH];
324 int rc;
325
326 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
327 return -1;
328
329 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
330
331 fd = open(buf, write ? O_RDWR : O_RDONLY);
332 if (fd < 0)
333 {
334 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
335 tc_stat.ready = false;
336 return fd;
337 }
338
339 /* Check the header. */
340 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
341 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
342 {
343 logf("header error");
344 tc_stat.ready = false;
345 close(fd);
346 return -2;
347 }
348
349 return fd;
350 }
351
352 static int open_master_fd(struct master_header *hdr, bool write)
353 {
354 int fd;
355 int rc;
356
357 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
358 if (fd < 0)
359 {
360 logf("master file open failed for R/W");
361 tc_stat.ready = false;
362 return fd;
363 }
364
365 tc_stat.econ = false;
366
367 /* Check the header. */
368 rc = read(fd, hdr, sizeof(struct master_header));
369 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
370 {
371 /* Success. */
372 return fd;
373 }
374
375 /* Trying to read again, this time with endianess correction enabled. */
376 lseek(fd, 0, SEEK_SET);
377
378 rc = ecread(fd, hdr, 1, master_header_ec, true);
379 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
380 {
381 logf("header error");
382 tc_stat.ready = false;
383 close(fd);
384 return -2;
385 }
386
387 tc_stat.econ = true;
388
389 return fd;
390 }
391
392 #ifndef __PCTOOL__
393 static bool do_timed_yield(void)
394 {
395 /* Sorting can lock up for quite a while, so yield occasionally */
396 static long wakeup_tick = 0;
397 if (current_tick >= wakeup_tick)
398 {
399 wakeup_tick = current_tick + (HZ/4);
400 yield();
401 return true;
402 }
403 return false;
404 }
405 #endif
406
407 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
408 static long find_entry_ram(const char *filename,
409 const struct dirent *dc)
410 {
411 static long last_pos = 0;
412 int i;
413
414 /* Check if we tagcache is loaded into ram. */
415 if (!tc_stat.ramcache)
416 return -1;
417
418 if (dc == NULL)
419 dc = dircache_get_entry_ptr(filename);
420
421 if (dc == NULL)
422 {
423 logf("tagcache: file not found.");
424 return -1;
425 }
426
427 try_again:
428
429 if (last_pos > 0)
430 i = last_pos;
431 else
432 i = 0;
433
434 for (; i < hdr->h.tch.entry_count; i++)
435 {
436 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
437 {
438 last_pos = MAX(0, i - 3);
439 return i;
440 }
441
442 do_timed_yield();
443 }
444
445 if (last_pos > 0)
446 {
447 last_pos = 0;
448 goto try_again;
449 }
450
451 return -1;
452 }
453 #endif
454
455 static long find_entry_disk(const char *filename)
456 {
457 struct tagcache_header tch;
458 static long last_pos = -1;
459 long pos_history[POS_HISTORY_COUNT];
460 long pos_history_idx = 0;
461 bool found = false;
462 struct tagfile_entry tfe;
463 int fd;
464 char buf[TAG_MAXLEN+32];
465 int i;
466 int pos = -1;
467
468 if (!tc_stat.ready)
469 return -2;
470
471 fd = filenametag_fd;
472 if (fd < 0)
473 {
474 last_pos = -1;
475 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
476 return -1;
477 }
478
479 check_again:
480
481 if (last_pos > 0)
482 lseek(fd, last_pos, SEEK_SET);
483 else
484 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
485
486 while (true)
487 {
488 pos = lseek(fd, 0, SEEK_CUR);
489 for (i = pos_history_idx-1; i >= 0; i--)
490 pos_history[i+1] = pos_history[i];
491 pos_history[0] = pos;
492
493 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
494 != sizeof(struct tagfile_entry))
495 {
496 break ;
497 }
498
499 if (tfe.tag_length >= (long)sizeof(buf))
500 {
501 logf("too long tag #1");
502 close(fd);
503 last_pos = -1;
504 return -2;
505 }
506
507 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
508 {
509 logf("read error #2");
510 close(fd);
511 last_pos = -1;
512 return -3;
513 }
514
515 if (!strcasecmp(filename, buf))
516 {
517 last_pos = pos_history[pos_history_idx];
518 found = true;
519 break ;
520 }
521
522 if (pos_history_idx < POS_HISTORY_COUNT - 1)
523 pos_history_idx++;
524 }
525
526 /* Not found? */
527 if (!found)
528 {
529 if (last_pos > 0)
530 {
531 last_pos = -1;
532 logf("seek again");
533 goto check_again;
534 }
535
536 if (fd != filenametag_fd)
537 close(fd);
538 return -4;
539 }
540
541 if (fd != filenametag_fd)
542 close(fd);
543
544 return tfe.idx_id;
545 }
546
547 static int find_index(const char *filename)
548 {
549 long idx_id = -1;
550
551 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
552 if (tc_stat.ramcache && is_dircache_intact())
553 idx_id = find_entry_ram(filename, NULL);
554 #endif
555
556 if (idx_id < 0)
557 idx_id = find_entry_disk(filename);
558
559 return idx_id;
560 }
561
562 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
563 {
564 int idx_id;
565
566 if (!tc_stat.ready)
567 return false;
568
569 idx_id = find_index(filename);
570 if (idx_id < 0)
571 return false;
572
573 if (!tagcache_search(tcs, tag_filename))
574 return false;
575
576 tcs->entry_count = 0;
577 tcs->idx_id = idx_id;
578
579 return true;
580 }
581
582 static bool get_index(int masterfd, int idxid,
583 struct index_entry *idx, bool use_ram)
584 {
585 bool localfd = false;
586
587 if (idxid < 0)
588 {
589 logf("Incorrect idxid: %d", idxid);
590 return false;
591 }
592
593 #ifdef HAVE_TC_RAMCACHE
594 if (tc_stat.ramcache && use_ram)
595 {
596 if (hdr->indices[idxid].flag & FLAG_DELETED)
597 return false;
598
599 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
600 return true;
601 }
602 #else
603 (void)use_ram;
604 #endif
605
606 if (masterfd < 0)
607 {
608 struct master_header tcmh;
609
610 localfd = true;
611 masterfd = open_master_fd(&tcmh, false);
612 if (masterfd < 0)
613 return false;
614 }
615
616 lseek(masterfd, idxid * sizeof(struct index_entry)
617 + sizeof(struct master_header), SEEK_SET);
618 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
619 != sizeof(struct index_entry))
620 {
621 logf("read error #3");
622 if (localfd)
623 close(masterfd);
624
625 return false;
626 }
627
628 if (localfd)
629 close(masterfd);
630
631 if (idx->flag & FLAG_DELETED)
632 return false;
633
634 return true;
635 }
636
637 #ifndef __PCTOOL__
638
639 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
640 {
641 /* We need to exclude all memory only flags & tags when writing to disk. */
642 if (idx->flag & FLAG_DIRCACHE)
643 {
644 logf("memory only flags!");
645 return false;
646 }
647
648 #ifdef HAVE_TC_RAMCACHE
649 /* Only update numeric data. Writing the whole index to RAM by memcpy
650 * destroys dircache pointers!
651 */
652 if (tc_stat.ramcache)
653 {
654 int tag;
655 struct index_entry *idx_ram = &hdr->indices[idxid];
656
657 for (tag = 0; tag < TAG_COUNT; tag++)
658 {
659 if (tagcache_is_numeric_tag(tag))
660 {
661 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
662 }
663 }
664
665 /* Don't touch the dircache flag or attributes. */
666 idx_ram->flag = (idx->flag & 0x0000ffff)
667 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
668 }
669 #endif
670
671 lseek(masterfd, idxid * sizeof(struct index_entry)
672 + sizeof(struct master_header), SEEK_SET);
673 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
674 != sizeof(struct index_entry))
675 {
676 logf("write error #3");
677 logf("idxid: %d", idxid);
678 return false;
679 }
680
681 return true;
682 }
683
684 #endif /* !__PCTOOL__ */
685
686 static bool open_files(struct tagcache_search *tcs, int tag)
687 {
688 if (tcs->idxfd[tag] < 0)
689 {
690 char fn[MAX_PATH];
691
692 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
693 tcs->idxfd[tag] = open(fn, O_RDONLY);
694 }
695
696 if (tcs->idxfd[tag] < 0)
697 {
698 logf("File not open!");
699 return false;
700 }
701
702 return true;
703 }
704
705 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
706 int tag, char *buf, long size)
707 {
708 struct tagfile_entry tfe;
709 long seek;
710
711 *buf = '\0';
712
713 if (tagcache_is_numeric_tag(tag))
714 return false;
715
716 seek = idx->tag_seek[tag];
717 if (seek < 0)
718 {
719 logf("Retrieve failed");
720 return false;
721 }
722
723 #ifdef HAVE_TC_RAMCACHE
724 if (tcs->ramsearch)
725 {
726 struct tagfile_entry *ep;
727
728 # ifdef HAVE_DIRCACHE
729 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
730 && is_dircache_intact())
731 {
732 dircache_copy_path((struct dirent *)seek,
733 buf, size);
734 return true;
735 }
736 else
737 # endif
738 if (tag != tag_filename)
739 {
740 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
741 strncpy(buf, ep->tag_data, size-1);
742
743 return true;
744 }
745 }
746 #endif
747
748 if (!open_files(tcs, tag))
749 return false;
750
751 lseek(tcs->idxfd[tag], seek, SEEK_SET);
752 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
753 != sizeof(struct tagfile_entry))
754 {
755 logf("read error #5");
756 return false;
757 }
758
759 if (tfe.tag_length >= size)
760 {
761 logf("too small buffer");
762 return false;
763 }
764
765 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
766 tfe.tag_length)
767 {
768 logf("read error #6");
769 return false;
770 }
771
772 buf[tfe.tag_length] = '\0';
773
774 return true;
775 }
776
777 static long check_virtual_tags(int tag, const struct index_entry *idx)
778 {
779 long data = 0;
780
781 switch (tag)
782 {
783 case tag_virt_length_sec:
784 data = (idx->tag_seek[tag_length]/1000) % 60;
785 break;
786
787 case tag_virt_length_min:
788 data = (idx->tag_seek[tag_length]/1000) / 60;
789 break;
790
791 case tag_virt_playtime_sec:
792 data = (idx->tag_seek[tag_playtime]/1000) % 60;
793 break;
794
795 case tag_virt_playtime_min:
796 data = (idx->tag_seek[tag_playtime]/1000) / 60;
797 break;
798
799 case tag_virt_autoscore:
800 if (idx->tag_seek[tag_length] == 0
801 || idx->tag_seek[tag_playcount] == 0)
802 {
803 data = 0;
804 }
805 else
806 {
807 data = 100 * idx->tag_seek[tag_playtime]
808 / idx->tag_seek[tag_length]
809 / idx->tag_seek[tag_playcount];
810 }
811 break;
812
813 /* How many commits before the file has been added to the DB. */
814 case tag_virt_entryage:
815 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
816 break;
817
818 default:
819 data = idx->tag_seek[tag];
820 }
821
822 return data;
823 }
824
825 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
826 {
827 struct index_entry idx;
828
829 if (!tc_stat.ready)
830 return false;
831
832 if (!tagcache_is_numeric_tag(tag))
833 return -1;
834
835 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
836 return -2;
837
838 return check_virtual_tags(tag, &idx);
839 }
840
841 inline static bool str_ends_with(const char *str1, const char *str2)
842 {
843 int str_len = strlen(str1);
844 int clause_len = strlen(str2);
845
846 if (clause_len > str_len)
847 return false;
848
849 return !strcasecmp(&str1[str_len - clause_len], str2);
850 }
851
852 inline static bool str_oneof(const char *str, const char *list)
853 {
854 const char *sep;
855 int l, len = strlen(str);
856
857 while (*list)
858 {
859 sep = strchr(list, '|');
860 l = sep ? (long)sep - (long)list : (int)strlen(list);
861 if ((l==len) && !strncasecmp(str, list, len))
862 return true;
863 list += sep ? l + 1 : l;
864 }
865
866 return false;
867 }
868
869 static bool check_against_clause(long numeric, const char *str,
870 const struct tagcache_search_clause *clause)
871 {
872 if (clause->numeric)
873 {
874 switch (clause->type)
875 {
876 case clause_is:
877 return numeric == clause->numeric_data;
878 case clause_is_not:
879 return numeric != clause->numeric_data;
880 case clause_gt:
881 return numeric > clause->numeric_data;
882 case clause_gteq:
883 return numeric >= clause->numeric_data;
884 case clause_lt:
885 return numeric < clause->numeric_data;
886 case clause_lteq:
887 return numeric <= clause->numeric_data;
888 default:
889 logf("Incorrect numeric tag: %d", clause->type);
890 }
891 }
892 else
893 {
894 switch (clause->type)
895 {
896 case clause_is:
897 return !strcasecmp(clause->str, str);
898 case clause_is_not:
899 return strcasecmp(clause->str, str);
900 case clause_gt:
901 return 0>strcasecmp(clause->str, str);
902 case clause_gteq:
903 return 0>=strcasecmp(clause->str, str);
904 case clause_lt:
905 return 0<strcasecmp(clause->str, str);
906 case clause_lteq:
907 return 0<=strcasecmp(clause->str, str);
908 case clause_contains:
909 return (strcasestr(str, clause->str) != NULL);
910 case clause_not_contains:
911 return (strcasestr(str, clause->str) == NULL);
912 case clause_begins_with:
913 return (strcasestr(str, clause->str) == str);
914 case clause_not_begins_with:
915 return (strcasestr(str, clause->str) != str);
916 case clause_ends_with:
917 return str_ends_with(str, clause->str);
918 case clause_not_ends_with:
919 return !str_ends_with(str, clause->str);
920 case clause_oneof:
921 return str_oneof(str, clause->str);
922
923 default:
924 logf("Incorrect tag: %d", clause->type);
925 }
926 }
927
928 return false;
929 }
930
931 static bool check_clauses(struct tagcache_search *tcs,
932 struct index_entry *idx,
933 struct tagcache_search_clause **clause, int count)
934 {
935 int i;
936
937 #ifdef HAVE_TC_RAMCACHE
938 if (tcs->ramsearch)
939 {
940 /* Go through all conditional clauses. */
941 for (i = 0; i < count; i++)
942 {
943 struct tagfile_entry *tfe;
944 int seek;
945 char buf[256];
946 char *str = NULL;
947
948 seek = check_virtual_tags(clause[i]->tag, idx);
949
950 if (!tagcache_is_numeric_tag(clause[i]->tag))
951 {
952 if (clause[i]->tag == tag_filename)
953 {
954 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
955 str = buf;
956 }
957 else
958 {
959 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
960 str = tfe->tag_data;
961 }
962 }
963
964 if (!check_against_clause(seek, str, clause[i]))
965 return false;
966 }
967 }
968 else
969 #endif
970 {
971 /* Check for conditions. */
972 for (i = 0; i < count; i++)
973 {
974 struct tagfile_entry tfe;
975 int seek;
976 char str[256];
977
978 seek = check_virtual_tags(clause[i]->tag, idx);
979
980 memset(str, 0, sizeof str);
981 if (!tagcache_is_numeric_tag(clause[i]->tag))
982 {
983 int fd = tcs->idxfd[clause[i]->tag];
984 lseek(fd, seek, SEEK_SET);
985 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
986 if (tfe.tag_length >= (int)sizeof(str))
987 {
988 logf("Too long tag read!");
989 break ;
990 }
991
992 read(fd, str, tfe.tag_length);
993
994 /* Check if entry has been deleted. */
995 if (str[0] == '\0')
996 break;
997 }
998
999 if (!check_against_clause(seek, str, clause[i]))
1000 return false;
1001 }
1002 }
1003
1004 return true;
1005 }
1006
1007 bool tagcache_check_clauses(struct tagcache_search *tcs,
1008 struct tagcache_search_clause **clause, int count)
1009 {
1010 struct index_entry idx;
1011
1012 if (count == 0)
1013 return true;
1014
1015 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1016 return false;
1017
1018 return check_clauses(tcs, &idx, clause, count);
1019 }
1020
1021 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1022 {
1023 int i;
1024
1025 /* If uniq buffer is not defined we must return true for search to work. */
1026 if (tcs->unique_list == NULL
1027 || (!tagcache_is_unique_tag(tcs->type)
1028 && !tagcache_is_numeric_tag(tcs->type)))
1029 {
1030 return true;
1031 }
1032
1033 for (i = 0; i < tcs->unique_list_count; i++)
1034 {
1035 /* Return false if entry is found. */
1036 if (tcs->unique_list[i] == id)
1037 return false;
1038 }
1039
1040 if (tcs->unique_list_count < tcs->unique_list_capacity)
1041 {
1042 tcs->unique_list[i] = id;
1043 tcs->unique_list_count++;
1044 }
1045
1046 return true;
1047 }
1048
1049 static bool build_lookup_list(struct tagcache_search *tcs)
1050 {
1051 struct index_entry entry;
1052 int i;
1053
1054 tcs->seek_list_count = 0;
1055
1056 #ifdef HAVE_TC_RAMCACHE
1057 if (tcs->ramsearch)
1058 {
1059 int j;
1060
1061 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1062 {
1063 struct index_entry *idx = &hdr->indices[i];
1064 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1065 break ;
1066
1067 /* Skip deleted files. */
1068 if (idx->flag & FLAG_DELETED)
1069 continue;
1070
1071 /* Go through all filters.. */
1072 for (j = 0; j < tcs->filter_count; j++)
1073 {
1074 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1075 {
1076 break ;
1077 }
1078 }
1079
1080 if (j < tcs->filter_count)
1081 continue ;
1082
1083 /* Check for conditions. */
1084 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1085 continue;
1086
1087 /* Add to the seek list if not already in uniq buffer. */
1088 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1089 continue;
1090
1091 /* Lets add it. */
1092 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1093 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1094 tcs->seek_list_count++;
1095 }
1096
1097 tcs->seek_pos = i;
1098
1099 return tcs->seek_list_count > 0;
1100 }
1101 #endif
1102
1103 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1104 sizeof(struct master_header), SEEK_SET);
1105
1106 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1107 == sizeof(struct index_entry))
1108 {
1109 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1110 break ;
1111
1112 tcs->seek_pos++;
1113
1114 /* Check if entry has been deleted. */
1115 if (entry.flag & FLAG_DELETED)
1116 continue;
1117
1118 /* Go through all filters.. */
1119 for (i = 0; i < tcs->filter_count; i++)
1120 {
1121 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1122 break ;
1123 }
1124
1125 if (i < tcs->filter_count)
1126 continue ;
1127
1128 /* Check for conditions. */
1129 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1130 continue;
1131
1132 /* Add to the seek list if not already in uniq buffer. */
1133 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1134 continue;
1135
1136 /* Lets add it. */
1137 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1138 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1139 tcs->seek_list_count++;
1140
1141 yield();
1142 }
1143
1144 return tcs->seek_list_count > 0;
1145 }
1146
1147
1148 static void remove_files(void)
1149 {
1150 int i;
1151 char buf[MAX_PATH];
1152
1153 tc_stat.ready = false;
1154 tc_stat.ramcache = false;
1155 tc_stat.econ = false;
1156 remove(TAGCACHE_FILE_MASTER);
1157 for (i = 0; i < TAG_COUNT; i++)
1158 {
1159 if (tagcache_is_numeric_tag(i))
1160 continue;
1161
1162 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1163 remove(buf);
1164 }
1165 }
1166
1167
1168 static bool check_all_headers(void)
1169 {
1170 struct master_header myhdr;
1171 struct tagcache_header tch;
1172 int tag;
1173 int fd;
1174
1175 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1176 return false;
1177
1178 close(fd);
1179 if (myhdr.dirty)
1180 {
1181 logf("tagcache is dirty!");
1182 return false;
1183 }
1184
1185 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1186
1187 for (tag = 0; tag < TAG_COUNT; tag++)
1188 {
1189 if (tagcache_is_numeric_tag(tag))
1190 continue;
1191
1192 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1193 return false;
1194
1195 close(fd);
1196 }
1197
1198 return true;
1199 }
1200
1201 bool tagcache_search(struct tagcache_search *tcs, int tag)
1202 {
1203 struct tagcache_header tag_hdr;
1204 struct master_header master_hdr;
1205 int i;
1206
1207 if (tcs->initialized)
1208 tagcache_search_finish(tcs);
1209
1210 while (read_lock)
1211 sleep(1);
1212
1213 memset(tcs, 0, sizeof(struct tagcache_search));
1214 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1215 return false;
1216
1217 tcs->position = sizeof(struct tagcache_header);
1218 tcs->type = tag;
1219 tcs->seek_pos = 0;
1220 tcs->seek_list_count = 0;
1221 tcs->filter_count = 0;
1222 tcs->masterfd = -1;
1223
1224 for (i = 0; i < TAG_COUNT; i++)
1225 tcs->idxfd[i] = -1;
1226
1227 #ifndef HAVE_TC_RAMCACHE
1228 tcs->ramsearch = false;
1229 #else
1230 tcs->ramsearch = tc_stat.ramcache;
1231 if (tcs->ramsearch)
1232 {
1233 tcs->entry_count = hdr->entry_count[tcs->type];
1234 }
1235 else
1236 #endif
1237 {
1238 if (!tagcache_is_numeric_tag(tcs->type))
1239 {
1240 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1241 if (tcs->idxfd[tcs->type] < 0)
1242 return false;
1243 }
1244
1245 /* Always open as R/W so we can pass tcs to functions that modify data also
1246 * without failing. */
1247 tcs->masterfd = open_master_fd(&master_hdr, true);
1248
1249 if (tcs->masterfd < 0)
1250 return false;
1251 }
1252
1253 tcs->valid = true;
1254 tcs->initialized = true;
1255 write_lock++;
1256
1257 return true;
1258 }
1259
1260 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1261 void *buffer, long length)
1262 {
1263 tcs->unique_list = (unsigned long *)buffer;
1264 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1265 tcs->unique_list_count = 0;
1266 }
1267
1268 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1269 int tag, int seek)
1270 {
1271 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1272 return false;
1273
1274 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1275 return false;
1276
1277 tcs->filter_tag[tcs->filter_count] = tag;
1278 tcs->filter_seek[tcs->filter_count] = seek;
1279 tcs->filter_count++;
1280
1281 return true;
1282 }
1283
1284 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1285 struct tagcache_search_clause *clause)
1286 {
1287 int i;
1288
1289 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1290 {
1291 logf("Too many clauses");
1292 return false;
1293 }
1294
1295 /* Check if there is already a similar filter in present (filters are
1296 * much faster than clauses).
1297 */
1298 for (i = 0; i < tcs->filter_count; i++)
1299 {
1300 if (tcs->filter_tag[i] == clause->tag)
1301 return true;
1302 }
1303
1304 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1305 {
1306 char buf[MAX_PATH];
1307
1308 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1309 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1310 }
1311
1312 tcs->clause[tcs->clause_count] = clause;
1313 tcs->clause_count++;
1314
1315 return true;
1316 }
1317
1318 /* TODO: Remove this mess. */
1319 #ifdef HAVE_DIRCACHE
1320 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1321 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1322 #else
1323 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1324 #endif
1325
1326 static bool get_next(struct tagcache_search *tcs)
1327 {
1328 static char buf[TAG_MAXLEN+32];
1329 struct tagfile_entry entry;
1330 long flag = 0;
1331
1332 if (!tcs->valid || !tc_stat.ready)
1333 return false;
1334
1335 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1336 #ifdef HAVE_TC_RAMCACHE
1337 && !tcs->ramsearch
1338 #endif
1339 )
1340 return false;
1341
1342 /* Relative fetch. */
1343 if (tcs->filter_count > 0 || tcs->clause_count > 0
1344 || tagcache_is_numeric_tag(tcs->type))
1345 {
1346 /* Check for end of list. */
1347 if (tcs->seek_list_count == 0)
1348 {
1349 /* Try to fetch more. */
1350 if (!build_lookup_list(tcs))
1351 {
1352 tcs->valid = false;
1353 return false;
1354 }
1355 }
1356
1357 tcs->seek_list_count--;
1358 flag = tcs->seek_flags[tcs->seek_list_count];
1359
1360 /* Seek stream to the correct position and continue to direct fetch. */
1361 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1362 && !tagcache_is_numeric_tag(tcs->type))
1363 {
1364 if (!open_files(tcs, tcs->type))
1365 return false;
1366
1367 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1368 }
1369 else
1370 tcs->position = tcs->seek_list[tcs->seek_list_count];
1371 }
1372
1373 if (tagcache_is_numeric_tag(tcs->type))
1374 {
1375 snprintf(buf, sizeof(buf), "%d", tcs->position);
1376 tcs->result_seek = tcs->position;
1377 tcs->result = buf;
1378 tcs->result_len = strlen(buf) + 1;
1379 return true;
1380 }
1381
1382 /* Direct fetch. */
1383 #ifdef HAVE_TC_RAMCACHE
1384 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1385 {
1386 struct tagfile_entry *ep;
1387
1388 if (tcs->entry_count == 0)
1389 {
1390 tcs->valid = false;
1391 return false;
1392 }
1393 tcs->entry_count--;
1394
1395 tcs->result_seek = tcs->position;
1396
1397 # ifdef HAVE_DIRCACHE
1398 if (tcs->type == tag_filename)
1399 {
1400 dircache_copy_path((struct dirent *)tcs->position,
1401 buf, sizeof buf);
1402 tcs->result = buf;
1403 tcs->result_len = strlen(buf) + 1;
1404 tcs->idx_id = FLAG_GET_ATTR(flag);
1405 tcs->ramresult = false;
1406
1407 return true;
1408 }
1409 # endif
1410
1411 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1412 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1413 tcs->result = ep->tag_data;
1414 tcs->result_len = strlen(tcs->result) + 1;
1415 tcs->idx_id = ep->idx_id;
1416 tcs->ramresult = true;
1417
1418 return true;
1419 }
1420 else
1421 #endif
1422 {
1423 if (!open_files(tcs, tcs->type))
1424 return false;
1425
1426 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1427 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1428 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1429 {
1430 /* End of data. */
1431 tcs->valid = false;
1432 return false;
1433 }
1434 }
1435
1436 if (entry.tag_length > (long)sizeof(buf))
1437 {
1438 tcs->valid = false;
1439 logf("too long tag #2");
1440 return false;
1441 }
1442
1443 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1444 {
1445 tcs->valid = false;
1446 logf("read error #4");
1447 return false;
1448 }
1449
1450 tcs->result = buf;
1451 tcs->result_len = strlen(tcs->result) + 1;
1452 tcs->idx_id = entry.idx_id;
1453 tcs->ramresult = false;
1454
1455 return true;
1456 }
1457
1458 bool tagcache_get_next(struct tagcache_search *tcs)
1459 {
1460 while (get_next(tcs))
1461 {
1462 if (tcs->result_len > 1)
1463 return true;
1464 }
1465
1466 return false;
1467 }
1468
1469 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1470 int tag, char *buf, long size)
1471 {
1472 struct index_entry idx;
1473
1474 *buf = '\0';
1475 if (!get_index(tcs->masterfd, idxid, &idx, true))
1476 return false;
1477
1478 return retrieve(tcs, &idx, tag, buf, size);
1479 }
1480
1481 static bool update_master_header(void)
1482 {
1483 struct master_header myhdr;
1484 int fd;
1485
1486 if (!tc_stat.ready)
1487 return false;
1488
1489 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1490 return false;
1491
1492 myhdr.serial = current_tcmh.serial;
1493 myhdr.commitid = current_tcmh.commitid;
1494 myhdr.dirty = current_tcmh.dirty;
1495
1496 /* Write it back */
1497 lseek(fd, 0, SEEK_SET);
1498 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1499 close(fd);
1500
1501 #ifdef HAVE_TC_RAMCACHE
1502 if (hdr)
1503 {
1504 hdr->h.serial = current_tcmh.serial;
1505 hdr->h.commitid = current_tcmh.commitid;
1506 hdr->h.dirty = current_tcmh.dirty;
1507 }
1508 #endif
1509
1510 return true;
1511 }
1512
1513 #if 0
1514
1515 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1516 {
1517 struct tagentry *entry;
1518
1519 if (tcs->type != tag_title)
1520 return ;
1521
1522 /* We will need reserve buffer for this. */
1523 if (tcs->ramcache)
1524 {
1525 struct tagfile_entry *ep;
1526
1527 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1528 tcs->seek_list[tcs->seek_list_count];
1529 }
1530
1531 entry = find_entry_ram();
1532
1533 }
1534 #endif
1535
1536 void tagcache_search_finish(struct tagcache_search *tcs)
1537 {
1538 int i;
1539
1540 if (!tcs->initialized)
1541 return;
1542
1543 if (tcs->masterfd >= 0)
1544 {
1545 close(tcs->masterfd);
1546 tcs->masterfd = -1;
1547 }
1548
1549 for (i = 0; i < TAG_COUNT; i++)
1550 {
1551 if (tcs->idxfd[i] >= 0)
1552 {
1553 close(tcs->idxfd[i]);
1554 tcs->idxfd[i] = -1;
1555 }
1556 }
1557
1558 tcs->ramsearch = false;
1559 tcs->valid = false;
1560 tcs->initialized = 0;
1561 if (write_lock > 0)
1562 write_lock--;
1563 }
1564
1565 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1566 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1567 {
1568 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1569 }
1570
1571 static long get_tag_numeric(const struct index_entry *entry, int tag)
1572 {
1573 return check_virtual_tags(tag, entry);
1574 }
1575
1576 static char* get_tag_string(const struct index_entry *entry, int tag)
1577 {
1578 char* s = get_tag(entry, tag)->tag_data;
1579 return strcmp(s, UNTAGGED) ? s : NULL;
1580 }
1581
1582 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1583 {
1584 struct index_entry *entry;
1585 int idx_id;
1586
1587 if (!tc_stat.ready)
1588 return false;
1589
1590 /* Find the corresponding entry in tagcache. */
1591 idx_id = find_entry_ram(filename, NULL);
1592 if (idx_id < 0 || !tc_stat.ramcache)
1593 return false;
1594
1595 entry = &hdr->indices[idx_id];
1596
1597 id3->title = get_tag_string(entry, tag_title);
1598 id3->artist = get_tag_string(entry, tag_artist);
1599 id3->album = get_tag_string(entry, tag_album);
1600 id3->genre_string = get_tag_string(entry, tag_genre);
1601 id3->composer = get_tag_string(entry, tag_composer);
1602 id3->comment = get_tag_string(entry, tag_comment);
1603 id3->albumartist = get_tag_string(entry, tag_albumartist);
1604 id3->grouping = get_tag_string(entry, tag_grouping);
1605
1606 id3->playcount = get_tag_numeric(entry, tag_playcount);
1607 id3->rating = get_tag_numeric(entry, tag_rating);
1608 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1609 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1610 id3->year = get_tag_numeric(entry, tag_year);
1611
1612 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1613 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1614 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1615 if (id3->bitrate == 0)
1616 id3->bitrate = 1;
1617
1618 return true;
1619 }
1620 #endif
1621
1622 static inline void write_item(const char *item)
1623 {
1624 int len = strlen(item) + 1;
1625
1626 data_size += len;
1627 write(cachefd, item, len);
1628 }
1629
1630 static int check_if_empty(char **tag)
1631 {
1632 int length;
1633
1634 if (*tag == NULL || **tag == '\0')
1635 {
1636 *tag = UNTAGGED;
1637 return sizeof(UNTAGGED); /* Tag length */
1638 }
1639
1640 length = strlen(*tag);
1641 if (length > TAG_MAXLEN)
1642 {
1643 logf("over length tag: %s", *tag);
1644 length = TAG_MAXLEN;
1645 (*tag)[length] = '\0';
1646 }
1647
1648 return length + 1;
1649 }
1650
1651 #define ADD_TAG(entry,tag,data) \
1652 /* Adding tag */ \
1653 entry.tag_offset[tag] = offset; \
1654 entry.tag_length[tag] = check_if_empty(data); \
1655 offset += entry.tag_length[tag]
1656
1657 static void add_tagcache(char *path, unsigned long mtime
1658 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1659 ,const struct dirent *dc
1660 #endif
1661 )
1662 {
1663 struct mp3entry id3;
1664 struct temp_file_entry entry;
1665 bool ret;
1666 int fd;
1667 int idx_id = -1;
1668 char tracknumfix[3];
1669 int offset = 0;
1670 int path_length = strlen(path);
1671 bool has_albumartist;
1672 bool has_grouping;
1673
1674 if (cachefd < 0)
1675 return ;
1676
1677 /* Check for overlength file path. */
1678 if (path_length > TAG_MAXLEN)
1679 {
1680 /* Path can't be shortened. */
1681 logf("Too long path: %s", path);
1682 return ;
1683 }
1684
1685 /* Check if the file is supported. */
1686 if (probe_file_format(path) == AFMT_UNKNOWN)
1687 return ;
1688
1689 /* Check if the file is already cached. */
1690 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1691 if (tc_stat.ramcache && is_dircache_intact())
1692 {
1693 idx_id = find_entry_ram(path, dc);
1694 }
1695 else
1696 #endif
1697 {
1698 if (filenametag_fd >= 0)
1699 {
1700 idx_id = find_entry_disk(path);
1701 }
1702 }
1703
1704 /* Check if file has been modified. */
1705 if (idx_id >= 0)
1706 {
1707 struct index_entry idx;
1708
1709 /* TODO: Mark that the index exists (for fast reverse scan) */
1710 //found_idx[idx_id/8] |= idx_id%8;
1711
1712 if (!get_index(-1, idx_id, &idx, true))
1713 {
1714 logf("failed to retrieve index entry");
1715 return ;
1716 }
1717
1718 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1719 {
1720 /* No changes to file. */
1721 return ;
1722 }
1723
1724 /* Metadata might have been changed. Delete the entry. */
1725 logf("Re-adding: %s", path);
1726 if (!delete_entry(idx_id))
1727 {
1728 logf("delete_entry failed: %d", idx_id);
1729 return ;
1730 }
1731 }
1732
1733 fd = open(path, O_RDONLY);
1734 if (fd < 0)
1735 {
1736 logf("open fail: %s", path);
1737 return ;
1738 }
1739
1740 memset(&id3, 0, sizeof(struct mp3entry));
1741 memset(&entry, 0, sizeof(struct temp_file_entry));
1742 memset(&tracknumfix, 0, sizeof(tracknumfix));
1743 ret = get_metadata(&id3, fd, path);
1744 close(fd);
1745
1746 if (!ret)
1747 return ;
1748
1749 logf("-> %s", path);
1750
1751 /* Generate track number if missing. */
1752 if (id3.tracknum <= 0)
1753 {
1754 const char *p = strrchr(path, '.');
1755
1756 if (p == NULL)
1757 p = &path[strlen(path)-1];
1758
1759 while (*p != '/')
1760 {
1761 if (isdigit(*p) && isdigit(*(p-1)))
1762 {
1763 tracknumfix[1] = *p--;
1764 tracknumfix[0] = *p;
1765 break;
1766 }
1767 p--;
1768 }
1769
1770 if (tracknumfix[0] != '\0')
1771 {
1772 id3.tracknum = atoi(tracknumfix);
1773 /* Set a flag to indicate track number has been generated. */
1774 entry.flag |= FLAG_TRKNUMGEN;
1775 }
1776 else
1777 {
1778 /* Unable to generate track number. */
1779 id3.tracknum = -1;
1780 }
1781 }
1782
1783 /* Numeric tags */
1784 entry.tag_offset[tag_year] = id3.year;
1785 entry.tag_offset[tag_discnumber] = id3.discnum;
1786 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1787 entry.tag_offset[tag_length] = id3.length;
1788 entry.tag_offset[tag_bitrate] = id3.bitrate;
1789 entry.tag_offset[tag_mtime] = mtime;
1790
1791 /* String tags. */
1792 has_albumartist = id3.albumartist != NULL
1793 && strlen(id3.albumartist) > 0;
1794 has_grouping = id3.grouping != NULL
1795 && strlen(id3.grouping) > 0;
1796
1797 ADD_TAG(entry, tag_filename, &path);
1798 ADD_TAG(entry, tag_title, &id3.title);
1799 ADD_TAG(entry, tag_artist, &id3.artist);
1800 ADD_TAG(entry, tag_album, &id3.album);
1801 ADD_TAG(entry, tag_genre, &id3.genre_string);
1802 ADD_TAG(entry, tag_composer, &id3.composer);
1803 ADD_TAG(entry, tag_comment, &id3.comment);
1804 if (has_albumartist)
1805 {
1806 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1807 }
1808 else
1809 {
1810 ADD_TAG(entry, tag_albumartist, &id3.artist);
1811 }
1812 if (has_grouping)
1813 {
1814 ADD_TAG(entry, tag_grouping, &id3.grouping);
1815 }
1816 else
1817 {
1818 ADD_TAG(entry, tag_grouping, &id3.title);
1819 }
1820 entry.data_length = offset;
1821
1822 /* Write the header */
1823 write(cachefd, &entry, sizeof(struct temp_file_entry));
1824
1825 /* And tags also... Correct order is critical */
1826 write_item(path);
1827 write_item(id3.title);
1828 write_item(id3.artist);
1829 write_item(id3.album);
1830 write_item(id3.genre_string);
1831 write_item(id3.composer);
1832 write_item(id3.comment);
1833 if (has_albumartist)
1834 {
1835 write_item(id3.albumartist);
1836 }
1837 else
1838 {
1839 write_item(id3.artist);
1840 }
1841 if (has_grouping)
1842 {
1843 write_item(id3.grouping);
1844 }
1845 else
1846 {
1847 write_item(id3.title);
1848 }
1849 total_entry_count++;
1850 }
1851
1852 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1853 {
1854 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1855 int len = strlen(str)+1;
1856 int i;
1857 unsigned crc32;
1858 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1859 char buf[TAG_MAXLEN+32];
1860
1861 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1862 buf[i] = tolower(str[i]);
1863 buf[i] = '\0';
1864
1865 crc32 = crc_32(buf, i, 0xffffffff);
1866
1867 if (unique)
1868 {
1869 /* Check if the crc does not exist -> entry does not exist for sure. */
1870 for (i = 0; i < tempbufidx; i++)
1871 {
1872 if (crcbuf[-i] != crc32)
1873 continue;
1874
1875 if (!strcasecmp(str, index[i].str))
1876 {
1877 if (id < 0 || id >= lookup_buffer_depth)
1878 {
1879 logf("lookup buf overf.: %d", id);
1880 return false;
1881 }
1882
1883 lookup[id] = &index[i];
1884 return true;
1885 }
1886 }
1887 }
1888
1889 /* Insert to CRC buffer. */
1890 crcbuf[-tempbufidx] = crc32;
1891 tempbuf_left -= 4;
1892
1893 /* Insert it to the buffer. */
1894 tempbuf_left -= len;
1895 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1896 return false;
1897
1898 if (id >= lookup_buffer_depth)
1899 {
1900 logf("lookup buf overf. #2: %d", id);
1901 return false;
1902 }
1903
1904 if (id >= 0)
1905 {
1906 lookup[id] = &index[tempbufidx];
1907 index[tempbufidx].idlist.id = id;
1908 }
1909 else
1910 index[tempbufidx].idlist.id = -1;
1911
1912 index[tempbufidx].idlist.next = NULL;
1913 index[tempbufidx].idx_id = idx_id;
1914 index[tempbufidx].seek = -1;
1915 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1916 memcpy(index[tempbufidx].str, str, len);
1917 tempbuf_pos += len;
1918 tempbufidx++;
1919
1920 return true;
1921 }
1922
1923 static int compare(const void *p1, const void *p2)
1924 {
1925 do_timed_yield();
1926
1927 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1928 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1929
1930 if (strcmp(e1->str, UNTAGGED) == 0)
1931 {
1932 if (strcmp(e2->str, UNTAGGED) == 0)
1933 return 0;
1934 return -1;
1935 }
1936 else if (strcmp(e2->str, UNTAGGED) == 0)
1937 return 1;
1938
1939 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1940 }
1941
1942 static int tempbuf_sort(int fd)
1943 {
1944 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1945 struct tagfile_entry fe;
1946 int i;
1947 int length;
1948
1949 /* Generate reverse lookup entries. */
1950 for (i = 0; i < lookup_buffer_depth; i++)
1951 {
1952 struct tempbuf_id_list *idlist;
1953
1954 if (!lookup[i])
1955 continue;
1956
1957 if (lookup[i]->idlist.id == i)
1958 continue;
1959
1960 idlist = &lookup[i]->idlist;
1961 while (idlist->next != NULL)
1962 idlist = idlist->next;
1963
1964 tempbuf_left -= sizeof(struct tempbuf_id_list);
1965 if (tempbuf_left - 4 < 0)
1966 return -1;
1967
1968 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1969 if (tempbuf_pos & 0x03)
1970 {
1971 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1972 tempbuf_left -= 3;
1973 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1974 }
1975 tempbuf_pos += sizeof(struct tempbuf_id_list);
1976
1977 idlist = idlist->next;
1978 idlist->id = i;
1979 idlist->next = NULL;
1980
1981 do_timed_yield();
1982 }
1983
1984 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1985 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1986
1987 for (i = 0; i < tempbufidx; i++)
1988 {
1989 struct tempbuf_id_list *idlist = &index[i].idlist;
1990
1991 /* Fix the lookup list. */
1992 while (idlist != NULL)
1993 {
1994 if (idlist->id >= 0)
1995 lookup[idlist->id] = &index[i];
1996 idlist = idlist->next;
1997 }
1998
1999 index[i].seek = lseek(fd, 0, SEEK_CUR);
2000 length = strlen(index[i].str) + 1;
2001 fe.tag_length = length;
2002 fe.idx_id = index[i].idx_id;
2003
2004 /* Check the chunk alignment. */
2005 if ((fe.tag_length + sizeof(struct tagfile_entry))
2006 % TAGFILE_ENTRY_CHUNK_LENGTH)
2007 {
2008 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2009 ((fe.tag_length + sizeof(struct tagfile_entry))
2010 % TAGFILE_ENTRY_CHUNK_LENGTH);
2011 }
2012
2013 #ifdef TAGCACHE_STRICT_ALIGN
2014 /* Make sure the entry is long aligned. */
2015 if (index[i].seek & 0x03)
2016 {
2017 logf("tempbuf_sort: alignment error!");
2018 return -3;
2019 }
2020 #endif
2021
2022 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2023 sizeof(struct tagfile_entry))
2024 {
2025 logf("tempbuf_sort: write error #1");
2026 return -1;
2027 }
2028
2029 if (write(fd, index[i].str, length) != length)
2030 {
2031 logf("tempbuf_sort: write error #2");
2032 return -2;
2033 }
2034
2035 /* Write some padding. */
2036 if (fe.tag_length - length > 0)
2037 write(fd, "XXXXXXXX", fe.tag_length - length);
2038 }
2039
2040 return i;
2041 }
2042
2043 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2044 {
2045 if (id < 0 || id >= lookup_buffer_depth)
2046 return NULL;
2047
2048 return lookup[id];
2049 }
2050
2051
2052 inline static int tempbuf_find_location(int id)
2053 {
2054 struct tempbuf_searchidx *entry;
2055
2056 entry = tempbuf_locate(id);
2057 if (entry == NULL)
2058 return -1;
2059
2060 return entry->seek;
2061 }
2062
2063 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2064 {
2065 struct master_header tcmh;
2066 struct index_entry idx;
2067 int masterfd;
2068 int masterfd_pos;
2069 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2070 int max_entries;
2071 int entries_processed = 0;
2072 int i, j;
2073 char buf[TAG_MAXLEN];
2074
2075 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2076
2077 logf("Building numeric indices...");
2078 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2079
2080 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2081 return false;
2082
2083 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2084 SEEK_CUR);
2085 if (masterfd_pos == filesize(masterfd))
2086 {
2087 logf("we can't append!");
2088 close(masterfd);
2089 return false;
2090 }
2091
2092 while (entries_processed < h->entry_count)
2093 {
2094 int count = MIN(h->entry_count - entries_processed, max_entries);
2095
2096 /* Read in as many entries as possible. */
2097 for (i = 0; i < count; i++)
2098 {
2099 struct temp_file_entry *tfe = &entrybuf[i];
2100 int datastart;
2101
2102 /* Read in numeric data. */
2103 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2104 sizeof(struct temp_file_entry))
2105 {
2106 logf("read fail #1");
2107 close(masterfd);
2108 return false;
2109 }
2110
2111 datastart = lseek(tmpfd, 0, SEEK_CUR);
2112
2113 /**
2114 * Read string data from the following tags:
2115 * - tag_filename
2116 * - tag_artist
2117 * - tag_album
2118 * - tag_title
2119 *
2120 * A crc32 hash is calculated from the read data
2121 * and stored back to the data offset field kept in memory.
2122 */
2123 #define tmpdb_read_string_tag(tag) \
2124 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2125 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2126 { \
2127 logf("read fail: buffer overflow"); \
2128 close(masterfd); \
2129 return false; \
2130 } \
2131 \
2132 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2133 tfe->tag_length[tag]) \
2134 { \
2135 logf("read fail #2"); \
2136 close(masterfd); \
2137 return false; \
2138 } \
2139 \
2140 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2141 lseek(tmpfd, datastart, SEEK_SET)
2142
2143 tmpdb_read_string_tag(tag_filename);
2144 tmpdb_read_string_tag(tag_artist);
2145 tmpdb_read_string_tag(tag_album);
2146 tmpdb_read_string_tag(tag_title);
2147
2148 /* Seek to the end of the string data. */
2149 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2150 }
2151
2152 /* Backup the master index position. */
2153 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2154 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2155
2156 /* Check if we can resurrect some deleted runtime statistics data. */
2157 for (i = 0; i < tcmh.tch.entry_count; i++)
2158 {
2159 /* Read the index entry. */
2160 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2161 != sizeof(struct index_entry))
2162 {
2163 logf("read fail #3");
2164 close(masterfd);
2165 return false;
2166 }
2167
2168 /**
2169 * Skip unless the entry is marked as being deleted
2170 * or the data has already been resurrected.
2171 */
2172 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2173 continue;
2174
2175 /* Now try to match the entry. */
2176 /**
2177 * To succesfully match a song, the following conditions
2178 * must apply:
2179 *
2180 * For numeric fields: tag_length
2181 * - Full identical match is required
2182 *
2183 * If tag_filename matches, no further checking necessary.
2184 *
2185 * For string hashes: tag_artist, tag_album, tag_title
2186 * - Two of these must match
2187 */
2188 for (j = 0; j < count; j++)
2189 {
2190 struct temp_file_entry *tfe = &entrybuf[j];
2191
2192 /* Try to match numeric fields first. */
2193 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2194 continue;
2195
2196 /* Now it's time to do the hash matching. */
2197 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2198 {
2199 int match_count = 0;
2200
2201 /* No filename match, check if we can match two other tags. */
2202 #define tmpdb_match(tag) \
2203 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2204 match_count++
2205
2206 tmpdb_match(tag_artist);
2207 tmpdb_match(tag_album);
2208 tmpdb_match(tag_title);
2209
2210 if (match_count < 2)
2211 {
2212 /* Still no match found, give up. */
2213 continue;
2214 }
2215 }
2216
2217 /* A match found, now copy & resurrect the statistical data. */
2218 #define tmpdb_copy_tag(tag) \
2219 tfe->tag_offset[tag] = idx.tag_seek[tag]
2220
2221 tmpdb_copy_tag(tag_playcount);
2222 tmpdb_copy_tag(tag_rating);
2223 tmpdb_copy_tag(tag_playtime);
2224 tmpdb_copy_tag(tag_lastplayed);
2225 tmpdb_copy_tag(tag_commitid);
2226
2227 /* Avoid processing this entry again. */
2228 idx.flag |= FLAG_RESURRECTED;
2229
2230 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2231 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2232 != sizeof(struct index_entry))
2233 {
2234 logf("masterfd writeback fail #1");
2235 close(masterfd);
2236 return false;
2237 }
2238
2239 logf("Entry resurrected");
2240 }
2241 }
2242
2243
2244 /* Restore the master index position. */
2245 lseek(masterfd, masterfd_pos, SEEK_SET);
2246
2247 /* Commit the data to the index. */
2248 for (i = 0; i < count; i++)
2249 {
2250 int loc = lseek(masterfd, 0, SEEK_CUR);
2251
2252 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2253 != sizeof(struct index_entry))
2254 {
2255 logf("read fail #3");
2256 close(masterfd);
2257 return false;
2258 }
2259
2260 for (j = 0; j < TAG_COUNT; j++)
2261 {
2262 if (!tagcache_is_numeric_tag(j))
2263 continue;
2264
2265 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2266 }
2267 idx.flag = entrybuf[i].flag;
2268
2269 if (idx.tag_seek[tag_commitid])
2270 {
2271 /* Data has been resurrected. */
2272 idx.flag |= FLAG_DIRTYNUM;
2273 }
2274 else if (tc_stat.ready && current_tcmh.commitid > 0)
2275 {
2276 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2277 idx.flag |= FLAG_DIRTYNUM;
2278 }
2279
2280 /* Write back the updated index. */
2281 lseek(masterfd, loc, SEEK_SET);
2282 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2283 != sizeof(struct index_entry))
2284 {
2285 logf("write fail");
2286 close(masterfd);
2287 return false;
2288 }
2289 }
2290
2291 entries_processed += count;
2292 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2293 }
2294
2295 close(masterfd);
2296
2297 return true;
2298 }
2299
2300 /**
2301 * Return values:
2302 * > 0 success
2303 * == 0 temporary failure
2304 * < 0 fatal error
2305 */
2306 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2307 {
2308 int i;
2309 struct tagcache_header tch;
2310 struct master_header tcmh;
2311 struct index_entry idxbuf[IDX_BUF_DEPTH];
2312 int idxbuf_pos;
2313 char buf[TAG_MAXLEN+32];
2314 int fd = -1, masterfd;
2315 bool error = false;
2316 int init;
2317 int masterfd_pos;
2318
2319 logf("Building index: %d", index_type);
2320
2321 /* Check the number of entries we need to allocate ram for. */
2322 commit_entry_count = h->entry_count + 1;
2323
2324 masterfd = open_master_fd(&tcmh, false);
2325 if (masterfd >= 0)
2326 {
2327 commit_entry_count += tcmh.tch.entry_count;
2328 close(masterfd);
2329 }
2330 else
2331 remove_files(); /* Just to be sure we are clean. */
2332
2333 /* Open the index file, which contains the tag names. */
2334 fd = open_tag_fd(&tch, index_type, true);
2335 if (fd >= 0)
2336 {
2337 logf("tch.datasize=%ld", tch.datasize);
2338 lookup_buffer_depth = 1 +
2339 /* First part */ commit_entry_count +
2340 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2341 }
2342 else
2343 {
2344 lookup_buffer_depth = 1 +
2345 /* First part */ commit_entry_count +
2346 /* Second part */ 0;
2347 }
2348
2349 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2350 logf("commit_entry_count=%ld", commit_entry_count);
2351
2352 /* Allocate buffer for all index entries from both old and new
2353 * tag files. */
2354 tempbufidx = 0;
2355 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2356
2357 /* Allocate lookup buffer. The first portion of commit_entry_count
2358 * contains the new tags in the temporary file and the second
2359 * part for locating entries already in the db.
2360 *
2361 * New tags Old tags
2362 * +---------+---------------------------+
2363 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2364 * +---------+---------------------------+
2365 *
2366 * Old tags are inserted to a temporary buffer with position:
2367 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2368 * And new tags with index:
2369 * tempbuf_insert(idx, ...);
2370 *
2371 * The buffer is sorted and written into tag file:
2372 * tempbuf_sort(...);
2373 * leaving master index locations messed up.
2374 *
2375 * That is fixed using the lookup buffer for old tags:
2376 * new_seek = tempbuf_find_location(old_seek, ...);
2377 * and for new tags:
2378 * new_seek = tempbuf_find_location(idx);
2379 */
2380 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2381 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2382 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2383
2384 /* And calculate the remaining data space used mainly for storing
2385 * tag data (strings). */
2386 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2387 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2388 {
2389 logf("Buffer way too small!");
2390 return 0;
2391 }
2392
2393 if (fd >= 0)
2394 {
2395 /**
2396 * If tag file contains unique tags (sorted index), we will load
2397 * it entirely into memory so we can resort it later for use with
2398 * chunked browsing.
2399 */
2400 if (tagcache_is_sorted_tag(index_type))
2401 {
2402 logf("loading tags...");
2403 for (i = 0; i < tch.entry_count; i++)
2404 {
2405 struct tagfile_entry entry;
2406 int loc = lseek(fd, 0, SEEK_CUR);
2407 bool ret;
2408
2409 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2410 != sizeof(struct tagfile_entry))
2411 {
2412 logf("read error #7");
2413 close(fd);
2414 return -2;
2415 }
2416
2417 if (entry.tag_length >= (int)sizeof(buf))
2418 {
2419 logf("too long tag #3");
2420 close(fd);
2421 return -2;
2422 }
2423
2424 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2425 {
2426 logf("read error #8");
2427 close(fd);
2428 return -2;
2429 }
2430
2431 /* Skip deleted entries. */
2432 if (buf[0] == '\0')
2433 continue;
2434
2435 /**
2436 * Save the tag and tag id in the memory buffer. Tag id
2437 * is saved so we can later reindex the master lookup
2438 * table when the index gets resorted.
2439 */
2440 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2441 + commit_entry_count, entry.idx_id,
2442 tagcache_is_unique_tag(index_type));
2443 if (!ret)
2444 {
2445 close(fd);
2446 return -3;
2447 }
2448 do_timed_yield();
2449 }
2450 logf("done");
2451 }
2452 else
2453 tempbufidx = tch.entry_count;
2454 }
2455 else
2456 {
2457 /**
2458 * Creating new index file to store the tags. No need to preload
2459 * anything whether the index type is sorted or not.
2460 */
2461 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2462 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2463 if (fd < 0)
2464 {
2465 logf("%s open fail", buf);
2466 return -2;
2467 }
2468
2469 tch.magic = TAGCACHE_MAGIC;
2470 tch.entry_count = 0;
2471 tch.datasize = 0;
2472
2473 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2474 != sizeof(struct tagcache_header))
2475 {
2476 logf("header write failed");
2477 close(fd);
2478 return -2;
2479 }
2480 }
2481
2482 /* Loading the tag lookup file as "master file". */
2483 logf("Loading index file");
2484 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2485
2486 if (masterfd < 0)
2487 {
2488 logf("Creating new DB");
2489 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2490
2491 if (masterfd < 0)
2492 {
2493 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2494 close(fd);
2495 return -2;
2496 }
2497
2498 /* Write the header (write real values later). */
2499 memset(&tcmh, 0, sizeof(struct master_header));
2500 tcmh.tch = *h;
2501 tcmh.tch.entry_count = 0;
2502 tcmh.tch.datasize = 0;
2503 tcmh.dirty = true;
2504 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2505 init = true;
2506 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2507 }
2508 else
2509 {
2510 /**
2511 * Master file already exists so we need to process the current
2512 * file first.
2513 */
2514 init = false;
2515
2516 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2517 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2518 {
2519 logf("header error");
2520 close(fd);
2521 close(masterfd);
2522 return -2;
2523 }
2524
2525 /**
2526 * If we reach end of the master file, we need to expand it to
2527 * hold new tags. If the current index is not sorted, we can
2528 * simply append new data to end of the file.
2529 * However, if the index is sorted, we need to update all tag
2530 * pointers in the master file for the current index.
2531 */
2532 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2533 SEEK_CUR);
2534 if (masterfd_pos == filesize(masterfd))
2535 {
2536 logf("appending...");
2537 init = true;
2538 }
2539 }
2540
2541 /**
2542 * Load new unique tags in memory to be sorted later and added
2543 * to the master lookup file.
2544 */
2545 if (tagcache_is_sorted_tag(index_type))
2546 {
2547 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2548 /* h is the header of the temporary file containing new tags. */
2549 logf("inserting new tags...");
2550 for (i = 0; i < h->entry_count; i++)
2551 {
2552 struct temp_file_entry entry;
2553
2554 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2555 sizeof(struct temp_file_entry))
2556 {
2557 logf("read fail #3");
2558 error = true;
2559 goto error_exit;
2560 }
2561
2562 /* Read data. */
2563 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2564 {
2565 logf("too long entry!");
2566 error = true;
2567 goto error_exit;
2568 }
2569
2570 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2571 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2572 entry.tag_length[index_type])
2573 {
2574 logf("read fail #4");
2575 error = true;
2576 goto error_exit;
2577 }
2578
2579 if (tagcache_is_unique_tag(index_type))
2580 error = !tempbuf_insert(buf, i, -1, true);
2581 else
2582 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2583
2584 if (error)
2585 {
2586 logf("insert error");
2587 goto error_exit;
2588 }
2589
2590 /* Skip to next. */
2591 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2592 entry.tag_length[index_type], SEEK_CUR);
2593 do_timed_yield();
2594 }
2595 logf("done");
2596
2597 /* Sort the buffer data and write it to the index file. */
2598 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2599 i = tempbuf_sort(fd);
2600 if (i < 0)
2601 goto error_exit;
2602 logf("sorted %d tags", i);
2603
2604 /**
2605 * Now update all indexes in the master lookup file.
2606 */
2607 logf("updating indices...");
2608 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2609 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2610 {
2611 int j;
2612 int loc = lseek(masterfd, 0, SEEK_CUR);
2613
2614 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2615
2616 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2617 != (int)sizeof(struct index_entry)*idxbuf_pos)
2618 {
2619 logf("read fail #5");
2620 error = true;
2621 goto error_exit ;
2622 }
2623 lseek(masterfd, loc, SEEK_SET);
2624
2625 for (j = 0; j < idxbuf_pos; j++)
2626 {
2627 if (idxbuf[j].flag & FLAG_DELETED)
2628 {
2629 /* We can just ignore deleted entries. */
2630 // idxbuf[j].tag_seek[index_type] = 0;
2631 continue;
2632 }
2633
2634 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2635 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2636 + commit_entry_count);
2637
2638 if (idxbuf[j].tag_seek[index_type] < 0)
2639 {
2640 logf("update error: %d/%d/%d",
2641 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2642 error = true;
2643 goto error_exit;
2644 }
2645
2646 do_timed_yield();
2647 }
2648
2649 /* Write back the updated index. */
2650 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2651 index_entry_ec, tc_stat.econ) !=
2652 (int)sizeof(struct index_entry)*idxbuf_pos)
2653 {
2654 logf("write fail");
2655 error = true;
2656 goto error_exit;
2657 }
2658 }
2659 logf("done");
2660 }
2661
2662 /**
2663 * Walk through the temporary file containing the new tags.
2664 */
2665 // build_normal_index(h, tmpfd, masterfd, idx);
2666 logf("updating new indices...");
2667 lseek(masterfd, masterfd_pos, SEEK_SET);
2668 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2669 lseek(fd, 0, SEEK_END);
2670 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2671 {
2672 int j;
2673
2674 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2675 if (init)
2676 {
2677 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2678 }
2679 else
2680 {
2681 int loc = lseek(masterfd, 0, SEEK_CUR);
2682
2683 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2684 != (int)sizeof(struct index_entry)*idxbuf_pos)
2685 {
2686 logf("read fail #6");
2687 error = true;
2688 break ;
2689 }
2690 lseek(masterfd, loc, SEEK_SET);
2691 }
2692
2693 /* Read entry headers. */
2694 for (j = 0; j < idxbuf_pos; j++)
2695 {
2696 if (!tagcache_is_sorted_tag(index_type))
2697 {
2698 struct temp_file_entry entry;
2699 struct tagfile_entry fe;
2700
2701 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2702 sizeof(struct temp_file_entry))
2703 {
2704 logf("read fail #7");
2705 error = true;
2706 break ;
2707 }
2708
2709 /* Read data. */
2710 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2711 {
2712 logf("too long entry!");
2713 logf("length=%d", entry.tag_length[index_type]);
2714 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2715 error = true;
2716 break ;
2717 }
2718
2719 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2720 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2721 entry.tag_length[index_type])
2722 {
2723 logf("read fail #8");
2724 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2725 logf("length=0x%02x", entry.tag_length[index_type]);
2726 error = true;
2727 break ;
2728 }
2729
2730 /* Write to index file. */
2731 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2732 fe.tag_length = entry.tag_length[index_type];
2733 fe.idx_id = tcmh.tch.entry_count + i + j;
2734 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2735 write(fd, buf, fe.tag_length);
2736 tempbufidx++;
2737
2738 /* Skip to next. */
2739 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2740 entry.tag_length[index_type], SEEK_CUR);
2741 }
2742 else
2743 {
2744 /* Locate the correct entry from the sorted array. */
2745 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2746 if (idxbuf[j].tag_seek[index_type] < 0)
2747 {
2748 logf("entry not found (%d)", j);
2749 error = true;
2750 break ;
2751 }
2752 }
2753 }
2754
2755 /* Write index. */
2756 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2757 index_entry_ec, tc_stat.econ) !=
2758 (int)sizeof(struct index_entry)*idxbuf_pos)
2759 {
2760 logf("tagcache: write fail #4");
2761 error = true;
2762 break ;
2763 }
2764
2765 do_timed_yield();
2766 }
2767 logf("done");
2768
2769 /* Finally write the header. */
2770 tch.magic = TAGCACHE_MAGIC;
2771 tch.entry_count = tempbufidx;
2772 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2773 lseek(fd, 0, SEEK_SET);
2774 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2775
2776 if (index_type != tag_filename)
2777 h->datasize += tch.datasize;
2778 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2779 error_exit:
2780
2781 close(fd);
2782 close(masterfd);
2783
2784 if (error)
2785 return -2;
2786
2787 return 1;
2788 }
2789
2790 static bool commit(void)
2791 {
2792 struct tagcache_header tch;
2793 struct master_header tcmh;
2794 int i, len, rc;
2795 int tmpfd;
2796 int masterfd;
2797 #ifdef HAVE_DIRCACHE
2798 bool dircache_buffer_stolen = false;
2799 #endif
2800 bool local_allocation = false;
2801
2802 logf("committing tagcache");
2803
2804 while (write_lock)
2805 sleep(1);
2806
2807 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2808 if (tmpfd < 0)
2809 {
2810 logf("nothing to commit");
2811 return true;
2812 }
2813
2814
2815 /* Load the header. */
2816 len = sizeof(struct tagcache_header);
2817 rc = read(tmpfd, &tch, len);
2818
2819 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2820 {
2821 logf("incorrect header");
2822 close(tmpfd);
2823 remove(TAGCACHE_FILE_TEMP);
2824 return false;
2825 }
2826
2827 if (tch.entry_count == 0)
2828 {
2829 logf("nothing to commit");
2830 close(tmpfd);
2831 remove(TAGCACHE_FILE_TEMP);
2832 return true;
2833 }
2834
2835 #ifdef HAVE_EEPROM_SETTINGS
2836 remove(TAGCACHE_STATEFILE);
2837 #endif
2838
2839 /* At first be sure to unload the ramcache! */
2840 #ifdef HAVE_TC_RAMCACHE
2841 tc_stat.ramcache = false;
2842 #endif
2843
2844 read_lock++;
2845
2846 /* Try to steal every buffer we can :) */
2847 if (tempbuf_size == 0)
2848 local_allocation = true;
2849
2850 #ifdef HAVE_DIRCACHE
2851 if (tempbuf_size == 0)
2852 {
2853 /* Try to steal the dircache buffer. */
2854 tempbuf = dircache_steal_buffer(&tempbuf_size);
2855 tempbuf_size &= ~0x03;
2856
2857 if (tempbuf_size > 0)
2858 {
2859 dircache_buffer_stolen = true;
2860 }
2861 }
2862 #endif
2863
2864 #ifdef HAVE_TC_RAMCACHE
2865 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2866 {
2867 tempbuf = (char *)(hdr + 1);
2868 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2869 tempbuf_size &= ~0x03;
2870 }
2871 #endif
2872
2873 /* And finally fail if there are no buffers available. */
2874 if (tempbuf_size == 0)
2875 {
2876 logf("delaying commit until next boot");
2877 tc_stat.commit_delayed = true;
2878 close(tmpfd);
2879 read_lock--;
2880 return false;
2881 }
2882
2883 logf("commit %ld entries...", tch.entry_count);
2884
2885 /* Mark DB dirty so it will stay disabled if commit fails. */
2886 current_tcmh.dirty = true;
2887 update_master_header();
2888
2889 /* Now create the index files. */
2890 tc_stat.commit_step = 0;
2891 tch.datasize = 0;
2892 tc_stat.commit_delayed = false;
2893
2894 for (i = 0; i < TAG_COUNT; i++)
2895 {
2896 int ret;
2897
2898 if (tagcache_is_numeric_tag(i))
2899 continue;
2900
2901 tc_stat.commit_step++;
2902 ret = build_index(i, &tch, tmpfd);
2903 if (ret <= 0)
2904 {
2905 close(tmpfd);
2906 logf("tagcache failed init");
2907 if (ret == 0)
2908 tc_stat.commit_delayed = true;
2909
2910 tc_stat.commit_step = 0;
2911 read_lock--;
2912 return false;
2913 }
2914 }
2915
2916 if (!build_numeric_indices(&tch, tmpfd))
2917 {
2918 logf("Failure to commit numeric indices");
2919 close(tmpfd);
2920 tc_stat.commit_step = 0;
2921 read_lock--;
2922 return false;
2923 }
2924
2925 close(tmpfd);
2926 remove(TAGCACHE_FILE_TEMP);
2927
2928 tc_stat.commit_step = 0;
2929
2930 /* Update the master index headers. */
2931 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2932 {
2933 read_lock--;
2934 return false;
2935 }
2936
2937 tcmh.tch.entry_count += tch.entry_count;
2938 tcmh.tch.datasize = sizeof(struct master_header)
2939 + sizeof(struct index_entry) * tcmh.tch.entry_count
2940 + tch.datasize;
2941 tcmh.dirty = false;
2942 tcmh.commitid++;
2943
2944 lseek(masterfd, 0, SEEK_SET);
2945 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2946 close(masterfd);
2947
2948 logf("tagcache committed");
2949 tc_stat.ready = check_all_headers();
2950 tc_stat.readyvalid = true;
2951
2952 if (local_allocation)
2953 {
2954 tempbuf = NULL;
2955 tempbuf_size = 0;
2956 }
2957
2958 #ifdef HAVE_DIRCACHE
2959 /* Rebuild the dircache, if we stole the buffer. */
2960 if (dircache_buffer_stolen)
2961 dircache_build(0);
2962 #endif
2963
2964 #ifdef HAVE_TC_RAMCACHE
2965 /* Reload tagcache. */
2966 if (tc_stat.ramcache_allocated > 0)
2967 tagcache_start_scan();
2968 #endif
2969
2970 read_lock--;
2971
2972 return true;
2973 }
2974
2975 static void allocate_tempbuf(void)
2976 {
2977 /* Yeah, malloc would be really nice now :) */
2978 #ifdef __PCTOOL__
2979 tempbuf_size = 32*1024*1024;
2980 tempbuf = malloc(tempbuf_size);
2981 #else
2982 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2983 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2984 audiobuf += tempbuf_size;
2985 #endif
2986 }
2987
2988 static void free_tempbuf(void)
2989 {
2990 if (tempbuf_size == 0)
2991 return ;
2992
2993 #ifdef __PCTOOL__
2994 free(tempbuf);
2995 #else
2996 audiobuf -= tempbuf_size;
2997 #endif
2998 tempbuf = NULL;
2999 tempbuf_size = 0;
3000 }
3001
3002 #ifndef __PCTOOL__
3003
3004 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3005 {
3006 struct index_entry idx;
3007
3008 if (!tc_stat.ready)
3009 return false;
3010
3011 if (!tagcache_is_numeric_tag(tag))
3012 return false;
3013
3014 if (!get_index(masterfd, idx_id, &idx, false))
3015 return false;
3016
3017 idx.tag_seek[tag] = data;
3018 idx.flag |= FLAG_DIRTYNUM;
3019
3020 return write_index(masterfd, idx_id, &idx);
3021 }
3022
3023 #if 0
3024 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3025 int tag, long data)
3026 {
3027 struct master_header myhdr;
3028
3029 if (tcs->masterfd < 0)
3030 {
3031 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3032 return false;
3033 }
3034
3035 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3036 }
3037 #endif
3038
3039 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3040
3041 static bool command_queue_is_full(void)
3042 {
3043 int next;
3044
3045 next = command_queue_widx + 1;
3046 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3047 next = 0;
3048
3049 return (next == command_queue_ridx);
3050 }
3051
3052 static bool command_queue_sync_callback(void)
3053 {
3054
3055 struct master_header myhdr;
3056 int masterfd;
3057
3058 mutex_lock(&command_queue_mutex);
3059
3060 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3061 return false;
3062
3063 while (command_queue_ridx != command_queue_widx)
3064 {
3065 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3066
3067 switch (ce->command)
3068 {
3069 case CMD_UPDATE_MASTER_HEADER:
3070 {
3071 close(masterfd);
3072 update_master_header();
3073
3074 /* Re-open the masterfd. */
3075 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3076 return true;
3077
3078 break;
3079 }
3080 case CMD_UPDATE_NUMERIC:
3081 {
3082 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3083 break;
3084 }
3085 }
3086
3087 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3088 command_queue_ridx = 0;
3089 }
3090
3091 close(masterfd);
3092
3093 tc_stat.queue_length = 0;
3094 mutex_unlock(&command_queue_mutex);
3095 return true;
3096 }
3097
3098 static void run_command_queue(bool force)
3099 {
3100 if (COMMAND_QUEUE_IS_EMPTY)
3101 return;
3102
3103 if (force || command_queue_is_full())
3104 command_queue_sync_callback();
3105 else
3106 register_storage_idle_func(command_queue_sync_callback);
3107 }
3108
3109 static void queue_command(int cmd, long idx_id, int tag, long data)
3110 {
3111 while (1)
3112 {
3113 int next;
3114
3115 mutex_lock(&command_queue_mutex);
3116 next = command_queue_widx + 1;
3117 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3118 next = 0;
3119
3120 /* Make sure queue is not full. */
3121 if (next != command_queue_ridx)
3122 {
3123 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3124
3125 ce->command = cmd;
3126 ce->idx_id = idx_id;
3127 ce->tag = tag;
3128 ce->data = data;
3129
3130 command_queue_widx = next;
3131
3132 tc_stat.queue_length++;
3133
3134 mutex_unlock(&command_queue_mutex);
3135 break;
3136 }
3137
3138 /* Queue is full, try again later... */
3139 mutex_unlock(&command_queue_mutex);
3140 sleep(1);
3141 }
3142 }
3143
3144 long tagcache_increase_serial(void)
3145 {
3146 long old;
3147
3148 if (!tc_stat.ready)
3149 return -2;
3150
3151 while (read_lock)
3152 sleep(1);
3153
3154 old = current_tcmh.serial++;
3155 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3156
3157 return old;
3158 }
3159
3160 void tagcache_update_numeric(int idx_id, int tag, long data)
3161 {
3162 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3163 }
3164 #endif /* !__PCTOOL__ */
3165
3166 long tagcache_get_serial(void)
3167 {
3168 return current_tcmh.serial;
3169 }
3170
3171 long tagcache_get_commitid(void)
3172 {
3173 return current_tcmh.commitid;
3174 }
3175
3176 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3177 {
3178 char buf[512];
3179 int i;
3180
3181 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3182 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3183 {
3184 if (*datastr == '\0')
3185 break;
3186
3187 if (*datastr == '"' || *datastr == '\\')
3188 buf[i++] = '\\';
3189
3190 buf[i] = *(datastr++);
3191 }
3192
3193 strcpy(&buf[i], "\" ");
3194
3195 write(fd, buf, i + 2);
3196
3197 return true;
3198 }
3199
3200 #ifndef __PCTOOL__
3201
3202 static bool read_tag(char *dest, long size,
3203 const char *src, const char *tagstr)
3204 {
3205 int pos;
3206 char current_tag[32];
3207
3208 while (*src != '\0')
3209 {
3210 /* Skip all whitespace */
3211 while (*src == ' ')
3212 src++;
3213
3214 if (*src == '\0')
3215 break;
3216
3217 pos = 0;
3218 /* Read in tag name */
3219 while (*src != '=' && *src != ' ')
3220 {
3221 current_tag[pos] = *src;
3222 src++;
3223 pos++;
3224
3225 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3226 return false;
3227 }
3228 current_tag[pos] = '\0';
3229
3230 /* Read in tag data */
3231
3232 /* Find the start. */
3233 while (*src != '"' && *src != '\0')
3234 src++;
3235
3236 if (*src == '\0' || *(++src) == '\0')
3237 return false;
3238
3239 /* Read the data. */
3240 for (pos = 0; pos < size; pos++)
3241 {
3242 if (*src == '\0')
3243 break;
3244
3245 if (*src == '\\')
3246 {
3247 dest[pos] = *(src+1);
3248 src += 2;
3249 continue;
3250 }
3251
3252 dest[pos] = *src;
3253
3254 if (*src == '"')
3255 {
3256 src++;
3257 break;
3258 }
3259
3260 if (*src == '\0')
3261 break;
3262
3263 src++;
3264 }
3265 dest[pos] = '\0';
3266
3267 if (!strcasecmp(tagstr, current_tag))
3268 return true;
3269 }
3270
3271 return false;
3272 }
3273
3274 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3275 {
3276 struct index_entry idx;
3277 char tag_data[TAG_MAXLEN+32];
3278 int idx_id;
3279 long masterfd = (long)parameters;
3280 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3281 tag_commitid };
3282 int i;
3283 (void)line_n;
3284
3285 if (*buf == '#')
3286 return 0;
3287
3288 logf("%d/%s", line_n, buf);
3289 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3290 {
3291 logf("filename missing");
3292 logf("-> %s", buf);
3293 return 0;
3294 }
3295
3296 idx_id = find_index(tag_data);
3297 if (idx_id < 0)
3298 {
3299 logf("entry not found");
3300 return 0;
3301 }
3302
3303 if (!get_index(masterfd, idx_id, &idx, false))
3304 {
3305 logf("failed to retrieve index entry");
3306 return 0;
3307 }
3308
3309 /* Stop if tag has already been modified. */
3310 if (idx.flag & FLAG_DIRTYNUM)
3311 return 0;
3312
3313 logf("import: %s", tag_data);
3314
3315 idx.flag |= FLAG_DIRTYNUM;
3316 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3317 {
3318 int data;
3319
3320 if (!read_tag(tag_data, sizeof tag_data, buf,
3321 tagcache_tag_to_str(import_tags[i])))
3322 {
3323 continue;
3324 }
3325
3326 data = atoi(tag_data);
3327 if (data < 0)
3328 continue;
3329
3330 idx.tag_seek[import_tags[i]] = data;
3331
3332 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3333 current_tcmh.serial = data + 1;
3334 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3335 current_tcmh.commitid = data + 1;
3336 }
3337
3338 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3339 }
3340
3341 bool tagcache_import_changelog(void)
3342 {
3343 struct master_header myhdr;
3344 struct tagcache_header tch;
3345 int clfd;
3346 long masterfd;
3347 char buf[2048];
3348
3349 if (!tc_stat.ready)
3350 return false;
3351
3352 while (read_lock)
3353 sleep(1);
3354
3355 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3356 if (clfd < 0)
3357 {
3358 logf("failure to open changelog");
3359 return false;
3360 }
3361
3362 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3363 {
3364 close(clfd);
3365 return false;
3366 }
3367
3368 write_lock++;
3369
3370 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3371
3372 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3373 parse_changelog_line);
3374
3375 close(clfd);
3376 close(masterfd);
3377
3378 if (filenametag_fd >= 0)
3379 close(filenametag_fd);
3380
3381 write_lock--;
3382
3383 update_master_header();
3384
3385 return true;
3386 }
3387
3388 #endif /* !__PCTOOL__ */
3389
3390 bool tagcache_create_changelog(struct tagcache_search *tcs)
3391 {
3392 struct master_header myhdr;
3393 struct index_entry idx;
3394 char buf[TAG_MAXLEN+32];
3395 char temp[32];
3396 int clfd;
3397 int i, j;
3398
3399 if (!tc_stat.ready)
3400 return false;
3401
3402 if (!tagcache_search(tcs, tag_filename))
3403 return false;
3404
3405 /* Initialize the changelog */
3406 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3407 if (clfd < 0)
3408 {
3409 logf("failure to open changelog");
3410 return false;
3411 }
3412
3413 if (tcs->masterfd < 0)
3414 {
3415 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3416 return false;
3417 }
3418 else
3419 {
3420 lseek(tcs->masterfd, 0, SEEK_SET);
3421 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3422 }
3423
3424 write(clfd, "## Changelog version 1\n", 23);
3425
3426 for (i = 0; i < myhdr.tch.entry_count; i++)
3427 {
3428 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3429 != sizeof(struct index_entry))
3430 {
3431 logf("read error #9");
3432 tagcache_search_finish(tcs);
3433 close(clfd);
3434 return false;
3435 }
3436
3437 /* Skip until the entry found has been modified. */
3438 if (! (idx.flag & FLAG_DIRTYNUM) )
3439 continue;
3440
3441 /* Skip deleted entries too. */
3442 if (idx.flag & FLAG_DELETED)
3443 continue;
3444
3445 /* Now retrieve all tags. */
3446 for (j = 0; j < TAG_COUNT; j++)
3447 {
3448 if (tagcache_is_numeric_tag(j))
3449 {
3450 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3451 write_tag(clfd, tagcache_tag_to_str(j), temp);
3452 continue;
3453 }
3454
3455 tcs->type = j;
3456 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3457 write_tag(clfd, tagcache_tag_to_str(j), buf);
3458 }
3459
3460 write(clfd, "\n", 1);
3461 do_timed_yield();
3462 }
3463
3464 close(clfd);
3465
3466 tagcache_search_finish(tcs);
3467
3468 return true;
3469 }
3470
3471 static bool delete_entry(long idx_id)
3472 {
3473 int fd = -1;
3474 int masterfd = -1;
3475 int tag, i;
3476 struct index_entry idx, myidx;
3477 struct master_header myhdr;
3478 char buf[TAG_MAXLEN+32];
3479 int in_use[TAG_COUNT];
3480
3481 logf("delete_entry(): %ld", idx_id);
3482
3483 #ifdef HAVE_TC_RAMCACHE
3484 /* At first mark the entry removed from ram cache. */
3485 if (tc_stat.ramcache)
3486 hdr->indices[idx_id].flag |= FLAG_DELETED;
3487 #endif
3488
3489 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3490 return false;
3491
3492 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3493 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3494 != sizeof(struct index_entry))
3495 {
3496 logf("delete_entry(): read error");
3497 goto cleanup;
3498 }
3499
3500 if (myidx.flag & FLAG_DELETED)
3501 {
3502 logf("delete_entry(): already deleted!");
3503 goto cleanup;
3504 }
3505
3506 myidx.flag |= FLAG_DELETED;
3507 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3508 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3509 != sizeof(struct index_entry))
3510 {
3511 logf("delete_entry(): write_error #1");
3512 goto cleanup;
3513 }
3514
3515 /* Now check which tags are no longer in use (if any) */
3516 for (tag = 0; tag < TAG_COUNT; tag++)
3517 in_use[tag] = 0;
3518
3519 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3520 for (i = 0; i < myhdr.tch.entry_count; i++)
3521 {
3522 struct index_entry *idxp;
3523
3524 #ifdef HAVE_TC_RAMCACHE
3525 /* Use RAM DB if available for greater speed */
3526 if (tc_stat.ramcache)
3527 idxp = &hdr->indices[i];
3528 else
3529 #endif
3530 {
3531 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3532 != sizeof(struct index_entry))
3533 {
3534 logf("delete_entry(): read error #2");
3535 goto cleanup;
3536 }
3537 idxp = &idx;
3538 }
3539
3540 if (idxp->flag & FLAG_DELETED)
3541 continue;
3542
3543 for (tag = 0; tag < TAG_COUNT; tag++)
3544 {
3545 if (tagcache_is_numeric_tag(tag))
3546 continue;
3547
3548 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3549 in_use[tag]++;
3550 }
3551 }
3552
3553 /* Now delete all tags no longer in use. */
3554 for (tag = 0; tag < TAG_COUNT; tag++)
3555 {
3556 struct tagcache_header tch;
3557 int oldseek = myidx.tag_seek[tag];
3558
3559 if (tagcache_is_numeric_tag(tag))
3560 continue;
3561
3562 /**
3563 * Replace tag seek with a hash value of the field string data.
3564 * That way runtime statistics of moved or altered files can be
3565 * resurrected.
3566 */
3567 #ifdef HAVE_TC_RAMCACHE
3568 if (tc_stat.ramcache && tag != tag_filename)
3569 {
3570 struct tagfile_entry *tfe;
3571 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3572
3573 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3574 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3575 myidx.tag_seek[tag] = *seek;
3576 }
3577 else
3578 #endif
3579 {
3580 struct tagfile_entry tfe;
3581
3582 /* Open the index file, which contains the tag names. */
3583 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3584 goto cleanup;
3585
3586 /* Skip the header block */
3587 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3588 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3589 != sizeof(struct tagfile_entry))
3590 {
3591 logf("delete_entry(): read error #3");
3592 goto cleanup;
3593 }
3594
3595 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3596 {
3597 logf("delete_entry(): read error #4");
3598 goto cleanup;
3599 }
3600
3601 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3602 }
3603
3604 if (in_use[tag])
3605 {
3606 logf("in use: %d/%d", tag, in_use[tag]);
3607 if (fd >= 0)
3608 {
3609 close(fd);
3610 fd = -1;
3611 }
3612 continue;
3613 }
3614
3615 #ifdef HAVE_TC_RAMCACHE
3616 /* Delete from ram. */
3617 if (tc_stat.ramcache && tag != tag_filename)
3618 {
3619 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3620 tagentry->tag_data[0] = '\0';
3621 }
3622 #endif
3623
3624 /* Open the index file, which contains the tag names. */
3625 if (fd < 0)
3626 {
3627 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3628 goto cleanup;
3629 }
3630
3631 /* Skip the header block */
3632 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3633
3634 /* Debug, print 10 first characters of the tag
3635 read(fd, buf, 10);
3636 buf[10]='\0';
3637 logf("TAG:%s", buf);
3638 lseek(fd, -10, SEEK_CUR);
3639 */
3640
3641 /* Write first data byte in tag as \0 */
3642 write(fd, "", 1);
3643
3644 /* Now tag data has been removed */
3645 close(fd);
3646 fd = -1;
3647 }
3648
3649 /* Write index entry back into master index. */
3650 lseek(masterfd, sizeof(struct master_header) +
3651 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3652 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3653 != sizeof(struct index_entry))
3654 {
3655 logf("delete_entry(): write_error #2");
3656 goto cleanup;
3657 }
3658
3659 close(masterfd);
3660
3661 return true;
3662
3663 cleanup:
3664 if (fd >= 0)
3665 close(fd);
3666 if (masterfd >= 0)
3667 close(masterfd);
3668
3669 return false;
3670 }
3671
3672 #ifndef __PCTOOL__
3673 /**
3674 * Returns true if there is an event waiting in the queue
3675 * that requires the current operation to be aborted.
3676 */
3677 static bool check_event_queue(void)
3678 {
3679 struct queue_event ev;
3680
3681 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3682 switch (ev.id)
3683 {
3684 case Q_STOP_SCAN:
3685 case SYS_POWEROFF:
3686 case SYS_USB_CONNECTED:
3687 /* Put the event back into the queue. */
3688 queue_post(&tagcache_queue, ev.id, ev.data);
3689 return true;
3690 }
3691
3692 return false;
3693 }
3694 #endif
3695
3696 #ifdef HAVE_TC_RAMCACHE
3697 static bool allocate_tagcache(void)
3698 {
3699 struct master_header tcmh;
3700 int fd;
3701
3702 /* Load the header. */
3703 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3704 {
3705 hdr = NULL;
3706 return false;
3707 }
3708
3709 close(fd);
3710
3711 /**
3712 * Now calculate the required cache size plus
3713 * some extra space for alignment fixes.
3714 */
3715 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3716 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3717 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3718 memset(hdr, 0, sizeof(struct ramcache_header));
3719 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3720 hdr->indices = (struct index_entry *)(hdr + 1);
3721 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3722
3723 return true;
3724 }
3725
3726 # ifdef HAVE_EEPROM_SETTINGS
3727 static bool tagcache_dumpload(void)
3728 {
3729 struct statefile_header shdr;
3730 int fd, rc;
3731 long offpos;
3732 int i;
3733
3734 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3735 if (fd < 0)
3736 {
3737 logf("no tagcache statedump");
3738 return false;
3739 }
3740
3741 /* Check the statefile memory placement */
3742 hdr = buffer_alloc(0);
3743 rc = read(fd, &shdr, sizeof(struct statefile_header));
3744 if (rc != sizeof(struct statefile_header)
3745 /* || (long)hdr != (long)shdr.hdr */)
3746 {
3747 logf("incorrect statefile");
3748 hdr = NULL;
3749 close(fd);
3750 return false;
3751 }
3752
3753 offpos = (long)hdr - (long)shdr.hdr;
3754
3755 /* Lets allocate real memory and load it */
3756 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3757 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3758 close(fd);
3759
3760 if (rc != shdr.tc_stat.ramcache_allocated)
3761 {
3762 logf("read failure!");
3763 hdr = NULL;
3764 return false;
3765 }
3766
3767 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3768
3769 /* Now fix the pointers */
3770 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3771 for (i = 0; i < TAG_COUNT; i++)
3772 hdr->tags[i] += offpos;
3773
3774 return true;
3775 }
3776
3777 static bool tagcache_dumpsave(void)
3778 {
3779 struct statefile_header shdr;
3780 int fd;
3781
3782 if (!tc_stat.ramcache)
3783 return false;
3784
3785 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3786 if (fd < 0)
3787 {
3788 logf("failed to create a statedump");
3789 return false;
3790 }
3791
3792 /* Create the header */
3793 shdr.hdr = hdr;
3794 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3795 write(fd, &shdr, sizeof(struct statefile_header));
3796
3797 /* And dump the data too */
3798 write(fd, hdr, tc_stat.ramcache_allocated);
3799 close(fd);
3800
3801 return true;
3802 }
3803 # endif
3804
3805 static bool load_tagcache(void)
3806 {
3807 struct tagcache_header *tch;
3808 long bytesleft = tc_stat.ramcache_allocated;
3809 struct index_entry *idx;
3810 int rc, fd;
3811 char *p;
3812 int i, tag;
3813
3814 # ifdef HAVE_DIRCACHE
3815 while (dircache_is_initializing())
3816 sleep(1);
3817
3818 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3819 # endif
3820
3821 logf("loading tagcache to ram...");
3822
3823 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3824 if (fd < 0)
3825 {
3826 logf("tagcache open failed");
3827 return false;
3828 }
3829
3830 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3831 != sizeof(struct master_header)
3832 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3833 {
3834 logf("incorrect header");
3835 return false;
3836 }
3837
3838 idx = hdr->indices;
3839
3840 /* Load the master index table. */
3841 for (i = 0; i < hdr->h.tch.entry_count; i++)
3842 {
3843 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3844 if (rc != sizeof(struct index_entry))
3845 {
3846 logf("read error #10");
3847 close(fd);
3848 return false;
3849 }
3850
3851 bytesleft -= sizeof(struct index_entry);
3852 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3853 {
3854 logf("too big tagcache.");
3855 close(fd);
3856 return false;
3857 }
3858
3859 idx++;
3860 }
3861
3862 close(fd);
3863
3864 /* Load the tags. */
3865 p = (char *)idx;
3866 for (tag = 0; tag < TAG_COUNT; tag++)
3867 {
3868 struct tagfile_entry *fe;
3869 char buf[TAG_MAXLEN+32];
3870
3871 if (tagcache_is_numeric_tag(tag))
3872 continue ;
3873
3874 //p = ((void *)p+1);
3875 p = (char *)((long)p & ~0x03) + 0x04;
3876 hdr->tags[tag] = p;
3877
3878 /* Check the header. */
3879 tch = (struct tagcache_header *)p;
3880 p += sizeof(struct tagcache_header);
3881
3882 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3883 return false;
3884
3885 for (hdr->entry_count[tag] = 0;
3886 hdr->entry_count[tag] < tch->entry_count;
3887 hdr->entry_count[tag]++)
3888 {
3889 long pos;
3890
3891 if (do_timed_yield())
3892 {
3893 /* Abort if we got a critical event in queue */
3894 if (check_event_queue())
3895 return false;
3896 }
3897
3898 fe = (struct tagfile_entry *)p;
3899 pos = lseek(fd, 0, SEEK_CUR);
3900 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3901 if (rc != sizeof(struct tagfile_entry))
3902 {
3903 /* End of lookup table. */
3904 logf("read error #11");
3905 close(fd);
3906 return false;
3907 }
3908
3909 /* We have a special handling for the filename tags. */
3910 if (tag == tag_filename)
3911 {
3912 # ifdef HAVE_DIRCACHE
3913 const struct dirent *dc;
3914 # endif
3915
3916 // FIXME: This is wrong!
3917 // idx = &hdr->indices[hdr->entry_count[i]];
3918 idx = &hdr->indices[fe->idx_id];
3919
3920 if (fe->tag_length >= (long)sizeof(buf)-1)
3921 {
3922 read(fd, buf, 10);
3923 buf[10] = '\0';
3924 logf("TAG:%s", buf);
3925 logf("too long filename");
3926 close(fd);
3927 return false;
3928 }
3929
3930 rc = read(fd, buf, fe->tag_length);
3931 if (rc != fe->tag_length)
3932 {
3933 logf("read error #12");
3934 close(fd);
3935 return false;
3936 }
3937
3938 /* Check if the entry has already been removed */
3939 if (idx->flag & FLAG_DELETED)
3940 continue;
3941
3942 /* This flag must not be used yet. */
3943 if (idx->flag & FLAG_DIRCACHE)
3944 {
3945 logf("internal error!");
3946 close(fd);
3947 return false;
3948 }
3949
3950 if (idx->tag_seek[tag] != pos)
3951 {
3952 logf("corrupt data structures!");
3953 close(fd);
3954 return false;
3955 }
3956
3957 # ifdef HAVE_DIRCACHE
3958 if (dircache_is_enabled())
3959 {
3960 dc = dircache_get_entry_ptr(buf);
3961 if (dc == NULL)
3962 {
3963 logf("Entry no longer valid.");
3964 logf("-> %s", buf);
3965 delete_entry(fe->idx_id);
3966 continue ;
3967 }
3968
3969 idx->flag |= FLAG_DIRCACHE;
3970 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3971 idx->tag_seek[tag_filename] = (long)dc;
3972 }
3973 else
3974 # endif
3975 {
3976 /* This will be very slow unless dircache is enabled
3977 or target is flash based, but do it anyway for
3978 consistency. */
3979 /* Check if entry has been removed. */
3980 if (global_settings.tagcache_autoupdate)
3981 {
3982 if (!file_exists(buf))
3983 {
3984 logf("Entry no longer valid.");
3985 logf("-> %s", buf);
3986 delete_entry(fe->idx_id);
3987 continue;
3988 }
3989 }
3990 }
3991
3992 continue ;
3993 }
3994
3995 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3996 if (bytesleft < 0)
3997 {
3998 logf("too big tagcache #2");
3999 logf("tl: %d", fe->tag_length);
4000 logf("bl: %ld", bytesleft);
4001 close(fd);
4002 return false;
4003 }
4004
4005 p = fe->tag_data;
4006 rc = read(fd, fe->tag_data, fe->tag_length);
4007 p += rc;
4008
4009 if (rc != fe->tag_length)
4010 {
4011 logf("read error #13");
4012 logf("rc=0x%04x", rc); // 0x431
4013 logf("len=0x%04x", fe->tag_length); // 0x4000
4014 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4015 logf("tag=0x%02x", tag); // 0x00
4016 close(fd);
4017 return false;
4018 }
4019 }
4020 close(fd);
4021 }
4022
4023 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4024 logf("tagcache loaded into ram!");
4025
4026 return true;
4027 }
4028 #endif /* HAVE_TC_RAMCACHE */
4029
4030 static bool check_deleted_files(void)
4031 {
4032 int fd;
4033 char buf[TAG_MAXLEN+32];
4034 struct tagfile_entry tfe;
4035
4036 logf("reverse scan...");
4037 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4038 fd = open(buf, O_RDONLY);
4039
4040 if (fd < 0)
4041 {
4042 logf("%s open fail", buf);
4043 return false;
4044 }
4045
4046 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4047 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4048 == sizeof(struct tagfile_entry)
4049 #ifndef __PCTOOL__
4050 && !check_event_queue()
4051 #endif
4052 )
4053 {
4054 if (tfe.tag_length >= (long)sizeof(buf)-1)
4055 {
4056 logf("too long tag");
4057 close(fd);
4058 return false;
4059 }
4060
4061 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4062 {
4063 logf("read error #14");
4064 close(fd);
4065 return false;
4066 }
4067
4068 /* Check if the file has already deleted from the db. */
4069 if (*buf == '\0')
4070 continue;
4071
4072 /* Now check if the file exists. */
4073 if (!file_exists(buf))
4074 {
4075 logf("Entry no longer valid.");
4076 logf("-> %s / %d", buf, tfe.tag_length);
4077 delete_entry(tfe.idx_id);
4078 }
4079 }
4080
4081 close(fd);
4082
4083 logf("done");
4084
4085 return true;
4086 }
4087
4088 static bool check_dir(const char *dirname, int add_files)
4089 {
4090 DIR *dir;
4091 int len;
4092 int success = false;
4093 int ignore, unignore;
4094 char newpath[MAX_PATH];
4095
4096 dir = opendir(dirname);
4097 if (!dir)
4098 {
4099 logf("tagcache: opendir() failed");
4100 return false;
4101 }
4102
4103 /* check for a database.ignore file */
4104 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4105 ignore = file_exists(newpath);
4106 /* check for a database.unignore file */
4107 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4108 unignore = file_exists(newpath);
4109
4110 /* don't do anything if both ignore and unignore are there */
4111 if (ignore != unignore)
4112 add_files = unignore;
4113
4114 /* Recursively scan the dir. */
4115 #ifdef __PCTOOL__
4116 while (1)
4117 #else
4118 while (!check_event_queue())
4119 #endif
4120 {
4121 struct dirent *entry;
4122
4123 entry = readdir(dir);
4124
4125 if (entry == NULL)
4126 {
4127 success = true;
4128 break ;
4129 }
4130
4131 if (!strcmp((char *)entry->d_name, ".") ||
4132 !strcmp((char *)entry->d_name, ".."))
4133 continue;
4134
4135 yield();
4136
4137 len = strlen(curpath);
4138 snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4139 entry->d_name);
4140
4141 processed_dir_count++;
4142 if (entry->attribute & ATTR_DIRECTORY)
4143 check_dir(curpath, add_files);
4144 else if (add_files)
4145 {
4146 tc_stat.curentry = curpath;
4147
4148 /* Add a new entry to the temporary db file. */
4149 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4150 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4151 , dir->internal_entry
4152 #endif
4153 );
4154
4155 /* Wait until current path for debug screen is read and unset. */
4156 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4157 yield();
4158
4159 tc_stat.curentry = NULL;
4160 }
4161
4162 curpath[len] = '\0';
4163 }
4164
4165 closedir(dir);
4166
4167 return success;
4168 }
4169
4170 void tagcache_screensync_event(void)
4171 {
4172 tc_stat.curentry = NULL;
4173 }
4174
4175 void tagcache_screensync_enable(bool state)
4176 {
4177 tc_stat.syncscreen = state;
4178 }
4179
4180 void tagcache_build(const char *path)
4181 {
4182 struct tagcache_header header;
4183 bool ret;
4184
4185 curpath[0] = '\0';
4186 data_size = 0;
4187 total_entry_count = 0;
4188 processed_dir_count = 0;
4189
4190 #ifdef HAVE_DIRCACHE
4191 while (dircache_is_initializing())
4192 sleep(1);
4193 #endif
4194
4195 logf("updating tagcache");
4196
4197 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4198 if (cachefd >= 0)
4199 {
4200 logf("skipping, cache already waiting for commit");
4201 close(cachefd);
4202 return ;
4203 }
4204
4205 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4206 if (cachefd < 0)
4207 {
4208 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4209 return ;
4210 }
4211
4212 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4213
4214 cpu_boost(true);
4215
4216 logf("Scanning files...");
4217 /* Scan for new files. */
4218 memset(&header, 0, sizeof(struct tagcache_header));
4219 write(cachefd, &header, sizeof(struct tagcache_header));
4220
4221 if (strcmp("/", path) != 0)
4222 strcpy(curpath, path);
4223 ret = check_dir(path, true);
4224
4225 /* Write the header. */
4226 header.magic = TAGCACHE_MAGIC;
4227 header.datasize = data_size;
4228 header.entry_count = total_entry_count;
4229 lseek(cachefd, 0, SEEK_SET);
4230 write(cachefd, &header, sizeof(struct tagcache_header));
4231 close(cachefd);
4232
4233 if (filenametag_fd >= 0)
4234 {
4235 close(filenametag_fd);
4236 filenametag_fd = -1;
4237 }
4238
4239 if (!ret)
4240 {
4241 logf("Aborted.");
4242 cpu_boost(false);
4243 return ;
4244 }
4245
4246 /* Commit changes to the database. */
4247 #ifdef __PCTOOL__
4248 allocate_tempbuf();
4249 #endif
4250 if (commit())
4251 {
4252 remove(TAGCACHE_FILE_TEMP);
4253 logf("tagcache built!");
4254 }
4255 #ifdef __PCTOOL__
4256 free_tempbuf();
4257 #endif
4258
4259 #ifdef HAVE_TC_RAMCACHE
4260 if (hdr)
4261 {
4262 /* Import runtime statistics if we just initialized the db. */
4263 if (hdr->h.serial == 0)
4264 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4265 }
4266 #endif
4267
4268 cpu_boost(false);
4269 }
4270
4271 #ifdef HAVE_TC_RAMCACHE
4272 static void load_ramcache(void)
4273 {
4274 if (!hdr)
4275 return ;
4276
4277 cpu_boost(true);
4278
4279 /* At first we should load the cache (if exists). */
4280 tc_stat.ramcache = load_tagcache();
4281
4282 if (!tc_stat.ramcache)
4283 {
4284 /* If loading failed, it must indicate some problem with the db
4285 * so disable it entirely to prevent further issues. */
4286 tc_stat.ready = false;
4287 hdr = NULL;
4288 }
4289
4290 cpu_boost(false);
4291 }
4292
4293 void tagcache_unload_ramcache(void)
4294 {
4295 tc_stat.ramcache = false;
4296 /* Just to make sure there is no statefile present. */
4297 // remove(TAGCACHE_STATEFILE);
4298 }
4299 #endif
4300
4301 #ifndef __PCTOOL__
4302 static void tagcache_thread(void)
4303 {
4304 struct queue_event ev;
4305 bool check_done = false;
4306
4307 /* If the previous cache build/update was interrupted, commit
4308 * the changes first in foreground. */
4309 cpu_boost(true);
4310 allocate_tempbuf();
4311 commit();
4312 free_tempbuf();
4313
4314 #ifdef HAVE_TC_RAMCACHE
4315 # ifdef HAVE_EEPROM_SETTINGS
4316 if (firmware_settings.initialized && firmware_settings.disk_clean)
4317 check_done = tagcache_dumpload();
4318
4319 remove(TAGCACHE_STATEFILE);
4320 # endif
4321
4322 /* Allocate space for the tagcache if found on disk. */
4323 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4324 allocate_tagcache();
4325 #endif
4326
4327 cpu_boost(false);
4328 tc_stat.initialized = true;
4329
4330 /* Don't delay bootup with the header check but do it on background. */
4331 sleep(HZ);
4332 tc_stat.ready = check_all_headers();
4333 tc_stat.readyvalid = true;
4334
4335 while (1)
4336 {
4337 run_command_queue(false);
4338
4339 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4340
4341 switch (ev.id)
4342 {
4343 case Q_IMPORT_CHANGELOG:
4344 tagcache_import_changelog();
4345 break;
4346
4347 case Q_REBUILD:
4348 remove_files();
4349 remove(TAGCACHE_FILE_TEMP);
4350 tagcache_build("/");
4351 break;
4352
4353 case Q_UPDATE:
4354 tagcache_build("/");
4355 #ifdef HAVE_TC_RAMCACHE
4356 load_ramcache();
4357 #endif
4358 check_deleted_files();
4359 break ;
4360
4361 case Q_START_SCAN:
4362 check_done = false;
4363 case SYS_TIMEOUT:
4364 if (check_done || !tc_stat.ready)
4365 break ;
4366
4367 #ifdef HAVE_TC_RAMCACHE
4368 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4369 {
4370 load_ramcache();
4371 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4372 tagcache_build("/");
4373 }
4374 else
4375 #endif
4376 if (global_settings.tagcache_autoupdate)
4377 {
4378 tagcache_build("/");
4379
4380 /* This will be very slow unless dircache is enabled
4381 or target is flash based, but do it anyway for
4382 consistency. */
4383 check_deleted_files();
4384 }
4385
4386 logf("tagcache check done");
4387
4388 check_done = true;
4389 break ;
4390
4391 case Q_STOP_SCAN:
4392 break ;
4393
4394 case SYS_POWEROFF:
4395 break ;
4396
4397 #ifndef SIMULATOR
4398 case SYS_USB_CONNECTED:
4399 logf("USB: TagCache");
4400 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4401 usb_wait_for_disconnect(&tagcache_queue);
4402 break ;
4403 #endif
4404 }
4405 }
4406 }
4407
4408 bool tagcache_prepare_shutdown(void)
4409 {
4410 if (tagcache_get_commit_step() > 0)
4411 return false;
4412
4413 tagcache_stop_scan();
4414 while (read_lock || write_lock)
4415 sleep(1);
4416
4417 return true;
4418 }
4419
4420 void tagcache_shutdown(void)
4421 {
4422 /* Flush the command queue. */
4423 run_command_queue(true);
4424
4425 #ifdef HAVE_EEPROM_SETTINGS
4426 if (tc_stat.ramcache)
4427 tagcache_dumpsave();
4428 #endif
4429 }
4430
4431 static int get_progress(void)
4432 {
4433 int total_count = -1;
4434
4435 #ifdef HAVE_DIRCACHE
4436 if (dircache_is_enabled())
4437 {
4438 total_count = dircache_get_entry_count();
4439 }
4440 else
4441 #endif
4442 #ifdef HAVE_TC_RAMCACHE
4443 {
4444 if (hdr && tc_stat.ramcache)
4445 total_count = hdr->h.tch.entry_count;
4446 }
4447 #endif
4448
4449 if (total_count < 0)
4450 return -1;
4451
4452 return processed_dir_count * 100 / total_count;
4453 }
4454
4455 struct tagcache_stat* tagcache_get_stat(void)
4456 {
4457 tc_stat.progress = get_progress();
4458 tc_stat.processed_entries = processed_dir_count;
4459
4460 return &tc_stat;
4461 }
4462
4463 void tagcache_start_scan(void)
4464 {
4465 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4466 }
4467
4468 bool tagcache_update(void)
4469 {
4470 if (!tc_stat.ready)
4471 return false;
4472
4473 queue_post(&tagcache_queue, Q_UPDATE, 0);
4474 return false;
4475 }
4476
4477 bool tagcache_rebuild()
4478 {
4479 queue_post(&tagcache_queue, Q_REBUILD, 0);
4480 return false;
4481 }
4482
4483 void tagcache_stop_scan(void)
4484 {
4485 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4486 }
4487
4488 #ifdef HAVE_TC_RAMCACHE
4489 bool tagcache_is_ramcache(void)
4490 {
4491 return tc_stat.ramcache;
4492 }
4493 #endif
4494
4495 #endif /* !__PCTOOL__ */
4496
4497
4498 void tagcache_init(void)
4499 {
4500 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4501 memset(&current_tcmh, 0, sizeof(struct master_header));
4502 filenametag_fd = -1;
4503 write_lock = read_lock = 0;
4504
4505 #ifndef __PCTOOL__
4506 mutex_init(&command_queue_mutex);
4507 queue_init(&tagcache_queue, true);
4508 create_thread(tagcache_thread, tagcache_stack,
4509 sizeof(tagcache_stack), 0, tagcache_thread_name
4510 IF_PRIO(, PRIORITY_BACKGROUND)
4511 IF_COP(, CPU));
4512 #else
4513 tc_stat.initialized = true;
4514 allocate_tempbuf();
4515 commit();
4516 free_tempbuf();
4517 tc_stat.ready = check_all_headers();
4518 #endif
4519 }
4520
4521 #ifdef __PCTOOL__
4522 void tagcache_reverse_scan(void)
4523 {
4524 logf("Checking for deleted files");
4525 check_deleted_files();
4526 }
4527 #endif
4528
4529 bool tagcache_is_initialized(void)
4530 {
4531 return tc_stat.initialized;
4532 }
4533 bool tagcache_is_usable(void)
4534 {
4535 return tc_stat.initialized && tc_stat.ready;
4536 }
4537 int tagcache_get_commit_step(void)
4538 {
4539 return tc_stat.commit_step;
4540 }
4541 int tagcache_get_max_commit_step(void)
4542 {
4543 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4544 }
4545