?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Added missing header file metadata_parsers.h to metadata parsers.
[kugel-rb.git] / apps / tagcache.c
blob8cf261f93f097a7752d6dae8e00b8da4639f1b97
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "ata_idle_notify.h"
61 #include "thread.h"
62 #include "kernel.h"
63 #include "system.h"
64 #include "logf.h"
65 #include "string.h"
66 #include "usb.h"
67 #include "metadata.h"
68 #include "id3.h"
69 #include "tagcache.h"
70 #include "buffer.h"
71 #include "crc32.h"
72 #include "misc.h"
73 #include "settings.h"
74 #include "dir.h"
75 #include "structec.h"
76 #ifndef __PCTOOL__
77 #include "splash.h"
78 #include "lang.h"
79 #include "eeprom_settings.h"
80 #endif
81
82 #ifdef __PCTOOL__
83 #define yield() do { } while(0)
84 #define sim_sleep(timeout) do { } while(0)
85 #define do_timed_yield() do { } while(0)
86 #endif
87
88 #ifndef __PCTOOL__
89 /* Tag Cache thread. */
90 static struct event_queue tagcache_queue;
91 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
92 static const char tagcache_thread_name[] = "tagcache";
93 #endif
94
95 #define UNTAGGED "<Untagged>"
96
97 /* Previous path when scanning directory tree recursively. */
98 static char curpath[TAG_MAXLEN+32];
99 static long curpath_size = sizeof(curpath);
100
101 /* Used when removing duplicates. */
102 static char *tempbuf; /* Allocated when needed. */
103 static long tempbufidx; /* Current location in buffer. */
104 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
105 static long tempbuf_left; /* Buffer space left. */
106 static long tempbuf_pos;
107
108 /* Tags we want to get sorted (loaded to the tempbuf). */
109 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
110 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
111
112 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
113 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
114 tag_composer, tag_comment, tag_albumartist, tag_grouping };
115
116 /* Numeric tags (we can use these tags with conditional clauses). */
117 static const int numeric_tags[] = { tag_year, tag_discnumber,
118 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
119 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
120 tag_virt_length_min, tag_virt_length_sec,
121 tag_virt_playtime_min, tag_virt_playtime_sec,
122 tag_virt_entryage, tag_virt_autoscore };
123
124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
125 static const char *tags_str[] = { "artist", "album", "genre", "title",
126 "filename", "composer", "comment", "albumartist", "grouping", "year",
127 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
128 "playtime", "lastplayed", "commitid", "mtime" };
129
130 /* Status information of the tagcache. */
131 static struct tagcache_stat tc_stat;
132
133 /* Queue commands. */
134 enum tagcache_queue {
135 Q_STOP_SCAN = 0,
136 Q_START_SCAN,
137 Q_IMPORT_CHANGELOG,
138 Q_UPDATE,
139 Q_REBUILD,
140
141 /* Internal tagcache command queue. */
142 CMD_UPDATE_MASTER_HEADER,
143 CMD_UPDATE_NUMERIC,
144 };
145
146 struct tagcache_command_entry {
147 int32_t command;
148 int32_t idx_id;
149 int32_t tag;
150 int32_t data;
151 };
152
153 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
154 static volatile int command_queue_widx = 0;
155 static volatile int command_queue_ridx = 0;
156 static struct mutex command_queue_mutex;
157
158 /* Tag database structures. */
159
160 /* Variable-length tag entry in tag files. */
161 struct tagfile_entry {
162 short tag_length; /* Length of the data in bytes including '\0' */
163 short idx_id; /* Corresponding entry location in index file of not unique tags */
164 char tag_data[0]; /* Begin of the tag data */
165 };
166
167 /* Fixed-size tag entry in master db index. */
168 struct index_entry {
169 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
170 int32_t flag; /* Status flags */
171 };
172
173 /* Header is the same in every file. */
174 struct tagcache_header {
175 int32_t magic; /* Header version number */
176 int32_t datasize; /* Data size in bytes */
177 int32_t entry_count; /* Number of entries in this file */
178 };
179
180 struct master_header {
181 struct tagcache_header tch;
182 int32_t serial; /* Increasing counting number */
183 int32_t commitid; /* Number of commits so far */
184 int32_t dirty;
185 };
186
187 /* For the endianess correction */
188 static const char *tagfile_entry_ec = "ss";
189 static const char *index_entry_ec = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
190 static const char *tagcache_header_ec = "lll";
191 static const char *master_header_ec = "llllll";
192
193 static struct master_header current_tcmh;
194
195 #ifdef HAVE_TC_RAMCACHE
196 /* Header is created when loading database to ram. */
197 struct ramcache_header {
198 struct master_header h; /* Header from the master index */
199 struct index_entry *indices; /* Master index file content */
200 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
201 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
202 };
203
204 # ifdef HAVE_EEPROM_SETTINGS
205 struct statefile_header {
206 struct ramcache_header *hdr;
207 struct tagcache_stat tc_stat;
208 };
209 # endif
210
211 /* Pointer to allocated ramcache_header */
212 static struct ramcache_header *hdr;
213 #endif
214
215 /**
216 * Full tag entries stored in a temporary file waiting
217 * for commit to the cache. */
218 struct temp_file_entry {
219 long tag_offset[TAG_COUNT];
220 short tag_length[TAG_COUNT];
221 long flag;
222
223 long data_length;
224 };
225
226 struct tempbuf_id_list {
227 long id;
228 struct tempbuf_id_list *next;
229 };
230
231 struct tempbuf_searchidx {
232 long idx_id;
233 char *str;
234 int seek;
235 struct tempbuf_id_list idlist;
236 };
237
238 /* Lookup buffer for fixing messed up index while after sorting. */
239 static long commit_entry_count;
240 static long lookup_buffer_depth;
241 static struct tempbuf_searchidx **lookup;
242
243 /* Used when building the temporary file. */
244 static int cachefd = -1, filenametag_fd;
245 static int total_entry_count = 0;
246 static int data_size = 0;
247 static int processed_dir_count;
248
249 /* Thread safe locking */
250 static volatile int write_lock;
251 static volatile int read_lock;
252
253 static bool delete_entry(long idx_id);
254
255 const char* tagcache_tag_to_str(int tag)
256 {
257 return tags_str[tag];
258 }
259
260 bool tagcache_is_numeric_tag(int type)
261 {
262 int i;
263
264 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
265 {
266 if (type == numeric_tags[i])
267 return true;
268 }
269
270 return false;
271 }
272
273 bool tagcache_is_unique_tag(int type)
274 {
275 int i;
276
277 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
278 {
279 if (type == unique_tags[i])
280 return true;
281 }
282
283 return false;
284 }
285
286 bool tagcache_is_sorted_tag(int type)
287 {
288 int i;
289
290 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
291 {
292 if (type == sorted_tags[i])
293 return true;
294 }
295
296 return false;
297 }
298
299 #ifdef HAVE_DIRCACHE
300 /**
301 * Returns true if specified flag is still present, i.e., dircache
302 * has not been reloaded.
303 */
304 static bool is_dircache_intact(void)
305 {
306 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
307 }
308 #endif
309
310 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
311 {
312 int fd;
313 char buf[MAX_PATH];
314 int rc;
315
316 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
317 return -1;
318
319 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
320
321 fd = open(buf, write ? O_RDWR : O_RDONLY);
322 if (fd < 0)
323 {
324 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
325 tc_stat.ready = false;
326 return fd;
327 }
328
329 /* Check the header. */
330 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
331 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
332 {
333 logf("header error");
334 tc_stat.ready = false;
335 close(fd);
336 return -2;
337 }
338
339 return fd;
340 }
341
342 static int open_master_fd(struct master_header *hdr, bool write)
343 {
344 int fd;
345 int rc;
346
347 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
348 if (fd < 0)
349 {
350 logf("master file open failed for R/W");
351 tc_stat.ready = false;
352 return fd;
353 }
354
355 tc_stat.econ = false;
356
357 /* Check the header. */
358 rc = read(fd, hdr, sizeof(struct master_header));
359 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
360 {
361 /* Success. */
362 return fd;
363 }
364
365 /* Trying to read again, this time with endianess correction enabled. */
366 lseek(fd, 0, SEEK_SET);
367
368 rc = ecread(fd, hdr, 1, master_header_ec, true);
369 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
370 {
371 logf("header error");
372 tc_stat.ready = false;
373 close(fd);
374 return -2;
375 }
376
377 tc_stat.econ = true;
378
379 return fd;
380 }
381
382 #ifndef __PCTOOL__
383 static bool do_timed_yield(void)
384 {
385 /* Sorting can lock up for quite a while, so yield occasionally */
386 static long wakeup_tick = 0;
387 if (current_tick >= wakeup_tick)
388 {
389 wakeup_tick = current_tick + (HZ/4);
390 yield();
391 return true;
392 }
393 return false;
394 }
395 #endif
396
397 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
398 static long find_entry_ram(const char *filename,
399 const struct dirent *dc)
400 {
401 static long last_pos = 0;
402 int i;
403
404 /* Check if we tagcache is loaded into ram. */
405 if (!tc_stat.ramcache)
406 return -1;
407
408 if (dc == NULL)
409 dc = dircache_get_entry_ptr(filename);
410
411 if (dc == NULL)
412 {
413 logf("tagcache: file not found.");
414 return -1;
415 }
416
417 try_again:
418
419 if (last_pos > 0)
420 i = last_pos;
421 else
422 i = 0;
423
424 for (; i < hdr->h.tch.entry_count; i++)
425 {
426 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
427 {
428 last_pos = MAX(0, i - 3);
429 return i;
430 }
431
432 do_timed_yield();
433 }
434
435 if (last_pos > 0)
436 {
437 last_pos = 0;
438 goto try_again;
439 }
440
441 return -1;
442 }
443 #endif
444
445 static long find_entry_disk(const char *filename)
446 {
447 struct tagcache_header tch;
448 static long last_pos = -1;
449 long pos_history[POS_HISTORY_COUNT];
450 long pos_history_idx = 0;
451 bool found = false;
452 struct tagfile_entry tfe;
453 int fd;
454 char buf[TAG_MAXLEN+32];
455 int i;
456 int pos = -1;
457
458 if (!tc_stat.ready)
459 return -2;
460
461 fd = filenametag_fd;
462 if (fd < 0)
463 {
464 last_pos = -1;
465 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
466 return -1;
467 }
468
469 check_again:
470
471 if (last_pos > 0)
472 lseek(fd, last_pos, SEEK_SET);
473 else
474 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
475
476 while (true)
477 {
478 pos = lseek(fd, 0, SEEK_CUR);
479 for (i = pos_history_idx-1; i >= 0; i--)
480 pos_history[i+1] = pos_history[i];
481 pos_history[0] = pos;
482
483 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
484 != sizeof(struct tagfile_entry))
485 {
486 break ;
487 }
488
489 if (tfe.tag_length >= (long)sizeof(buf))
490 {
491 logf("too long tag #1");
492 close(fd);
493 last_pos = -1;
494 return -2;
495 }
496
497 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
498 {
499 logf("read error #2");
500 close(fd);
501 last_pos = -1;
502 return -3;
503 }
504
505 if (!strcasecmp(filename, buf))
506 {
507 last_pos = pos_history[pos_history_idx];
508 found = true;
509 break ;
510 }
511
512 if (pos_history_idx < POS_HISTORY_COUNT - 1)
513 pos_history_idx++;
514 }
515
516 /* Not found? */
517 if (!found)
518 {
519 if (last_pos > 0)
520 {
521 last_pos = -1;
522 logf("seek again");
523 goto check_again;
524 }
525
526 if (fd != filenametag_fd)
527 close(fd);
528 return -4;
529 }
530
531 if (fd != filenametag_fd)
532 close(fd);
533
534 return tfe.idx_id;
535 }
536
537 static int find_index(const char *filename)
538 {
539 long idx_id = -1;
540
541 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
542 if (tc_stat.ramcache && is_dircache_intact())
543 idx_id = find_entry_ram(filename, NULL);
544 #endif
545
546 if (idx_id < 0)
547 idx_id = find_entry_disk(filename);
548
549 return idx_id;
550 }
551
552 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
553 {
554 int idx_id;
555
556 if (!tc_stat.ready)
557 return false;
558
559 idx_id = find_index(filename);
560 if (idx_id < 0)
561 return false;
562
563 if (!tagcache_search(tcs, tag_filename))
564 return false;
565
566 tcs->entry_count = 0;
567 tcs->idx_id = idx_id;
568
569 return true;
570 }
571
572 static bool get_index(int masterfd, int idxid,
573 struct index_entry *idx, bool use_ram)
574 {
575 bool localfd = false;
576
577 if (idxid < 0)
578 {
579 logf("Incorrect idxid: %d", idxid);
580 return false;
581 }
582
583 #ifdef HAVE_TC_RAMCACHE
584 if (tc_stat.ramcache && use_ram)
585 {
586 if (hdr->indices[idxid].flag & FLAG_DELETED)
587 return false;
588
589 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
590 return true;
591 }
592 #else
593 (void)use_ram;
594 #endif
595
596 if (masterfd < 0)
597 {
598 struct master_header tcmh;
599
600 localfd = true;
601 masterfd = open_master_fd(&tcmh, false);
602 if (masterfd < 0)
603 return false;
604 }
605
606 lseek(masterfd, idxid * sizeof(struct index_entry)
607 + sizeof(struct master_header), SEEK_SET);
608 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
609 != sizeof(struct index_entry))
610 {
611 logf("read error #3");
612 if (localfd)
613 close(masterfd);
614
615 return false;
616 }
617
618 if (localfd)
619 close(masterfd);
620
621 if (idx->flag & FLAG_DELETED)
622 return false;
623
624 return true;
625 }
626
627 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
628 {
629 /* We need to exclude all memory only flags & tags when writing to disk. */
630 if (idx->flag & FLAG_DIRCACHE)
631 {
632 logf("memory only flags!");
633 return false;
634 }
635
636 #ifdef HAVE_TC_RAMCACHE
637 /* Only update numeric data. Writing the whole index to RAM by memcpy
638 * destroys dircache pointers!
639 */
640 if (tc_stat.ramcache)
641 {
642 int tag;
643 struct index_entry *idx_ram = &hdr->indices[idxid];
644
645 for (tag = 0; tag < TAG_COUNT; tag++)
646 {
647 if (tagcache_is_numeric_tag(tag))
648 {
649 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
650 }
651 }
652
653 /* Don't touch the dircache flag. */
654 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
655 }
656 #endif
657
658 lseek(masterfd, idxid * sizeof(struct index_entry)
659 + sizeof(struct master_header), SEEK_SET);
660 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
661 != sizeof(struct index_entry))
662 {
663 logf("write error #3");
664 logf("idxid: %d", idxid);
665 return false;
666 }
667
668 return true;
669 }
670
671 static bool open_files(struct tagcache_search *tcs, int tag)
672 {
673 if (tcs->idxfd[tag] < 0)
674 {
675 char fn[MAX_PATH];
676
677 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
678 tcs->idxfd[tag] = open(fn, O_RDONLY);
679 }
680
681 if (tcs->idxfd[tag] < 0)
682 {
683 logf("File not open!");
684 return false;
685 }
686
687 return true;
688 }
689
690 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
691 int tag, char *buf, long size)
692 {
693 struct tagfile_entry tfe;
694 long seek;
695
696 *buf = '\0';
697
698 if (tagcache_is_numeric_tag(tag))
699 return false;
700
701 seek = idx->tag_seek[tag];
702 if (seek < 0)
703 {
704 logf("Retrieve failed");
705 return false;
706 }
707
708 #ifdef HAVE_TC_RAMCACHE
709 if (tcs->ramsearch)
710 {
711 struct tagfile_entry *ep;
712
713 # ifdef HAVE_DIRCACHE
714 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
715 && is_dircache_intact())
716 {
717 dircache_copy_path((struct dirent *)seek,
718 buf, size);
719 return true;
720 }
721 else
722 # endif
723 if (tag != tag_filename)
724 {
725 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
726 strncpy(buf, ep->tag_data, size-1);
727
728 return true;
729 }
730 }
731 #endif
732
733 if (!open_files(tcs, tag))
734 return false;
735
736 lseek(tcs->idxfd[tag], seek, SEEK_SET);
737 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
738 != sizeof(struct tagfile_entry))
739 {
740 logf("read error #5");
741 return false;
742 }
743
744 if (tfe.tag_length >= size)
745 {
746 logf("too small buffer");
747 return false;
748 }
749
750 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
751 tfe.tag_length)
752 {
753 logf("read error #6");
754 return false;
755 }
756
757 buf[tfe.tag_length] = '\0';
758
759 return true;
760 }
761
762 static long check_virtual_tags(int tag, const struct index_entry *idx)
763 {
764 long data = 0;
765
766 switch (tag)
767 {
768 case tag_virt_length_sec:
769 data = (idx->tag_seek[tag_length]/1000) % 60;
770 break;
771
772 case tag_virt_length_min:
773 data = (idx->tag_seek[tag_length]/1000) / 60;
774 break;
775
776 case tag_virt_playtime_sec:
777 data = (idx->tag_seek[tag_playtime]/1000) % 60;
778 break;
779
780 case tag_virt_playtime_min:
781 data = (idx->tag_seek[tag_playtime]/1000) / 60;
782 break;
783
784 case tag_virt_autoscore:
785 if (idx->tag_seek[tag_length] == 0
786 || idx->tag_seek[tag_playcount] == 0)
787 {
788 data = 0;
789 }
790 else
791 {
792 data = 100 * idx->tag_seek[tag_playtime]
793 / idx->tag_seek[tag_length]
794 / idx->tag_seek[tag_playcount];
795 }
796 break;
797
798 /* How many commits before the file has been added to the DB. */
799 case tag_virt_entryage:
800 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
801 break;
802
803 default:
804 data = idx->tag_seek[tag];
805 }
806
807 return data;
808 }
809
810 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
811 {
812 struct index_entry idx;
813
814 if (!tc_stat.ready)
815 return false;
816
817 if (!tagcache_is_numeric_tag(tag))
818 return -1;
819
820 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
821 return -2;
822
823 return check_virtual_tags(tag, &idx);
824 }
825
826 inline static bool str_ends_with(const char *str1, const char *str2)
827 {
828 int str_len = strlen(str1);
829 int clause_len = strlen(str2);
830
831 if (clause_len > str_len)
832 return false;
833
834 return !strcasecmp(&str1[str_len - clause_len], str2);
835 }
836
837 inline static bool str_oneof(const char *str, const char *list)
838 {
839 const char *sep;
840 int l, len = strlen(str);
841
842 while (*list)
843 {
844 sep = strchr(list, '|');
845 l = sep ? (long)sep - (long)list : (int)strlen(list);
846 if ((l==len) && !strncasecmp(str, list, len))
847 return true;
848 list += sep ? l + 1 : l;
849 }
850
851 return false;
852 }
853
854 static bool check_against_clause(long numeric, const char *str,
855 const struct tagcache_search_clause *clause)
856 {
857 if (clause->numeric)
858 {
859 switch (clause->type)
860 {
861 case clause_is:
862 return numeric == clause->numeric_data;
863 case clause_is_not:
864 return numeric != clause->numeric_data;
865 case clause_gt:
866 return numeric > clause->numeric_data;
867 case clause_gteq:
868 return numeric >= clause->numeric_data;
869 case clause_lt:
870 return numeric < clause->numeric_data;
871 case clause_lteq:
872 return numeric <= clause->numeric_data;
873 default:
874 logf("Incorrect numeric tag: %d", clause->type);
875 }
876 }
877 else
878 {
879 switch (clause->type)
880 {
881 case clause_is:
882 return !strcasecmp(clause->str, str);
883 case clause_is_not:
884 return strcasecmp(clause->str, str);
885 case clause_gt:
886 return 0>strcasecmp(clause->str, str);
887 case clause_gteq:
888 return 0>=strcasecmp(clause->str, str);
889 case clause_lt:
890 return 0<strcasecmp(clause->str, str);
891 case clause_lteq:
892 return 0<=strcasecmp(clause->str, str);
893 case clause_contains:
894 return (strcasestr(str, clause->str) != NULL);
895 case clause_not_contains:
896 return (strcasestr(str, clause->str) == NULL);
897 case clause_begins_with:
898 return (strcasestr(str, clause->str) == str);
899 case clause_not_begins_with:
900 return (strcasestr(str, clause->str) != str);
901 case clause_ends_with:
902 return str_ends_with(str, clause->str);
903 case clause_not_ends_with:
904 return !str_ends_with(str, clause->str);
905 case clause_oneof:
906 return str_oneof(str, clause->str);
907
908 default:
909 logf("Incorrect tag: %d", clause->type);
910 }
911 }
912
913 return false;
914 }
915
916 static bool check_clauses(struct tagcache_search *tcs,
917 struct index_entry *idx,
918 struct tagcache_search_clause **clause, int count)
919 {
920 int i;
921
922 #ifdef HAVE_TC_RAMCACHE
923 if (tcs->ramsearch)
924 {
925 /* Go through all conditional clauses. */
926 for (i = 0; i < count; i++)
927 {
928 struct tagfile_entry *tfe;
929 int seek;
930 char buf[256];
931 char *str = NULL;
932
933 seek = check_virtual_tags(clause[i]->tag, idx);
934
935 if (!tagcache_is_numeric_tag(clause[i]->tag))
936 {
937 if (clause[i]->tag == tag_filename)
938 {
939 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
940 str = buf;
941 }
942 else
943 {
944 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
945 str = tfe->tag_data;
946 }
947 }
948
949 if (!check_against_clause(seek, str, clause[i]))
950 return false;
951 }
952 }
953 else
954 #endif
955 {
956 /* Check for conditions. */
957 for (i = 0; i < count; i++)
958 {
959 struct tagfile_entry tfe;
960 int seek;
961 char str[256];
962
963 seek = check_virtual_tags(clause[i]->tag, idx);
964
965 memset(str, 0, sizeof str);
966 if (!tagcache_is_numeric_tag(clause[i]->tag))
967 {
968 int fd = tcs->idxfd[clause[i]->tag];
969 lseek(fd, seek, SEEK_SET);
970 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
971 if (tfe.tag_length >= (int)sizeof(str))
972 {
973 logf("Too long tag read!");
974 break ;
975 }
976
977 read(fd, str, tfe.tag_length);
978
979 /* Check if entry has been deleted. */
980 if (str[0] == '\0')
981 break;
982 }
983
984 if (!check_against_clause(seek, str, clause[i]))
985 return false;
986 }
987 }
988
989 return true;
990 }
991
992 bool tagcache_check_clauses(struct tagcache_search *tcs,
993 struct tagcache_search_clause **clause, int count)
994 {
995 struct index_entry idx;
996
997 if (count == 0)
998 return true;
999
1000 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1001 return false;
1002
1003 return check_clauses(tcs, &idx, clause, count);
1004 }
1005
1006 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1007 {
1008 int i;
1009
1010 /* If uniq buffer is not defined we must return true for search to work. */
1011 if (tcs->unique_list == NULL
1012 || (!tagcache_is_unique_tag(tcs->type)
1013 && !tagcache_is_numeric_tag(tcs->type)))
1014 {
1015 return true;
1016 }
1017
1018 for (i = 0; i < tcs->unique_list_count; i++)
1019 {
1020 /* Return false if entry is found. */
1021 if (tcs->unique_list[i] == id)
1022 return false;
1023 }
1024
1025 if (tcs->unique_list_count < tcs->unique_list_capacity)
1026 {
1027 tcs->unique_list[i] = id;
1028 tcs->unique_list_count++;
1029 }
1030
1031 return true;
1032 }
1033
1034 static bool build_lookup_list(struct tagcache_search *tcs)
1035 {
1036 struct index_entry entry;
1037 int i;
1038
1039 tcs->seek_list_count = 0;
1040
1041 #ifdef HAVE_TC_RAMCACHE
1042 if (tcs->ramsearch)
1043 {
1044 int j;
1045
1046 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1047 {
1048 struct index_entry *idx = &hdr->indices[i];
1049 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1050 break ;
1051
1052 /* Skip deleted files. */
1053 if (idx->flag & FLAG_DELETED)
1054 continue;
1055
1056 /* Go through all filters.. */
1057 for (j = 0; j < tcs->filter_count; j++)
1058 {
1059 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1060 {
1061 break ;
1062 }
1063 }
1064
1065 if (j < tcs->filter_count)
1066 continue ;
1067
1068 /* Check for conditions. */
1069 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1070 continue;
1071
1072 /* Add to the seek list if not already in uniq buffer. */
1073 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1074 continue;
1075
1076 /* Lets add it. */
1077 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1078 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1079 tcs->seek_list_count++;
1080 }
1081
1082 tcs->seek_pos = i;
1083
1084 return tcs->seek_list_count > 0;
1085 }
1086 #endif
1087
1088 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1089 sizeof(struct master_header), SEEK_SET);
1090
1091 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1092 == sizeof(struct index_entry))
1093 {
1094 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1095 break ;
1096
1097 tcs->seek_pos++;
1098
1099 /* Check if entry has been deleted. */
1100 if (entry.flag & FLAG_DELETED)
1101 continue;
1102
1103 /* Go through all filters.. */
1104 for (i = 0; i < tcs->filter_count; i++)
1105 {
1106 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1107 break ;
1108 }
1109
1110 if (i < tcs->filter_count)
1111 continue ;
1112
1113 /* Check for conditions. */
1114 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1115 continue;
1116
1117 /* Add to the seek list if not already in uniq buffer. */
1118 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1119 continue;
1120
1121 /* Lets add it. */
1122 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1123 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1124 tcs->seek_list_count++;
1125
1126 yield();
1127 }
1128
1129 return tcs->seek_list_count > 0;
1130 }
1131
1132
1133 static void remove_files(void)
1134 {
1135 int i;
1136 char buf[MAX_PATH];
1137
1138 tc_stat.ready = false;
1139 tc_stat.ramcache = false;
1140 tc_stat.econ = false;
1141 remove(TAGCACHE_FILE_MASTER);
1142 for (i = 0; i < TAG_COUNT; i++)
1143 {
1144 if (tagcache_is_numeric_tag(i))
1145 continue;
1146
1147 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1148 remove(buf);
1149 }
1150 }
1151
1152
1153 static bool check_all_headers(void)
1154 {
1155 struct master_header myhdr;
1156 struct tagcache_header tch;
1157 int tag;
1158 int fd;
1159
1160 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1161 return false;
1162
1163 close(fd);
1164 if (myhdr.dirty)
1165 {
1166 logf("tagcache is dirty!");
1167 return false;
1168 }
1169
1170 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1171
1172 for (tag = 0; tag < TAG_COUNT; tag++)
1173 {
1174 if (tagcache_is_numeric_tag(tag))
1175 continue;
1176
1177 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1178 return false;
1179
1180 close(fd);
1181 }
1182
1183 return true;
1184 }
1185
1186 bool tagcache_search(struct tagcache_search *tcs, int tag)
1187 {
1188 struct tagcache_header tag_hdr;
1189 struct master_header master_hdr;
1190 int i;
1191
1192 if (tcs->initialized)
1193 tagcache_search_finish(tcs);
1194
1195 while (read_lock)
1196 sleep(1);
1197
1198 memset(tcs, 0, sizeof(struct tagcache_search));
1199 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1200 return false;
1201
1202 tcs->position = sizeof(struct tagcache_header);
1203 tcs->type = tag;
1204 tcs->seek_pos = 0;
1205 tcs->seek_list_count = 0;
1206 tcs->filter_count = 0;
1207 tcs->masterfd = -1;
1208
1209 for (i = 0; i < TAG_COUNT; i++)
1210 tcs->idxfd[i] = -1;
1211
1212 #ifndef HAVE_TC_RAMCACHE
1213 tcs->ramsearch = false;
1214 #else
1215 tcs->ramsearch = tc_stat.ramcache;
1216 if (tcs->ramsearch)
1217 {
1218 tcs->entry_count = hdr->entry_count[tcs->type];
1219 }
1220 else
1221 #endif
1222 {
1223 if (!tagcache_is_numeric_tag(tcs->type))
1224 {
1225 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1226 if (tcs->idxfd[tcs->type] < 0)
1227 return false;
1228 }
1229
1230 /* Always open as R/W so we can pass tcs to functions that modify data also
1231 * without failing. */
1232 tcs->masterfd = open_master_fd(&master_hdr, true);
1233
1234 if (tcs->masterfd < 0)
1235 return false;
1236 }
1237
1238 tcs->valid = true;
1239 tcs->initialized = true;
1240 write_lock++;
1241
1242 return true;
1243 }
1244
1245 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1246 void *buffer, long length)
1247 {
1248 tcs->unique_list = (unsigned long *)buffer;
1249 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1250 tcs->unique_list_count = 0;
1251 }
1252
1253 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1254 int tag, int seek)
1255 {
1256 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1257 return false;
1258
1259 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1260 return false;
1261
1262 tcs->filter_tag[tcs->filter_count] = tag;
1263 tcs->filter_seek[tcs->filter_count] = seek;
1264 tcs->filter_count++;
1265
1266 return true;
1267 }
1268
1269 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1270 struct tagcache_search_clause *clause)
1271 {
1272 int i;
1273
1274 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1275 {
1276 logf("Too many clauses");
1277 return false;
1278 }
1279
1280 /* Check if there is already a similar filter in present (filters are
1281 * much faster than clauses).
1282 */
1283 for (i = 0; i < tcs->filter_count; i++)
1284 {
1285 if (tcs->filter_tag[i] == clause->tag)
1286 return true;
1287 }
1288
1289 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1290 {
1291 char buf[MAX_PATH];
1292
1293 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1294 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1295 }
1296
1297 tcs->clause[tcs->clause_count] = clause;
1298 tcs->clause_count++;
1299
1300 return true;
1301 }
1302
1303 /* TODO: Remove this mess. */
1304 #ifdef HAVE_DIRCACHE
1305 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1306 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1307 #else
1308 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1309 #endif
1310
1311 static bool get_next(struct tagcache_search *tcs)
1312 {
1313 static char buf[TAG_MAXLEN+32];
1314 struct tagfile_entry entry;
1315 long flag = 0;
1316
1317 if (!tcs->valid || !tc_stat.ready)
1318 return false;
1319
1320 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1321 #ifdef HAVE_TC_RAMCACHE
1322 && !tcs->ramsearch
1323 #endif
1324 )
1325 return false;
1326
1327 /* Relative fetch. */
1328 if (tcs->filter_count > 0 || tcs->clause_count > 0
1329 || tagcache_is_numeric_tag(tcs->type))
1330 {
1331 /* Check for end of list. */
1332 if (tcs->seek_list_count == 0)
1333 {
1334 /* Try to fetch more. */
1335 if (!build_lookup_list(tcs))
1336 {
1337 tcs->valid = false;
1338 return false;
1339 }
1340 }
1341
1342 tcs->seek_list_count--;
1343 flag = tcs->seek_flags[tcs->seek_list_count];
1344
1345 /* Seek stream to the correct position and continue to direct fetch. */
1346 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1347 && !tagcache_is_numeric_tag(tcs->type))
1348 {
1349 if (!open_files(tcs, tcs->type))
1350 return false;
1351
1352 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1353 }
1354 else
1355 tcs->position = tcs->seek_list[tcs->seek_list_count];
1356 }
1357
1358 if (tagcache_is_numeric_tag(tcs->type))
1359 {
1360 snprintf(buf, sizeof(buf), "%d", tcs->position);
1361 tcs->result_seek = tcs->position;
1362 tcs->result = buf;
1363 tcs->result_len = strlen(buf) + 1;
1364 return true;
1365 }
1366
1367 /* Direct fetch. */
1368 #ifdef HAVE_TC_RAMCACHE
1369 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1370 {
1371 struct tagfile_entry *ep;
1372
1373 if (tcs->entry_count == 0)
1374 {
1375 tcs->valid = false;
1376 return false;
1377 }
1378 tcs->entry_count--;
1379
1380 tcs->result_seek = tcs->position;
1381
1382 # ifdef HAVE_DIRCACHE
1383 if (tcs->type == tag_filename)
1384 {
1385 dircache_copy_path((struct dirent *)tcs->position,
1386 buf, sizeof buf);
1387 tcs->result = buf;
1388 tcs->result_len = strlen(buf) + 1;
1389 tcs->idx_id = FLAG_GET_ATTR(flag);
1390 tcs->ramresult = false;
1391
1392 return true;
1393 }
1394 # endif
1395
1396 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1397 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1398 tcs->result = ep->tag_data;
1399 tcs->result_len = strlen(tcs->result) + 1;
1400 tcs->idx_id = ep->idx_id;
1401 tcs->ramresult = true;
1402
1403 return true;
1404 }
1405 else
1406 #endif
1407 {
1408 if (!open_files(tcs, tcs->type))
1409 return false;
1410
1411 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1412 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1413 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1414 {
1415 /* End of data. */
1416 tcs->valid = false;
1417 return false;
1418 }
1419 }
1420
1421 if (entry.tag_length > (long)sizeof(buf))
1422 {
1423 tcs->valid = false;
1424 logf("too long tag #2");
1425 return false;
1426 }
1427
1428 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1429 {
1430 tcs->valid = false;
1431 logf("read error #4");
1432 return false;
1433 }
1434
1435 tcs->result = buf;
1436 tcs->result_len = strlen(tcs->result) + 1;
1437 tcs->idx_id = entry.idx_id;
1438 tcs->ramresult = false;
1439
1440 return true;
1441 }
1442
1443 bool tagcache_get_next(struct tagcache_search *tcs)
1444 {
1445 while (get_next(tcs))
1446 {
1447 if (tcs->result_len > 1)
1448 return true;
1449 }
1450
1451 return false;
1452 }
1453
1454 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1455 int tag, char *buf, long size)
1456 {
1457 struct index_entry idx;
1458
1459 *buf = '\0';
1460 if (!get_index(tcs->masterfd, idxid, &idx, true))
1461 return false;
1462
1463 return retrieve(tcs, &idx, tag, buf, size);
1464 }
1465
1466 static bool update_master_header(void)
1467 {
1468 struct master_header myhdr;
1469 int fd;
1470
1471 if (!tc_stat.ready)
1472 return false;
1473
1474 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1475 return false;
1476
1477 myhdr.serial = current_tcmh.serial;
1478 myhdr.commitid = current_tcmh.commitid;
1479
1480 /* Write it back */
1481 lseek(fd, 0, SEEK_SET);
1482 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1483 close(fd);
1484
1485 #ifdef HAVE_TC_RAMCACHE
1486 if (hdr)
1487 {
1488 hdr->h.serial = current_tcmh.serial;
1489 hdr->h.commitid = current_tcmh.commitid;
1490 }
1491 #endif
1492
1493 return true;
1494 }
1495
1496 #if 0
1497
1498 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1499 {
1500 struct tagentry *entry;
1501
1502 if (tcs->type != tag_title)
1503 return ;
1504
1505 /* We will need reserve buffer for this. */
1506 if (tcs->ramcache)
1507 {
1508 struct tagfile_entry *ep;
1509
1510 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1511 tcs->seek_list[tcs->seek_list_count];
1512 }
1513
1514 entry = find_entry_ram();
1515
1516 }
1517 #endif
1518
1519 void tagcache_search_finish(struct tagcache_search *tcs)
1520 {
1521 int i;
1522
1523 if (!tcs->initialized)
1524 return;
1525
1526 if (tcs->masterfd >= 0)
1527 {
1528 close(tcs->masterfd);
1529 tcs->masterfd = -1;
1530 }
1531
1532 for (i = 0; i < TAG_COUNT; i++)
1533 {
1534 if (tcs->idxfd[i] >= 0)
1535 {
1536 close(tcs->idxfd[i]);
1537 tcs->idxfd[i] = -1;
1538 }
1539 }
1540
1541 tcs->ramsearch = false;
1542 tcs->valid = false;
1543 tcs->initialized = 0;
1544 if (write_lock > 0)
1545 write_lock--;
1546 }
1547
1548 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1549 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1550 {
1551 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1552 }
1553
1554 static long get_tag_numeric(const struct index_entry *entry, int tag)
1555 {
1556 return check_virtual_tags(tag, entry);
1557 }
1558
1559 static char* get_tag_string(const struct index_entry *entry, int tag)
1560 {
1561 char* s = get_tag(entry, tag)->tag_data;
1562 return strcmp(s, UNTAGGED) ? s : NULL;
1563 }
1564
1565 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1566 {
1567 struct index_entry *entry;
1568 int idx_id;
1569
1570 if (!tc_stat.ready)
1571 return false;
1572
1573 /* Find the corresponding entry in tagcache. */
1574 idx_id = find_entry_ram(filename, NULL);
1575 if (idx_id < 0 || !tc_stat.ramcache)
1576 return false;
1577
1578 entry = &hdr->indices[idx_id];
1579
1580 id3->title = get_tag_string(entry, tag_title);
1581 id3->artist = get_tag_string(entry, tag_artist);
1582 id3->album = get_tag_string(entry, tag_album);
1583 id3->genre_string = get_tag_string(entry, tag_genre);
1584 id3->composer = get_tag_string(entry, tag_composer);
1585 id3->comment = get_tag_string(entry, tag_comment);
1586 id3->albumartist = get_tag_string(entry, tag_albumartist);
1587 id3->grouping = get_tag_string(entry, tag_grouping);
1588
1589 id3->playcount = get_tag_numeric(entry, tag_playcount);
1590 id3->rating = get_tag_numeric(entry, tag_rating);
1591 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1592 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1593 id3->year = get_tag_numeric(entry, tag_year);
1594
1595 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1596 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1597 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1598 if (id3->bitrate == 0)
1599 id3->bitrate = 1;
1600
1601 return true;
1602 }
1603 #endif
1604
1605 static inline void write_item(const char *item)
1606 {
1607 int len = strlen(item) + 1;
1608
1609 data_size += len;
1610 write(cachefd, item, len);
1611 }
1612
1613 static int check_if_empty(char **tag)
1614 {
1615 int length;
1616
1617 if (*tag == NULL || **tag == '\0')
1618 {
1619 *tag = UNTAGGED;
1620 return sizeof(UNTAGGED); /* Tag length */
1621 }
1622
1623 length = strlen(*tag);
1624 if (length > TAG_MAXLEN)
1625 {
1626 logf("over length tag: %s", *tag);
1627 length = TAG_MAXLEN;
1628 (*tag)[length] = '\0';
1629 }
1630
1631 return length + 1;
1632 }
1633
1634 #define ADD_TAG(entry,tag,data) \
1635 /* Adding tag */ \
1636 entry.tag_offset[tag] = offset; \
1637 entry.tag_length[tag] = check_if_empty(data); \
1638 offset += entry.tag_length[tag]
1639
1640 static void add_tagcache(char *path, unsigned long mtime
1641 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1642 ,const struct dirent *dc
1643 #endif
1644 )
1645 {
1646 struct mp3entry id3;
1647 struct temp_file_entry entry;
1648 bool ret;
1649 int fd;
1650 int idx_id = -1;
1651 char tracknumfix[3];
1652 int offset = 0;
1653 int path_length = strlen(path);
1654 bool has_albumartist;
1655 bool has_grouping;
1656
1657 if (cachefd < 0)
1658 return ;
1659
1660 /* Check for overlength file path. */
1661 if (path_length > TAG_MAXLEN)
1662 {
1663 /* Path can't be shortened. */
1664 logf("Too long path: %s", path);
1665 return ;
1666 }
1667
1668 /* Check if the file is supported. */
1669 if (probe_file_format(path) == AFMT_UNKNOWN)
1670 return ;
1671
1672 /* Check if the file is already cached. */
1673 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1674 if (tc_stat.ramcache && is_dircache_intact())
1675 {
1676 idx_id = find_entry_ram(path, dc);
1677 }
1678 else
1679 #endif
1680 {
1681 if (filenametag_fd >= 0)
1682 {
1683 idx_id = find_entry_disk(path);
1684 }
1685 }
1686
1687 /* Check if file has been modified. */
1688 if (idx_id >= 0)
1689 {
1690 struct index_entry idx;
1691
1692 if (!get_index(-1, idx_id, &idx, true))
1693 {
1694 logf("failed to retrieve index entry");
1695 return ;
1696 }
1697
1698 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1699 {
1700 /* No changes to file. */
1701 return ;
1702 }
1703
1704 /* Metadata might have been changed. Delete the entry. */
1705 logf("Re-adding: %s", path);
1706 if (!delete_entry(idx_id))
1707 {
1708 logf("delete_entry failed: %d", idx_id);
1709 return ;
1710 }
1711 }
1712
1713 fd = open(path, O_RDONLY);
1714 if (fd < 0)
1715 {
1716 logf("open fail: %s", path);
1717 return ;
1718 }
1719
1720 memset(&id3, 0, sizeof(struct mp3entry));
1721 memset(&entry, 0, sizeof(struct temp_file_entry));
1722 memset(&tracknumfix, 0, sizeof(tracknumfix));
1723 ret = get_metadata(&id3, fd, path);
1724 close(fd);
1725
1726 if (!ret)
1727 return ;
1728
1729 logf("-> %s", path);
1730
1731 /* Generate track number if missing. */
1732 if (id3.tracknum <= 0)
1733 {
1734 const char *p = strrchr(path, '.');
1735
1736 if (p == NULL)
1737 p = &path[strlen(path)-1];
1738
1739 while (*p != '/')
1740 {
1741 if (isdigit(*p) && isdigit(*(p-1)))
1742 {
1743 tracknumfix[1] = *p--;
1744 tracknumfix[0] = *p;
1745 break;
1746 }
1747 p--;
1748 }
1749
1750 if (tracknumfix[0] != '\0')
1751 {
1752 id3.tracknum = atoi(tracknumfix);
1753 /* Set a flag to indicate track number has been generated. */
1754 entry.flag |= FLAG_TRKNUMGEN;
1755 }
1756 else
1757 {
1758 /* Unable to generate track number. */
1759 id3.tracknum = -1;
1760 }
1761 }
1762
1763 /* Numeric tags */
1764 entry.tag_offset[tag_year] = id3.year;
1765 entry.tag_offset[tag_discnumber] = id3.discnum;
1766 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1767 entry.tag_offset[tag_length] = id3.length;
1768 entry.tag_offset[tag_bitrate] = id3.bitrate;
1769 entry.tag_offset[tag_mtime] = mtime;
1770
1771 /* String tags. */
1772 has_albumartist = id3.albumartist != NULL
1773 && strlen(id3.albumartist) > 0;
1774 has_grouping = id3.grouping != NULL
1775 && strlen(id3.grouping) > 0;
1776
1777 ADD_TAG(entry, tag_filename, &path);
1778 ADD_TAG(entry, tag_title, &id3.title);
1779 ADD_TAG(entry, tag_artist, &id3.artist);
1780 ADD_TAG(entry, tag_album, &id3.album);
1781 ADD_TAG(entry, tag_genre, &id3.genre_string);
1782 ADD_TAG(entry, tag_composer, &id3.composer);
1783 ADD_TAG(entry, tag_comment, &id3.comment);
1784 if (has_albumartist)
1785 {
1786 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1787 }
1788 else
1789 {
1790 ADD_TAG(entry, tag_albumartist, &id3.artist);
1791 }
1792 if (has_grouping)
1793 {
1794 ADD_TAG(entry, tag_grouping, &id3.grouping);
1795 }
1796 else
1797 {
1798 ADD_TAG(entry, tag_grouping, &id3.title);
1799 }
1800 entry.data_length = offset;
1801
1802 /* Write the header */
1803 write(cachefd, &entry, sizeof(struct temp_file_entry));
1804
1805 /* And tags also... Correct order is critical */
1806 write_item(path);
1807 write_item(id3.title);
1808 write_item(id3.artist);
1809 write_item(id3.album);
1810 write_item(id3.genre_string);
1811 write_item(id3.composer);
1812 write_item(id3.comment);
1813 if (has_albumartist)
1814 {
1815 write_item(id3.albumartist);
1816 }
1817 else
1818 {
1819 write_item(id3.artist);
1820 }
1821 if (has_grouping)
1822 {
1823 write_item(id3.grouping);
1824 }
1825 else
1826 {
1827 write_item(id3.title);
1828 }
1829 total_entry_count++;
1830 }
1831
1832 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1833 {
1834 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1835 int len = strlen(str)+1;
1836 int i;
1837 unsigned crc32;
1838 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1839 char buf[TAG_MAXLEN+32];
1840
1841 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1842 buf[i] = tolower(str[i]);
1843 buf[i] = '\0';
1844
1845 crc32 = crc_32(buf, i, 0xffffffff);
1846
1847 if (unique)
1848 {
1849 /* Check if the crc does not exist -> entry does not exist for sure. */
1850 for (i = 0; i < tempbufidx; i++)
1851 {
1852 if (crcbuf[-i] != crc32)
1853 continue;
1854
1855 if (!strcasecmp(str, index[i].str))
1856 {
1857 if (id < 0 || id >= lookup_buffer_depth)
1858 {
1859 logf("lookup buf overf.: %d", id);
1860 return false;
1861 }
1862
1863 lookup[id] = &index[i];
1864 return true;
1865 }
1866 }
1867 }
1868
1869 /* Insert to CRC buffer. */
1870 crcbuf[-tempbufidx] = crc32;
1871 tempbuf_left -= 4;
1872
1873 /* Insert it to the buffer. */
1874 tempbuf_left -= len;
1875 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1876 return false;
1877
1878 if (id >= lookup_buffer_depth)
1879 {
1880 logf("lookup buf overf. #2: %d", id);
1881 return false;
1882 }
1883
1884 if (id >= 0)
1885 {
1886 lookup[id] = &index[tempbufidx];
1887 index[tempbufidx].idlist.id = id;
1888 }
1889 else
1890 index[tempbufidx].idlist.id = -1;
1891
1892 index[tempbufidx].idlist.next = NULL;
1893 index[tempbufidx].idx_id = idx_id;
1894 index[tempbufidx].seek = -1;
1895 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1896 memcpy(index[tempbufidx].str, str, len);
1897 tempbuf_pos += len;
1898 tempbufidx++;
1899
1900 return true;
1901 }
1902
1903 static int compare(const void *p1, const void *p2)
1904 {
1905 do_timed_yield();
1906
1907 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1908 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1909
1910 if (strcmp(e1->str, UNTAGGED) == 0)
1911 {
1912 if (strcmp(e2->str, UNTAGGED) == 0)
1913 return 0;
1914 return -1;
1915 }
1916 else if (strcmp(e2->str, UNTAGGED) == 0)
1917 return 1;
1918
1919 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1920 }
1921
1922 static int tempbuf_sort(int fd)
1923 {
1924 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1925 struct tagfile_entry fe;
1926 int i;
1927 int length;
1928
1929 /* Generate reverse lookup entries. */
1930 for (i = 0; i < lookup_buffer_depth; i++)
1931 {
1932 struct tempbuf_id_list *idlist;
1933
1934 if (!lookup[i])
1935 continue;
1936
1937 if (lookup[i]->idlist.id == i)
1938 continue;
1939
1940 idlist = &lookup[i]->idlist;
1941 while (idlist->next != NULL)
1942 idlist = idlist->next;
1943
1944 tempbuf_left -= sizeof(struct tempbuf_id_list);
1945 if (tempbuf_left - 4 < 0)
1946 return -1;
1947
1948 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1949 if (tempbuf_pos & 0x03)
1950 {
1951 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1952 tempbuf_left -= 3;
1953 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1954 }
1955 tempbuf_pos += sizeof(struct tempbuf_id_list);
1956
1957 idlist = idlist->next;
1958 idlist->id = i;
1959 idlist->next = NULL;
1960
1961 do_timed_yield();
1962 }
1963
1964 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1965 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1966
1967 for (i = 0; i < tempbufidx; i++)
1968 {
1969 struct tempbuf_id_list *idlist = &index[i].idlist;
1970
1971 /* Fix the lookup list. */
1972 while (idlist != NULL)
1973 {
1974 if (idlist->id >= 0)
1975 lookup[idlist->id] = &index[i];
1976 idlist = idlist->next;
1977 }
1978
1979 index[i].seek = lseek(fd, 0, SEEK_CUR);
1980 length = strlen(index[i].str) + 1;
1981 fe.tag_length = length;
1982 fe.idx_id = index[i].idx_id;
1983
1984 /* Check the chunk alignment. */
1985 if ((fe.tag_length + sizeof(struct tagfile_entry))
1986 % TAGFILE_ENTRY_CHUNK_LENGTH)
1987 {
1988 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1989 ((fe.tag_length + sizeof(struct tagfile_entry))
1990 % TAGFILE_ENTRY_CHUNK_LENGTH);
1991 }
1992
1993 #ifdef TAGCACHE_STRICT_ALIGN
1994 /* Make sure the entry is long aligned. */
1995 if (index[i].seek & 0x03)
1996 {
1997 logf("tempbuf_sort: alignment error!");
1998 return -3;
1999 }
2000 #endif
2001
2002 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2003 sizeof(struct tagfile_entry))
2004 {
2005 logf("tempbuf_sort: write error #1");
2006 return -1;
2007 }
2008
2009 if (write(fd, index[i].str, length) != length)
2010 {
2011 logf("tempbuf_sort: write error #2");
2012 return -2;
2013 }
2014
2015 /* Write some padding. */
2016 if (fe.tag_length - length > 0)
2017 write(fd, "XXXXXXXX", fe.tag_length - length);
2018 }
2019
2020 return i;
2021 }
2022
2023 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2024 {
2025 if (id < 0 || id >= lookup_buffer_depth)
2026 return NULL;
2027
2028 return lookup[id];
2029 }
2030
2031
2032 inline static int tempbuf_find_location(int id)
2033 {
2034 struct tempbuf_searchidx *entry;
2035
2036 entry = tempbuf_locate(id);
2037 if (entry == NULL)
2038 return -1;
2039
2040 return entry->seek;
2041 }
2042
2043 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2044 {
2045 struct master_header tcmh;
2046 struct index_entry idx;
2047 int masterfd;
2048 int masterfd_pos;
2049 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2050 int max_entries;
2051 int entries_processed = 0;
2052 int i, j;
2053 char buf[TAG_MAXLEN];
2054
2055 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2056
2057 logf("Building numeric indices...");
2058 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2059
2060 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2061 return false;
2062
2063 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2064 SEEK_CUR);
2065 if (masterfd_pos == filesize(masterfd))
2066 {
2067 logf("we can't append!");
2068 close(masterfd);
2069 return false;
2070 }
2071
2072 while (entries_processed < h->entry_count)
2073 {
2074 int count = MIN(h->entry_count - entries_processed, max_entries);
2075
2076 /* Read in as many entries as possible. */
2077 for (i = 0; i < count; i++)
2078 {
2079 struct temp_file_entry *tfe = &entrybuf[i];
2080 int datastart;
2081
2082 /* Read in numeric data. */
2083 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2084 sizeof(struct temp_file_entry))
2085 {
2086 logf("read fail #1");
2087 close(masterfd);
2088 return false;
2089 }
2090
2091 datastart = lseek(tmpfd, 0, SEEK_CUR);
2092
2093 /**
2094 * Read string data from the following tags:
2095 * - tag_filename
2096 * - tag_artist
2097 * - tag_album
2098 * - tag_title
2099 *
2100 * A crc32 hash is calculated from the read data
2101 * and stored back to the data offset field kept in memory.
2102 */
2103 #define tmpdb_read_string_tag(tag) \
2104 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2105 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2106 { \
2107 logf("read fail: buffer overflow"); \
2108 close(masterfd); \
2109 return false; \
2110 } \
2111 \
2112 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2113 tfe->tag_length[tag]) \
2114 { \
2115 logf("read fail #2"); \
2116 close(masterfd); \
2117 return false; \
2118 } \
2119 \
2120 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2121 lseek(tmpfd, datastart, SEEK_SET)
2122
2123 tmpdb_read_string_tag(tag_filename);
2124 tmpdb_read_string_tag(tag_artist);
2125 tmpdb_read_string_tag(tag_album);
2126 tmpdb_read_string_tag(tag_title);
2127
2128 /* Seek to the end of the string data. */
2129 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2130 }
2131
2132 /* Backup the master index position. */
2133 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2134 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2135
2136 /* Check if we can resurrect some deleted runtime statistics data. */
2137 for (i = 0; i < tcmh.tch.entry_count; i++)
2138 {
2139 /* Read the index entry. */
2140 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2141 != sizeof(struct index_entry))
2142 {
2143 logf("read fail #3");
2144 close(masterfd);
2145 return false;
2146 }
2147
2148 /**
2149 * Skip unless the entry is marked as being deleted
2150 * or the data has already been resurrected.
2151 */
2152 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2153 continue;
2154
2155 /* Now try to match the entry. */
2156 /**
2157 * To succesfully match a song, the following conditions
2158 * must apply:
2159 *
2160 * For numeric fields: tag_length
2161 * - Full identical match is required
2162 *
2163 * If tag_filename matches, no further checking necessary.
2164 *
2165 * For string hashes: tag_artist, tag_album, tag_title
2166 * - Two of these must match
2167 */
2168 for (j = 0; j < count; j++)
2169 {
2170 struct temp_file_entry *tfe = &entrybuf[j];
2171
2172 /* Try to match numeric fields first. */
2173 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2174 continue;
2175
2176 /* Now it's time to do the hash matching. */
2177 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2178 {
2179 int match_count = 0;
2180
2181 /* No filename match, check if we can match two other tags. */
2182 #define tmpdb_match(tag) \
2183 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2184 match_count++
2185
2186 tmpdb_match(tag_artist);
2187 tmpdb_match(tag_album);
2188 tmpdb_match(tag_title);
2189
2190 if (match_count < 2)
2191 {
2192 /* Still no match found, give up. */
2193 continue;
2194 }
2195 }
2196
2197 /* A match found, now copy & resurrect the statistical data. */
2198 #define tmpdb_copy_tag(tag) \
2199 tfe->tag_offset[tag] = idx.tag_seek[tag]
2200
2201 tmpdb_copy_tag(tag_playcount);
2202 tmpdb_copy_tag(tag_rating);
2203 tmpdb_copy_tag(tag_playtime);
2204 tmpdb_copy_tag(tag_lastplayed);
2205 tmpdb_copy_tag(tag_commitid);
2206
2207 /* Avoid processing this entry again. */
2208 idx.flag |= FLAG_RESURRECTED;
2209
2210 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2211 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2212 != sizeof(struct index_entry))
2213 {
2214 logf("masterfd writeback fail #1");
2215 close(masterfd);
2216 return false;
2217 }
2218
2219 logf("Entry resurrected");
2220 }
2221 }
2222
2223
2224 /* Restore the master index position. */
2225 lseek(masterfd, masterfd_pos, SEEK_SET);
2226
2227 /* Commit the data to the index. */
2228 for (i = 0; i < count; i++)
2229 {
2230 int loc = lseek(masterfd, 0, SEEK_CUR);
2231
2232 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2233 != sizeof(struct index_entry))
2234 {
2235 logf("read fail #3");
2236 close(masterfd);
2237 return false;
2238 }
2239
2240 for (j = 0; j < TAG_COUNT; j++)
2241 {
2242 if (!tagcache_is_numeric_tag(j))
2243 continue;
2244
2245 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2246 }
2247 idx.flag = entrybuf[i].flag;
2248
2249 if (idx.tag_seek[tag_commitid])
2250 {
2251 /* Data has been resurrected. */
2252 idx.flag |= FLAG_DIRTYNUM;
2253 }
2254 else if (tc_stat.ready && current_tcmh.commitid > 0)
2255 {
2256 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2257 idx.flag |= FLAG_DIRTYNUM;
2258 }
2259
2260 /* Write back the updated index. */
2261 lseek(masterfd, loc, SEEK_SET);
2262 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2263 != sizeof(struct index_entry))
2264 {
2265 logf("write fail");
2266 close(masterfd);
2267 return false;
2268 }
2269 }
2270
2271 entries_processed += count;
2272 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2273 }
2274
2275 close(masterfd);
2276
2277 return true;
2278 }
2279
2280 /**
2281 * Return values:
2282 * > 0 success
2283 * == 0 temporary failure
2284 * < 0 fatal error
2285 */
2286 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2287 {
2288 int i;
2289 struct tagcache_header tch;
2290 struct master_header tcmh;
2291 struct index_entry idxbuf[IDX_BUF_DEPTH];
2292 int idxbuf_pos;
2293 char buf[TAG_MAXLEN+32];
2294 int fd = -1, masterfd;
2295 bool error = false;
2296 int init;
2297 int masterfd_pos;
2298
2299 logf("Building index: %d", index_type);
2300
2301 /* Check the number of entries we need to allocate ram for. */
2302 commit_entry_count = h->entry_count + 1;
2303
2304 masterfd = open_master_fd(&tcmh, false);
2305 if (masterfd >= 0)
2306 {
2307 commit_entry_count += tcmh.tch.entry_count;
2308 close(masterfd);
2309 }
2310 else
2311 remove_files(); /* Just to be sure we are clean. */
2312
2313 /* Open the index file, which contains the tag names. */
2314 fd = open_tag_fd(&tch, index_type, true);
2315 if (fd >= 0)
2316 {
2317 logf("tch.datasize=%ld", tch.datasize);
2318 lookup_buffer_depth = 1 +
2319 /* First part */ commit_entry_count +
2320 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2321 }
2322 else
2323 {
2324 lookup_buffer_depth = 1 +
2325 /* First part */ commit_entry_count +
2326 /* Second part */ 0;
2327 }
2328
2329 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2330 logf("commit_entry_count=%ld", commit_entry_count);
2331
2332 /* Allocate buffer for all index entries from both old and new
2333 * tag files. */
2334 tempbufidx = 0;
2335 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2336
2337 /* Allocate lookup buffer. The first portion of commit_entry_count
2338 * contains the new tags in the temporary file and the second
2339 * part for locating entries already in the db.
2340 *
2341 * New tags Old tags
2342 * +---------+---------------------------+
2343 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2344 * +---------+---------------------------+
2345 *
2346 * Old tags are inserted to a temporary buffer with position:
2347 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2348 * And new tags with index:
2349 * tempbuf_insert(idx, ...);
2350 *
2351 * The buffer is sorted and written into tag file:
2352 * tempbuf_sort(...);
2353 * leaving master index locations messed up.
2354 *
2355 * That is fixed using the lookup buffer for old tags:
2356 * new_seek = tempbuf_find_location(old_seek, ...);
2357 * and for new tags:
2358 * new_seek = tempbuf_find_location(idx);
2359 */
2360 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2361 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2362 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2363
2364 /* And calculate the remaining data space used mainly for storing
2365 * tag data (strings). */
2366 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2367 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2368 {
2369 logf("Buffer way too small!");
2370 return 0;
2371 }
2372
2373 if (fd >= 0)
2374 {
2375 /**
2376 * If tag file contains unique tags (sorted index), we will load
2377 * it entirely into memory so we can resort it later for use with
2378 * chunked browsing.
2379 */
2380 if (tagcache_is_sorted_tag(index_type))
2381 {
2382 logf("loading tags...");
2383 for (i = 0; i < tch.entry_count; i++)
2384 {
2385 struct tagfile_entry entry;
2386 int loc = lseek(fd, 0, SEEK_CUR);
2387 bool ret;
2388
2389 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2390 != sizeof(struct tagfile_entry))
2391 {
2392 logf("read error #7");
2393 close(fd);
2394 return -2;
2395 }
2396
2397 if (entry.tag_length >= (int)sizeof(buf))
2398 {
2399 logf("too long tag #3");
2400 close(fd);
2401 return -2;
2402 }
2403
2404 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2405 {
2406 logf("read error #8");
2407 close(fd);
2408 return -2;
2409 }
2410
2411 /* Skip deleted entries. */
2412 if (buf[0] == '\0')
2413 continue;
2414
2415 /**
2416 * Save the tag and tag id in the memory buffer. Tag id
2417 * is saved so we can later reindex the master lookup
2418 * table when the index gets resorted.
2419 */
2420 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2421 + commit_entry_count, entry.idx_id,
2422 tagcache_is_unique_tag(index_type));
2423 if (!ret)
2424 {
2425 close(fd);
2426 return -3;
2427 }
2428 do_timed_yield();
2429 }
2430 logf("done");
2431 }
2432 else
2433 tempbufidx = tch.entry_count;
2434 }
2435 else
2436 {
2437 /**
2438 * Creating new index file to store the tags. No need to preload
2439 * anything whether the index type is sorted or not.
2440 */
2441 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2442 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2443 if (fd < 0)
2444 {
2445 logf("%s open fail", buf);
2446 return -2;
2447 }
2448
2449 tch.magic = TAGCACHE_MAGIC;
2450 tch.entry_count = 0;
2451 tch.datasize = 0;
2452
2453 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2454 != sizeof(struct tagcache_header))
2455 {
2456 logf("header write failed");
2457 close(fd);
2458 return -2;
2459 }
2460 }
2461
2462 /* Loading the tag lookup file as "master file". */
2463 logf("Loading index file");
2464 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2465
2466 if (masterfd < 0)
2467 {
2468 logf("Creating new DB");
2469 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2470
2471 if (masterfd < 0)
2472 {
2473 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2474 close(fd);
2475 return -2;
2476 }
2477
2478 /* Write the header (write real values later). */
2479 memset(&tcmh, 0, sizeof(struct master_header));
2480 tcmh.tch = *h;
2481 tcmh.tch.entry_count = 0;
2482 tcmh.tch.datasize = 0;
2483 tcmh.dirty = true;
2484 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2485 init = true;
2486 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2487 }
2488 else
2489 {
2490 /**
2491 * Master file already exists so we need to process the current
2492 * file first.
2493 */
2494 init = false;
2495
2496 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2497 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2498 {
2499 logf("header error");
2500 close(fd);
2501 close(masterfd);
2502 return -2;
2503 }
2504
2505 /**
2506 * If we reach end of the master file, we need to expand it to
2507 * hold new tags. If the current index is not sorted, we can
2508 * simply append new data to end of the file.
2509 * However, if the index is sorted, we need to update all tag
2510 * pointers in the master file for the current index.
2511 */
2512 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2513 SEEK_CUR);
2514 if (masterfd_pos == filesize(masterfd))
2515 {
2516 logf("appending...");
2517 init = true;
2518 }
2519 }
2520
2521 /**
2522 * Load new unique tags in memory to be sorted later and added
2523 * to the master lookup file.
2524 */
2525 if (tagcache_is_sorted_tag(index_type))
2526 {
2527 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2528 /* h is the header of the temporary file containing new tags. */
2529 logf("inserting new tags...");
2530 for (i = 0; i < h->entry_count; i++)
2531 {
2532 struct temp_file_entry entry;
2533
2534 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2535 sizeof(struct temp_file_entry))
2536 {
2537 logf("read fail #3");
2538 error = true;
2539 goto error_exit;
2540 }
2541
2542 /* Read data. */
2543 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2544 {
2545 logf("too long entry!");
2546 error = true;
2547 goto error_exit;
2548 }
2549
2550 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2551 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2552 entry.tag_length[index_type])
2553 {
2554 logf("read fail #4");
2555 error = true;
2556 goto error_exit;
2557 }
2558
2559 if (tagcache_is_unique_tag(index_type))
2560 error = !tempbuf_insert(buf, i, -1, true);
2561 else
2562 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2563
2564 if (error)
2565 {
2566 logf("insert error");
2567 goto error_exit;
2568 }
2569
2570 /* Skip to next. */
2571 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2572 entry.tag_length[index_type], SEEK_CUR);
2573 do_timed_yield();
2574 }
2575 logf("done");
2576
2577 /* Sort the buffer data and write it to the index file. */
2578 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2579 i = tempbuf_sort(fd);
2580 if (i < 0)
2581 goto error_exit;
2582 logf("sorted %d tags", i);
2583
2584 /**
2585 * Now update all indexes in the master lookup file.
2586 */
2587 logf("updating indices...");
2588 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2589 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2590 {
2591 int j;
2592 int loc = lseek(masterfd, 0, SEEK_CUR);
2593
2594 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2595
2596 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2597 != (int)sizeof(struct index_entry)*idxbuf_pos)
2598 {
2599 logf("read fail #5");
2600 error = true;
2601 goto error_exit ;
2602 }
2603 lseek(masterfd, loc, SEEK_SET);
2604
2605 for (j = 0; j < idxbuf_pos; j++)
2606 {
2607 if (idxbuf[j].flag & FLAG_DELETED)
2608 {
2609 /* We can just ignore deleted entries. */
2610 // idxbuf[j].tag_seek[index_type] = 0;
2611 continue;
2612 }
2613
2614 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2615 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2616 + commit_entry_count);
2617
2618 if (idxbuf[j].tag_seek[index_type] < 0)
2619 {
2620 logf("update error: %d/%d/%ld",
2621 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2622 error = true;
2623 goto error_exit;
2624 }
2625
2626 do_timed_yield();
2627 }
2628
2629 /* Write back the updated index. */
2630 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2631 index_entry_ec, tc_stat.econ) !=
2632 (int)sizeof(struct index_entry)*idxbuf_pos)
2633 {
2634 logf("write fail");
2635 error = true;
2636 goto error_exit;
2637 }
2638 }
2639 logf("done");
2640 }
2641
2642 /**
2643 * Walk through the temporary file containing the new tags.
2644 */
2645 // build_normal_index(h, tmpfd, masterfd, idx);
2646 logf("updating new indices...");
2647 lseek(masterfd, masterfd_pos, SEEK_SET);
2648 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2649 lseek(fd, 0, SEEK_END);
2650 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2651 {
2652 int j;
2653
2654 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2655 if (init)
2656 {
2657 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2658 }
2659 else
2660 {
2661 int loc = lseek(masterfd, 0, SEEK_CUR);
2662
2663 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2664 != (int)sizeof(struct index_entry)*idxbuf_pos)
2665 {
2666 logf("read fail #6");
2667 error = true;
2668 break ;
2669 }
2670 lseek(masterfd, loc, SEEK_SET);
2671 }
2672
2673 /* Read entry headers. */
2674 for (j = 0; j < idxbuf_pos; j++)
2675 {
2676 if (!tagcache_is_sorted_tag(index_type))
2677 {
2678 struct temp_file_entry entry;
2679 struct tagfile_entry fe;
2680
2681 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2682 sizeof(struct temp_file_entry))
2683 {
2684 logf("read fail #7");
2685 error = true;
2686 break ;
2687 }
2688
2689 /* Read data. */
2690 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2691 {
2692 logf("too long entry!");
2693 logf("length=%d", entry.tag_length[index_type]);
2694 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2695 error = true;
2696 break ;
2697 }
2698
2699 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2700 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2701 entry.tag_length[index_type])
2702 {
2703 logf("read fail #8");
2704 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2705 logf("length=0x%02x", entry.tag_length[index_type]);
2706 error = true;
2707 break ;
2708 }
2709
2710 /* Write to index file. */
2711 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2712 fe.tag_length = entry.tag_length[index_type];
2713 fe.idx_id = tcmh.tch.entry_count + i + j;
2714 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2715 write(fd, buf, fe.tag_length);
2716 tempbufidx++;
2717
2718 /* Skip to next. */
2719 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2720 entry.tag_length[index_type], SEEK_CUR);
2721 }
2722 else
2723 {
2724 /* Locate the correct entry from the sorted array. */
2725 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2726 if (idxbuf[j].tag_seek[index_type] < 0)
2727 {
2728 logf("entry not found (%d)", j);
2729 error = true;
2730 break ;
2731 }
2732 }
2733 }
2734
2735 /* Write index. */
2736 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2737 index_entry_ec, tc_stat.econ) !=
2738 (int)sizeof(struct index_entry)*idxbuf_pos)
2739 {
2740 logf("tagcache: write fail #4");
2741 error = true;
2742 break ;
2743 }
2744
2745 do_timed_yield();
2746 }
2747 logf("done");
2748
2749 /* Finally write the header. */
2750 tch.magic = TAGCACHE_MAGIC;
2751 tch.entry_count = tempbufidx;
2752 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2753 lseek(fd, 0, SEEK_SET);
2754 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2755
2756 if (index_type != tag_filename)
2757 h->datasize += tch.datasize;
2758 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2759 error_exit:
2760
2761 close(fd);
2762 close(masterfd);
2763
2764 if (error)
2765 return -2;
2766
2767 return 1;
2768 }
2769
2770 static bool commit(void)
2771 {
2772 struct tagcache_header tch;
2773 struct master_header tcmh;
2774 int i, len, rc;
2775 int tmpfd;
2776 int masterfd;
2777 #ifdef HAVE_DIRCACHE
2778 bool dircache_buffer_stolen = false;
2779 #endif
2780 bool local_allocation = false;
2781
2782 logf("committing tagcache");
2783
2784 while (write_lock)
2785 sleep(1);
2786
2787 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2788 if (tmpfd < 0)
2789 {
2790 logf("nothing to commit");
2791 return true;
2792 }
2793
2794
2795 /* Load the header. */
2796 len = sizeof(struct tagcache_header);
2797 rc = read(tmpfd, &tch, len);
2798
2799 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2800 {
2801 logf("incorrect header");
2802 close(tmpfd);
2803 remove(TAGCACHE_FILE_TEMP);
2804 return false;
2805 }
2806
2807 if (tch.entry_count == 0)
2808 {
2809 logf("nothing to commit");
2810 close(tmpfd);
2811 remove(TAGCACHE_FILE_TEMP);
2812 return true;
2813 }
2814
2815 #ifdef HAVE_EEPROM_SETTINGS
2816 remove(TAGCACHE_STATEFILE);
2817 #endif
2818
2819 /* At first be sure to unload the ramcache! */
2820 #ifdef HAVE_TC_RAMCACHE
2821 tc_stat.ramcache = false;
2822 #endif
2823
2824 read_lock++;
2825
2826 /* Try to steal every buffer we can :) */
2827 if (tempbuf_size == 0)
2828 local_allocation = true;
2829
2830 #ifdef HAVE_DIRCACHE
2831 if (tempbuf_size == 0)
2832 {
2833 /* Try to steal the dircache buffer. */
2834 tempbuf = dircache_steal_buffer(&tempbuf_size);
2835 tempbuf_size &= ~0x03;
2836
2837 if (tempbuf_size > 0)
2838 {
2839 dircache_buffer_stolen = true;
2840 }
2841 }
2842 #endif
2843
2844 #ifdef HAVE_TC_RAMCACHE
2845 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2846 {
2847 tempbuf = (char *)(hdr + 1);
2848 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2849 tempbuf_size &= ~0x03;
2850 }
2851 #endif
2852
2853 /* And finally fail if there are no buffers available. */
2854 if (tempbuf_size == 0)
2855 {
2856 logf("delaying commit until next boot");
2857 tc_stat.commit_delayed = true;
2858 close(tmpfd);
2859 read_lock--;
2860 return false;
2861 }
2862
2863 logf("commit %ld entries...", tch.entry_count);
2864
2865 /* Mark DB dirty so it will stay disabled if commit fails. */
2866 current_tcmh.dirty = true;
2867 update_master_header();
2868
2869 /* Now create the index files. */
2870 tc_stat.commit_step = 0;
2871 tch.datasize = 0;
2872 tc_stat.commit_delayed = false;
2873
2874 for (i = 0; i < TAG_COUNT; i++)
2875 {
2876 int ret;
2877
2878 if (tagcache_is_numeric_tag(i))
2879 continue;
2880
2881 tc_stat.commit_step++;
2882 ret = build_index(i, &tch, tmpfd);
2883 if (ret <= 0)
2884 {
2885 close(tmpfd);
2886 logf("tagcache failed init");
2887 if (ret < 0)
2888 remove_files();
2889 else
2890 tc_stat.commit_delayed = true;
2891 tc_stat.commit_step = 0;
2892 read_lock--;
2893 return false;
2894 }
2895 }
2896
2897 if (!build_numeric_indices(&tch, tmpfd))
2898 {
2899 logf("Failure to commit numeric indices");
2900 close(tmpfd);
2901 remove_files();
2902 tc_stat.commit_step = 0;
2903 read_lock--;
2904 return false;
2905 }
2906
2907 close(tmpfd);
2908 tc_stat.commit_step = 0;
2909
2910 /* Update the master index headers. */
2911 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2912 {
2913 read_lock--;
2914 return false;
2915 }
2916
2917 tcmh.tch.entry_count += tch.entry_count;
2918 tcmh.tch.datasize = sizeof(struct master_header)
2919 + sizeof(struct index_entry) * tcmh.tch.entry_count
2920 + tch.datasize;
2921 tcmh.dirty = false;
2922 tcmh.commitid++;
2923
2924 lseek(masterfd, 0, SEEK_SET);
2925 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2926 close(masterfd);
2927
2928 logf("tagcache committed");
2929 remove(TAGCACHE_FILE_TEMP);
2930 tc_stat.ready = check_all_headers();
2931 tc_stat.readyvalid = true;
2932
2933 if (local_allocation)
2934 {
2935 tempbuf = NULL;
2936 tempbuf_size = 0;
2937 }
2938
2939 #ifdef HAVE_DIRCACHE
2940 /* Rebuild the dircache, if we stole the buffer. */
2941 if (dircache_buffer_stolen)
2942 dircache_build(0);
2943 #endif
2944
2945 #ifdef HAVE_TC_RAMCACHE
2946 /* Reload tagcache. */
2947 if (tc_stat.ramcache_allocated > 0)
2948 tagcache_start_scan();
2949 #endif
2950
2951 read_lock--;
2952
2953 return true;
2954 }
2955
2956 static void allocate_tempbuf(void)
2957 {
2958 /* Yeah, malloc would be really nice now :) */
2959 #ifdef __PCTOOL__
2960 tempbuf_size = 32*1024*1024;
2961 tempbuf = malloc(tempbuf_size);
2962 #else
2963 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2964 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2965 audiobuf += tempbuf_size;
2966 #endif
2967 }
2968
2969 static void free_tempbuf(void)
2970 {
2971 if (tempbuf_size == 0)
2972 return ;
2973
2974 #ifdef __PCTOOL__
2975 free(tempbuf);
2976 #else
2977 audiobuf -= tempbuf_size;
2978 #endif
2979 tempbuf = NULL;
2980 tempbuf_size = 0;
2981 }
2982
2983 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2984 {
2985 struct index_entry idx;
2986
2987 if (!tc_stat.ready)
2988 return false;
2989
2990 if (!tagcache_is_numeric_tag(tag))
2991 return false;
2992
2993 if (!get_index(masterfd, idx_id, &idx, false))
2994 return false;
2995
2996 idx.tag_seek[tag] = data;
2997 idx.flag |= FLAG_DIRTYNUM;
2998
2999 return write_index(masterfd, idx_id, &idx);
3000 }
3001
3002 #if 0
3003 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3004 int tag, long data)
3005 {
3006 struct master_header myhdr;
3007
3008 if (tcs->masterfd < 0)
3009 {
3010 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3011 return false;
3012 }
3013
3014 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3015 }
3016 #endif
3017
3018 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3019
3020 static bool command_queue_is_full(void)
3021 {
3022 int next;
3023
3024 next = command_queue_widx + 1;
3025 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3026 next = 0;
3027
3028 return (next == command_queue_ridx);
3029 }
3030 static bool command_queue_sync_callback(void)
3031 {
3032
3033 struct master_header myhdr;
3034 int masterfd;
3035
3036 mutex_lock(&command_queue_mutex);
3037
3038 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3039 return false;
3040
3041 while (command_queue_ridx != command_queue_widx)
3042 {
3043 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3044
3045 switch (ce->command)
3046 {
3047 case CMD_UPDATE_MASTER_HEADER:
3048 {
3049 close(masterfd);
3050 update_master_header();
3051
3052 /* Re-open the masterfd. */
3053 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3054 return true;
3055
3056 break;
3057 }
3058 case CMD_UPDATE_NUMERIC:
3059 {
3060 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3061 break;
3062 }
3063 }
3064
3065 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3066 command_queue_ridx = 0;
3067 }
3068
3069 close(masterfd);
3070
3071 tc_stat.queue_length = 0;
3072 mutex_unlock(&command_queue_mutex);
3073 return true;
3074 }
3075
3076 static void run_command_queue(bool force)
3077 {
3078 if (COMMAND_QUEUE_IS_EMPTY)
3079 return;
3080
3081 if (force || command_queue_is_full())
3082 command_queue_sync_callback();
3083 else
3084 register_ata_idle_func(command_queue_sync_callback);
3085 }
3086
3087 static void queue_command(int cmd, long idx_id, int tag, long data)
3088 {
3089 while (1)
3090 {
3091 int next;
3092
3093 mutex_lock(&command_queue_mutex);
3094 next = command_queue_widx + 1;
3095 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3096 next = 0;
3097
3098 /* Make sure queue is not full. */
3099 if (next != command_queue_ridx)
3100 {
3101 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3102
3103 ce->command = cmd;
3104 ce->idx_id = idx_id;
3105 ce->tag = tag;
3106 ce->data = data;
3107
3108 command_queue_widx = next;
3109
3110 tc_stat.queue_length++;
3111
3112 mutex_unlock(&command_queue_mutex);
3113 break;
3114 }
3115
3116 /* Queue is full, try again later... */
3117 mutex_unlock(&command_queue_mutex);
3118 sleep(1);
3119 }
3120 }
3121
3122 long tagcache_increase_serial(void)
3123 {
3124 long old;
3125
3126 if (!tc_stat.ready)
3127 return -2;
3128
3129 while (read_lock)
3130 sleep(1);
3131
3132 old = current_tcmh.serial++;
3133 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3134
3135 return old;
3136 }
3137
3138 void tagcache_update_numeric(int idx_id, int tag, long data)
3139 {
3140 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3141 }
3142
3143 long tagcache_get_serial(void)
3144 {
3145 return current_tcmh.serial;
3146 }
3147
3148 long tagcache_get_commitid(void)
3149 {
3150 return current_tcmh.commitid;
3151 }
3152
3153 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3154 {
3155 char buf[512];
3156 int i;
3157
3158 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3159 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3160 {
3161 if (*datastr == '\0')
3162 break;
3163
3164 if (*datastr == '"' || *datastr == '\\')
3165 buf[i++] = '\\';
3166
3167 buf[i] = *(datastr++);
3168 }
3169
3170 strcpy(&buf[i], "\" ");
3171
3172 write(fd, buf, i + 2);
3173
3174 return true;
3175 }
3176
3177 static bool read_tag(char *dest, long size,
3178 const char *src, const char *tagstr)
3179 {
3180 int pos;
3181 char current_tag[32];
3182
3183 while (*src != '\0')
3184 {
3185 /* Skip all whitespace */
3186 while (*src == ' ')
3187 src++;
3188
3189 if (*src == '\0')
3190 break;
3191
3192 pos = 0;
3193 /* Read in tag name */
3194 while (*src != '=' && *src != ' ')
3195 {
3196 current_tag[pos] = *src;
3197 src++;
3198 pos++;
3199
3200 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3201 return false;
3202 }
3203 current_tag[pos] = '\0';
3204
3205 /* Read in tag data */
3206
3207 /* Find the start. */
3208 while (*src != '"' && *src != '\0')
3209 src++;
3210
3211 if (*src == '\0' || *(++src) == '\0')
3212 return false;
3213
3214 /* Read the data. */
3215 for (pos = 0; pos < size; pos++)
3216 {
3217 if (*src == '\0')
3218 break;
3219
3220 if (*src == '\\')
3221 {
3222 dest[pos] = *(src+1);
3223 src += 2;
3224 continue;
3225 }
3226
3227 dest[pos] = *src;
3228
3229 if (*src == '"')
3230 {
3231 src++;
3232 break;
3233 }
3234
3235 if (*src == '\0')
3236 break;
3237
3238 src++;
3239 }
3240 dest[pos] = '\0';
3241
3242 if (!strcasecmp(tagstr, current_tag))
3243 return true;
3244 }
3245
3246 return false;
3247 }
3248
3249 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3250 {
3251 struct index_entry idx;
3252 char tag_data[TAG_MAXLEN+32];
3253 int idx_id;
3254 long masterfd = (long)parameters;
3255 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3256 tag_commitid };
3257 int i;
3258 (void)line_n;
3259
3260 if (*buf == '#')
3261 return 0;
3262
3263 logf("%d/%s", line_n, buf);
3264 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3265 {
3266 logf("filename missing");
3267 logf("-> %s", buf);
3268 return 0;
3269 }
3270
3271 idx_id = find_index(tag_data);
3272 if (idx_id < 0)
3273 {
3274 logf("entry not found");
3275 return 0;
3276 }
3277
3278 if (!get_index(masterfd, idx_id, &idx, false))
3279 {
3280 logf("failed to retrieve index entry");
3281 return 0;
3282 }
3283
3284 /* Stop if tag has already been modified. */
3285 if (idx.flag & FLAG_DIRTYNUM)
3286 return 0;
3287
3288 logf("import: %s", tag_data);
3289
3290 idx.flag |= FLAG_DIRTYNUM;
3291 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3292 {
3293 int data;
3294
3295 if (!read_tag(tag_data, sizeof tag_data, buf,
3296 tagcache_tag_to_str(import_tags[i])))
3297 {
3298 continue;
3299 }
3300
3301 data = atoi(tag_data);
3302 if (data < 0)
3303 continue;
3304
3305 idx.tag_seek[import_tags[i]] = data;
3306
3307 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3308 current_tcmh.serial = data;
3309 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3310 current_tcmh.commitid = data + 1;
3311 }
3312
3313 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3314 }
3315
3316 #ifndef __PCTOOL__
3317 bool tagcache_import_changelog(void)
3318 {
3319 struct master_header myhdr;
3320 struct tagcache_header tch;
3321 int clfd;
3322 long masterfd;
3323 char buf[2048];
3324
3325 if (!tc_stat.ready)
3326 return false;
3327
3328 while (read_lock)
3329 sleep(1);
3330
3331 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3332 if (clfd < 0)
3333 {
3334 logf("failure to open changelog");
3335 return false;
3336 }
3337
3338 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3339 {
3340 close(clfd);
3341 return false;
3342 }
3343
3344 write_lock++;
3345
3346 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3347
3348 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3349 parse_changelog_line);
3350
3351 close(clfd);
3352 close(masterfd);
3353
3354 if (filenametag_fd >= 0)
3355 close(filenametag_fd);
3356
3357 write_lock--;
3358
3359 update_master_header();
3360
3361 return true;
3362 }
3363 #endif
3364
3365 bool tagcache_create_changelog(struct tagcache_search *tcs)
3366 {
3367 struct master_header myhdr;
3368 struct index_entry idx;
3369 char buf[TAG_MAXLEN+32];
3370 char temp[32];
3371 int clfd;
3372 int i, j;
3373
3374 if (!tc_stat.ready)
3375 return false;
3376
3377 if (!tagcache_search(tcs, tag_filename))
3378 return false;
3379
3380 /* Initialize the changelog */
3381 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3382 if (clfd < 0)
3383 {
3384 logf("failure to open changelog");
3385 return false;
3386 }
3387
3388 if (tcs->masterfd < 0)
3389 {
3390 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3391 return false;
3392 }
3393 else
3394 {
3395 lseek(tcs->masterfd, 0, SEEK_SET);
3396 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3397 }
3398
3399 write(clfd, "## Changelog version 1\n", 23);
3400
3401 for (i = 0; i < myhdr.tch.entry_count; i++)
3402 {
3403 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3404 != sizeof(struct index_entry))
3405 {
3406 logf("read error #9");
3407 tagcache_search_finish(tcs);
3408 close(clfd);
3409 return false;
3410 }
3411
3412 /* Skip until the entry found has been modified. */
3413 if (! (idx.flag & FLAG_DIRTYNUM) )
3414 continue;
3415
3416 /* Skip deleted entries too. */
3417 if (idx.flag & FLAG_DELETED)
3418 continue;
3419
3420 /* Now retrieve all tags. */
3421 for (j = 0; j < TAG_COUNT; j++)
3422 {
3423 if (tagcache_is_numeric_tag(j))
3424 {
3425 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3426 write_tag(clfd, tagcache_tag_to_str(j), temp);
3427 continue;
3428 }
3429
3430 tcs->type = j;
3431 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3432 write_tag(clfd, tagcache_tag_to_str(j), buf);
3433 }
3434
3435 write(clfd, "\n", 1);
3436 do_timed_yield();
3437 }
3438
3439 close(clfd);
3440
3441 tagcache_search_finish(tcs);
3442
3443 return true;
3444 }
3445
3446 static bool delete_entry(long idx_id)
3447 {
3448 int fd = -1;
3449 int masterfd = -1;
3450 int tag, i;
3451 struct index_entry idx, myidx;
3452 struct master_header myhdr;
3453 char buf[TAG_MAXLEN+32];
3454 int in_use[TAG_COUNT];
3455
3456 logf("delete_entry(): %ld", idx_id);
3457
3458 #ifdef HAVE_TC_RAMCACHE
3459 /* At first mark the entry removed from ram cache. */
3460 if (tc_stat.ramcache)
3461 hdr->indices[idx_id].flag |= FLAG_DELETED;
3462 #endif
3463
3464 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3465 return false;
3466
3467 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3468 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3469 != sizeof(struct index_entry))
3470 {
3471 logf("delete_entry(): read error");
3472 goto cleanup;
3473 }
3474
3475 if (myidx.flag & FLAG_DELETED)
3476 {
3477 logf("delete_entry(): already deleted!");
3478 goto cleanup;
3479 }
3480
3481 myidx.flag |= FLAG_DELETED;
3482 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3483 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3484 != sizeof(struct index_entry))
3485 {
3486 logf("delete_entry(): write_error #1");
3487 goto cleanup;
3488 }
3489
3490 /* Now check which tags are no longer in use (if any) */
3491 for (tag = 0; tag < TAG_COUNT; tag++)
3492 in_use[tag] = 0;
3493
3494 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3495 for (i = 0; i < myhdr.tch.entry_count; i++)
3496 {
3497 struct index_entry *idxp;
3498
3499 #ifdef HAVE_TC_RAMCACHE
3500 /* Use RAM DB if available for greater speed */
3501 if (tc_stat.ramcache)
3502 idxp = &hdr->indices[i];
3503 else
3504 #endif
3505 {
3506 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3507 != sizeof(struct index_entry))
3508 {
3509 logf("delete_entry(): read error #2");
3510 goto cleanup;
3511 }
3512 idxp = &idx;
3513 }
3514
3515 if (idxp->flag & FLAG_DELETED)
3516 continue;
3517
3518 for (tag = 0; tag < TAG_COUNT; tag++)
3519 {
3520 if (tagcache_is_numeric_tag(tag))
3521 continue;
3522
3523 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3524 in_use[tag]++;
3525 }
3526 }
3527
3528 /* Now delete all tags no longer in use. */
3529 for (tag = 0; tag < TAG_COUNT; tag++)
3530 {
3531 struct tagcache_header tch;
3532 int oldseek = myidx.tag_seek[tag];
3533
3534 if (tagcache_is_numeric_tag(tag))
3535 continue;
3536
3537 /**
3538 * Replace tag seek with a hash value of the field string data.
3539 * That way runtime statistics of moved or altered files can be
3540 * resurrected.
3541 */
3542 #ifdef HAVE_TC_RAMCACHE
3543 if (tc_stat.ramcache && tag != tag_filename)
3544 {
3545 struct tagfile_entry *tfe;
3546 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3547
3548 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3549 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3550 myidx.tag_seek[tag] = *seek;
3551 }
3552 else
3553 #endif
3554 {
3555 struct tagfile_entry tfe;
3556
3557 /* Open the index file, which contains the tag names. */
3558 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3559 goto cleanup;
3560
3561 /* Skip the header block */
3562 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3563 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3564 != sizeof(struct tagfile_entry))
3565 {
3566 logf("delete_entry(): read error #3");
3567 goto cleanup;
3568 }
3569
3570 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3571 {
3572 logf("delete_entry(): read error #4");
3573 goto cleanup;
3574 }
3575
3576 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3577 }
3578
3579 if (in_use[tag])
3580 {
3581 logf("in use: %d/%d", tag, in_use[tag]);
3582 if (fd >= 0)
3583 {
3584 close(fd);
3585 fd = -1;
3586 }
3587 continue;
3588 }
3589
3590 #ifdef HAVE_TC_RAMCACHE
3591 /* Delete from ram. */
3592 if (tc_stat.ramcache && tag != tag_filename)
3593 {
3594 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3595 tagentry->tag_data[0] = '\0';
3596 }
3597 #endif
3598
3599 /* Open the index file, which contains the tag names. */
3600 if (fd < 0)
3601 {
3602 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3603 goto cleanup;
3604 }
3605
3606 /* Skip the header block */
3607 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3608
3609 /* Debug, print 10 first characters of the tag
3610 read(fd, buf, 10);
3611 buf[10]='\0';
3612 logf("TAG:%s", buf);
3613 lseek(fd, -10, SEEK_CUR);
3614 */
3615
3616 /* Write first data byte in tag as \0 */
3617 write(fd, "", 1);
3618
3619 /* Now tag data has been removed */
3620 close(fd);
3621 fd = -1;
3622 }
3623
3624 /* Write index entry back into master index. */
3625 lseek(masterfd, sizeof(struct master_header) +
3626 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3627 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3628 != sizeof(struct index_entry))
3629 {
3630 logf("delete_entry(): write_error #2");
3631 goto cleanup;
3632 }
3633
3634 close(masterfd);
3635
3636 return true;
3637
3638 cleanup:
3639 if (fd >= 0)
3640 close(fd);
3641 if (masterfd >= 0)
3642 close(masterfd);
3643
3644 return false;
3645 }
3646
3647 #ifndef __PCTOOL__
3648 /**
3649 * Returns true if there is an event waiting in the queue
3650 * that requires the current operation to be aborted.
3651 */
3652 static bool check_event_queue(void)
3653 {
3654 struct queue_event ev;
3655
3656 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3657 switch (ev.id)
3658 {
3659 case Q_STOP_SCAN:
3660 case SYS_POWEROFF:
3661 case SYS_USB_CONNECTED:
3662 /* Put the event back into the queue. */
3663 queue_post(&tagcache_queue, ev.id, ev.data);
3664 return true;
3665 }
3666
3667 return false;
3668 }
3669 #endif
3670
3671 #ifdef HAVE_TC_RAMCACHE
3672 static bool allocate_tagcache(void)
3673 {
3674 struct master_header tcmh;
3675 int fd;
3676
3677 /* Load the header. */
3678 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3679 {
3680 hdr = NULL;
3681 return false;
3682 }
3683
3684 close(fd);
3685
3686 /**
3687 * Now calculate the required cache size plus
3688 * some extra space for alignment fixes.
3689 */
3690 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3691 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3692 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3693 memset(hdr, 0, sizeof(struct ramcache_header));
3694 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3695 hdr->indices = (struct index_entry *)(hdr + 1);
3696 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3697
3698 return true;
3699 }
3700
3701 # ifdef HAVE_EEPROM_SETTINGS
3702 static bool tagcache_dumpload(void)
3703 {
3704 struct statefile_header shdr;
3705 int fd, rc;
3706 long offpos;
3707 int i;
3708
3709 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3710 if (fd < 0)
3711 {
3712 logf("no tagcache statedump");
3713 return false;
3714 }
3715
3716 /* Check the statefile memory placement */
3717 hdr = buffer_alloc(0);
3718 rc = read(fd, &shdr, sizeof(struct statefile_header));
3719 if (rc != sizeof(struct statefile_header)
3720 /* || (long)hdr != (long)shdr.hdr */)
3721 {
3722 logf("incorrect statefile");
3723 hdr = NULL;
3724 close(fd);
3725 return false;
3726 }
3727
3728 offpos = (long)hdr - (long)shdr.hdr;
3729
3730 /* Lets allocate real memory and load it */
3731 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3732 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3733 close(fd);
3734
3735 if (rc != shdr.tc_stat.ramcache_allocated)
3736 {
3737 logf("read failure!");
3738 hdr = NULL;
3739 return false;
3740 }
3741
3742 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3743
3744 /* Now fix the pointers */
3745 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3746 for (i = 0; i < TAG_COUNT; i++)
3747 hdr->tags[i] += offpos;
3748
3749 return true;
3750 }
3751
3752 static bool tagcache_dumpsave(void)
3753 {
3754 struct statefile_header shdr;
3755 int fd;
3756
3757 if (!tc_stat.ramcache)
3758 return false;
3759
3760 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3761 if (fd < 0)
3762 {
3763 logf("failed to create a statedump");
3764 return false;
3765 }
3766
3767 /* Create the header */
3768 shdr.hdr = hdr;
3769 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3770 write(fd, &shdr, sizeof(struct statefile_header));
3771
3772 /* And dump the data too */
3773 write(fd, hdr, tc_stat.ramcache_allocated);
3774 close(fd);
3775
3776 return true;
3777 }
3778 # endif
3779
3780 static bool load_tagcache(void)
3781 {
3782 struct tagcache_header *tch;
3783 long bytesleft = tc_stat.ramcache_allocated;
3784 struct index_entry *idx;
3785 int rc, fd;
3786 char *p;
3787 int i, tag;
3788
3789 # ifdef HAVE_DIRCACHE
3790 while (dircache_is_initializing())
3791 sleep(1);
3792
3793 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3794 # endif
3795
3796 logf("loading tagcache to ram...");
3797
3798 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3799 if (fd < 0)
3800 {
3801 logf("tagcache open failed");
3802 return false;
3803 }
3804
3805 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3806 != sizeof(struct master_header)
3807 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3808 {
3809 logf("incorrect header");
3810 return false;
3811 }
3812
3813 idx = hdr->indices;
3814
3815 /* Load the master index table. */
3816 for (i = 0; i < hdr->h.tch.entry_count; i++)
3817 {
3818 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3819 if (rc != sizeof(struct index_entry))
3820 {
3821 logf("read error #10");
3822 close(fd);
3823 return false;
3824 }
3825
3826 bytesleft -= sizeof(struct index_entry);
3827 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3828 {
3829 logf("too big tagcache.");
3830 close(fd);
3831 return false;
3832 }
3833
3834 idx++;
3835 }
3836
3837 close(fd);
3838
3839 /* Load the tags. */
3840 p = (char *)idx;
3841 for (tag = 0; tag < TAG_COUNT; tag++)
3842 {
3843 struct tagfile_entry *fe;
3844 char buf[TAG_MAXLEN+32];
3845
3846 if (tagcache_is_numeric_tag(tag))
3847 continue ;
3848
3849 //p = ((void *)p+1);
3850 p = (char *)((long)p & ~0x03) + 0x04;
3851 hdr->tags[tag] = p;
3852
3853 /* Check the header. */
3854 tch = (struct tagcache_header *)p;
3855 p += sizeof(struct tagcache_header);
3856
3857 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3858 return false;
3859
3860 for (hdr->entry_count[tag] = 0;
3861 hdr->entry_count[tag] < tch->entry_count;
3862 hdr->entry_count[tag]++)
3863 {
3864 long pos;
3865
3866 if (do_timed_yield())
3867 {
3868 /* Abort if we got a critical event in queue */
3869 if (check_event_queue())
3870 return false;
3871 }
3872
3873 fe = (struct tagfile_entry *)p;
3874 pos = lseek(fd, 0, SEEK_CUR);
3875 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3876 if (rc != sizeof(struct tagfile_entry))
3877 {
3878 /* End of lookup table. */
3879 logf("read error #11");
3880 close(fd);
3881 return false;
3882 }
3883
3884 /* We have a special handling for the filename tags. */
3885 if (tag == tag_filename)
3886 {
3887 # ifdef HAVE_DIRCACHE
3888 const struct dirent *dc;
3889 # endif
3890
3891 // FIXME: This is wrong!
3892 // idx = &hdr->indices[hdr->entry_count[i]];
3893 idx = &hdr->indices[fe->idx_id];
3894
3895 if (fe->tag_length >= (long)sizeof(buf)-1)
3896 {
3897 read(fd, buf, 10);
3898 buf[10] = '\0';
3899 logf("TAG:%s", buf);
3900 logf("too long filename");
3901 close(fd);
3902 return false;
3903 }
3904
3905 rc = read(fd, buf, fe->tag_length);
3906 if (rc != fe->tag_length)
3907 {
3908 logf("read error #12");
3909 close(fd);
3910 return false;
3911 }
3912
3913 /* Check if the entry has already been removed */
3914 if (idx->flag & FLAG_DELETED)
3915 continue;
3916
3917 /* This flag must not be used yet. */
3918 if (idx->flag & FLAG_DIRCACHE)
3919 {
3920 logf("internal error!");
3921 close(fd);
3922 return false;
3923 }
3924
3925 if (idx->tag_seek[tag] != pos)
3926 {
3927 logf("corrupt data structures!");
3928 close(fd);
3929 return false;
3930 }
3931
3932 # ifdef HAVE_DIRCACHE
3933 if (dircache_is_enabled())
3934 {
3935 dc = dircache_get_entry_ptr(buf);
3936 if (dc == NULL)
3937 {
3938 logf("Entry no longer valid.");
3939 logf("-> %s", buf);
3940 delete_entry(fe->idx_id);
3941 continue ;
3942 }
3943
3944 idx->flag |= FLAG_DIRCACHE;
3945 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3946 idx->tag_seek[tag_filename] = (long)dc;
3947 }
3948 else
3949 # endif
3950 {
3951 /* This will be very slow unless dircache is enabled
3952 or target is flash based, but do it anyway for
3953 consistency. */
3954 /* Check if entry has been removed. */
3955 if (global_settings.tagcache_autoupdate)
3956 {
3957 if (!file_exists(buf))
3958 {
3959 logf("Entry no longer valid.");
3960 logf("-> %s", buf);
3961 delete_entry(fe->idx_id);
3962 continue;
3963 }
3964 }
3965 }
3966
3967 continue ;
3968 }
3969
3970 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3971 if (bytesleft < 0)
3972 {
3973 logf("too big tagcache #2");
3974 logf("tl: %d", fe->tag_length);
3975 logf("bl: %ld", bytesleft);
3976 close(fd);
3977 return false;
3978 }
3979
3980 p = fe->tag_data;
3981 rc = read(fd, fe->tag_data, fe->tag_length);
3982 p += rc;
3983
3984 if (rc != fe->tag_length)
3985 {
3986 logf("read error #13");
3987 logf("rc=0x%04x", rc); // 0x431
3988 logf("len=0x%04x", fe->tag_length); // 0x4000
3989 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3990 logf("tag=0x%02x", tag); // 0x00
3991 close(fd);
3992 return false;
3993 }
3994 }
3995 close(fd);
3996 }
3997
3998 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
3999 logf("tagcache loaded into ram!");
4000
4001 return true;
4002 }
4003 #endif /* HAVE_TC_RAMCACHE */
4004
4005 static bool check_deleted_files(void)
4006 {
4007 int fd;
4008 char buf[TAG_MAXLEN+32];
4009 struct tagfile_entry tfe;
4010
4011 logf("reverse scan...");
4012 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4013 fd = open(buf, O_RDONLY);
4014
4015 if (fd < 0)
4016 {
4017 logf("%s open fail", buf);
4018 return false;
4019 }
4020
4021 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4022 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4023 == sizeof(struct tagfile_entry)
4024 #ifndef __PCTOOL__
4025 && !check_event_queue()
4026 #endif
4027 )
4028 {
4029 if (tfe.tag_length >= (long)sizeof(buf)-1)
4030 {
4031 logf("too long tag");
4032 close(fd);
4033 return false;
4034 }
4035
4036 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4037 {
4038 logf("read error #14");
4039 close(fd);
4040 return false;
4041 }
4042
4043 /* Check if the file has already deleted from the db. */
4044 if (*buf == '\0')
4045 continue;
4046
4047 /* Now check if the file exists. */
4048 if (!file_exists(buf))
4049 {
4050 logf("Entry no longer valid.");
4051 logf("-> %s / %d", buf, tfe.tag_length);
4052 delete_entry(tfe.idx_id);
4053 }
4054 }
4055
4056 close(fd);
4057
4058 logf("done");
4059
4060 return true;
4061 }
4062
4063 static bool check_dir(const char *dirname, int add_files)
4064 {
4065 DIR *dir;
4066 int len;
4067 int success = false;
4068 int ignore, unignore;
4069 char newpath[MAX_PATH];
4070
4071 dir = opendir(dirname);
4072 if (!dir)
4073 {
4074 logf("tagcache: opendir() failed");
4075 return false;
4076 }
4077
4078 /* check for a database.ignore file */
4079 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4080 ignore = file_exists(newpath);
4081 /* check for a database.unignore file */
4082 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4083 unignore = file_exists(newpath);
4084
4085 /* don't do anything if both ignore and unignore are there */
4086 if (ignore != unignore)
4087 add_files = unignore;
4088
4089 /* Recursively scan the dir. */
4090 #ifdef __PCTOOL__
4091 while (1)
4092 #else
4093 while (!check_event_queue())
4094 #endif
4095 {
4096 struct dirent *entry;
4097
4098 entry = readdir(dir);
4099
4100 if (entry == NULL)
4101 {
4102 success = true;
4103 break ;
4104 }
4105
4106 if (!strcmp((char *)entry->d_name, ".") ||
4107 !strcmp((char *)entry->d_name, ".."))
4108 continue;
4109
4110 yield();
4111
4112 len = strlen(curpath);
4113 snprintf(&curpath[len], curpath_size - len, "/%s",
4114 entry->d_name);
4115
4116 processed_dir_count++;
4117 if (entry->attribute & ATTR_DIRECTORY)
4118 check_dir(curpath, add_files);
4119 else if (add_files)
4120 {
4121 tc_stat.curentry = curpath;
4122
4123 /* Add a new entry to the temporary db file. */
4124 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4125 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4126 , dir->internal_entry
4127 #endif
4128 );
4129
4130 /* Wait until current path for debug screen is read and unset. */
4131 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4132 yield();
4133
4134 tc_stat.curentry = NULL;
4135 }
4136
4137 curpath[len] = '\0';
4138 }
4139
4140 closedir(dir);
4141
4142 return success;
4143 }
4144
4145 void tagcache_screensync_event(void)
4146 {
4147 tc_stat.curentry = NULL;
4148 }
4149
4150 void tagcache_screensync_enable(bool state)
4151 {
4152 tc_stat.syncscreen = state;
4153 }
4154
4155 void build_tagcache(const char *path)
4156 {
4157 struct tagcache_header header;
4158 bool ret;
4159
4160 curpath[0] = '\0';
4161 data_size = 0;
4162 total_entry_count = 0;
4163 processed_dir_count = 0;
4164
4165 #ifdef HAVE_DIRCACHE
4166 while (dircache_is_initializing())
4167 sleep(1);
4168 #endif
4169
4170 logf("updating tagcache");
4171
4172 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4173 if (cachefd >= 0)
4174 {
4175 logf("skipping, cache already waiting for commit");
4176 close(cachefd);
4177 return ;
4178 }
4179
4180 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4181 if (cachefd < 0)
4182 {
4183 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4184 return ;
4185 }
4186
4187 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4188
4189 cpu_boost(true);
4190
4191 logf("Scanning files...");
4192 /* Scan for new files. */
4193 memset(&header, 0, sizeof(struct tagcache_header));
4194 write(cachefd, &header, sizeof(struct tagcache_header));
4195
4196 if (strcmp("/", path) != 0)
4197 strcpy(curpath, path);
4198 ret = check_dir(path, true);
4199
4200 /* Write the header. */
4201 header.magic = TAGCACHE_MAGIC;
4202 header.datasize = data_size;
4203 header.entry_count = total_entry_count;
4204 lseek(cachefd, 0, SEEK_SET);
4205 write(cachefd, &header, sizeof(struct tagcache_header));
4206 close(cachefd);
4207
4208 if (filenametag_fd >= 0)
4209 {
4210 close(filenametag_fd);
4211 filenametag_fd = -1;
4212 }
4213
4214 if (!ret)
4215 {
4216 logf("Aborted.");
4217 cpu_boost(false);
4218 return ;
4219 }
4220
4221 /* Commit changes to the database. */
4222 #ifdef __PCTOOL__
4223 allocate_tempbuf();
4224 #endif
4225 if (commit())
4226 {
4227 remove(TAGCACHE_FILE_TEMP);
4228 logf("tagcache built!");
4229 }
4230 #ifdef __PCTOOL__
4231 free_tempbuf();
4232 #endif
4233
4234 #ifdef HAVE_TC_RAMCACHE
4235 if (hdr)
4236 {
4237 /* Import runtime statistics if we just initialized the db. */
4238 if (hdr->h.serial == 0)
4239 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4240 }
4241 #endif
4242
4243 cpu_boost(false);
4244 }
4245
4246 #ifdef HAVE_TC_RAMCACHE
4247 static void load_ramcache(void)
4248 {
4249 if (!hdr)
4250 return ;
4251
4252 cpu_boost(true);
4253
4254 /* At first we should load the cache (if exists). */
4255 tc_stat.ramcache = load_tagcache();
4256
4257 if (!tc_stat.ramcache)
4258 {
4259 /* If loading failed, it must indicate some problem with the db
4260 * so disable it entirely to prevent further issues. */
4261 tc_stat.ready = false;
4262 hdr = NULL;
4263 }
4264
4265 cpu_boost(false);
4266 }
4267
4268 void tagcache_unload_ramcache(void)
4269 {
4270 tc_stat.ramcache = false;
4271 /* Just to make sure there is no statefile present. */
4272 // remove(TAGCACHE_STATEFILE);
4273 }
4274 #endif
4275
4276 #ifndef __PCTOOL__
4277 static void tagcache_thread(void)
4278 {
4279 struct queue_event ev;
4280 bool check_done = false;
4281
4282 /* If the previous cache build/update was interrupted, commit
4283 * the changes first in foreground. */
4284 cpu_boost(true);
4285 allocate_tempbuf();
4286 commit();
4287 free_tempbuf();
4288
4289 #ifdef HAVE_TC_RAMCACHE
4290 # ifdef HAVE_EEPROM_SETTINGS
4291 if (firmware_settings.initialized && firmware_settings.disk_clean)
4292 check_done = tagcache_dumpload();
4293
4294 remove(TAGCACHE_STATEFILE);
4295 # endif
4296
4297 /* Allocate space for the tagcache if found on disk. */
4298 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4299 allocate_tagcache();
4300 #endif
4301
4302 cpu_boost(false);
4303 tc_stat.initialized = true;
4304
4305 /* Don't delay bootup with the header check but do it on background. */
4306 sleep(HZ);
4307 tc_stat.ready = check_all_headers();
4308 tc_stat.readyvalid = true;
4309
4310 while (1)
4311 {
4312 run_command_queue(false);
4313
4314 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4315
4316 switch (ev.id)
4317 {
4318 case Q_IMPORT_CHANGELOG:
4319 tagcache_import_changelog();
4320 break;
4321
4322 case Q_REBUILD:
4323 remove_files();
4324 build_tagcache("/");
4325 break;
4326
4327 case Q_UPDATE:
4328 build_tagcache("/");
4329 #ifdef HAVE_TC_RAMCACHE
4330 load_ramcache();
4331 #endif
4332 check_deleted_files();
4333 break ;
4334
4335 case Q_START_SCAN:
4336 check_done = false;
4337 case SYS_TIMEOUT:
4338 if (check_done || !tc_stat.ready)
4339 break ;
4340
4341 #ifdef HAVE_TC_RAMCACHE
4342 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4343 {
4344 load_ramcache();
4345 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4346 build_tagcache("/");
4347 }
4348 else
4349 #endif
4350 if (global_settings.tagcache_autoupdate)
4351 {
4352 build_tagcache("/");
4353
4354 /* This will be very slow unless dircache is enabled
4355 or target is flash based, but do it anyway for
4356 consistency. */
4357 check_deleted_files();
4358 }
4359
4360 logf("tagcache check done");
4361
4362 check_done = true;
4363 break ;
4364
4365 case Q_STOP_SCAN:
4366 break ;
4367
4368 case SYS_POWEROFF:
4369 break ;
4370
4371 #ifndef SIMULATOR
4372 case SYS_USB_CONNECTED:
4373 logf("USB: TagCache");
4374 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4375 usb_wait_for_disconnect(&tagcache_queue);
4376 break ;
4377 #endif
4378 }
4379 }
4380 }
4381
4382 bool tagcache_prepare_shutdown(void)
4383 {
4384 if (tagcache_get_commit_step() > 0)
4385 return false;
4386
4387 tagcache_stop_scan();
4388 while (read_lock || write_lock)
4389 sleep(1);
4390
4391 return true;
4392 }
4393
4394 void tagcache_shutdown(void)
4395 {
4396 /* Flush the command queue. */
4397 run_command_queue(true);
4398
4399 #ifdef HAVE_EEPROM_SETTINGS
4400 if (tc_stat.ramcache)
4401 tagcache_dumpsave();
4402 #endif
4403 }
4404
4405 static int get_progress(void)
4406 {
4407 int total_count = -1;
4408
4409 #ifdef HAVE_DIRCACHE
4410 if (dircache_is_enabled())
4411 {
4412 total_count = dircache_get_entry_count();
4413 }
4414 else
4415 #endif
4416 #ifdef HAVE_TC_RAMCACHE
4417 {
4418 if (hdr && tc_stat.ramcache)
4419 total_count = hdr->h.tch.entry_count;
4420 }
4421 #endif
4422
4423 if (total_count < 0)
4424 return -1;
4425
4426 return processed_dir_count * 100 / total_count;
4427 }
4428
4429 struct tagcache_stat* tagcache_get_stat(void)
4430 {
4431 tc_stat.progress = get_progress();
4432 tc_stat.processed_entries = processed_dir_count;
4433
4434 return &tc_stat;
4435 }
4436
4437 void tagcache_start_scan(void)
4438 {
4439 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4440 }
4441
4442 bool tagcache_update(void)
4443 {
4444 if (!tc_stat.ready)
4445 return false;
4446
4447 queue_post(&tagcache_queue, Q_UPDATE, 0);
4448 return false;
4449 }
4450
4451 bool tagcache_rebuild()
4452 {
4453 queue_post(&tagcache_queue, Q_REBUILD, 0);
4454 return false;
4455 }
4456
4457 void tagcache_stop_scan(void)
4458 {
4459 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4460 }
4461
4462 #ifdef HAVE_TC_RAMCACHE
4463 bool tagcache_is_ramcache(void)
4464 {
4465 return tc_stat.ramcache;
4466 }
4467 #endif
4468
4469 #endif /* !__PCTOOL__ */
4470
4471
4472 void tagcache_init(void)
4473 {
4474 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4475 memset(&current_tcmh, 0, sizeof(struct master_header));
4476 filenametag_fd = -1;
4477 write_lock = read_lock = 0;
4478
4479 #ifndef __PCTOOL__
4480 mutex_init(&command_queue_mutex);
4481 queue_init(&tagcache_queue, true);
4482 create_thread(tagcache_thread, tagcache_stack,
4483 sizeof(tagcache_stack), 0, tagcache_thread_name
4484 IF_PRIO(, PRIORITY_BACKGROUND)
4485 IF_COP(, CPU));
4486 #else
4487 tc_stat.initialized = true;
4488 allocate_tempbuf();
4489 commit();
4490 free_tempbuf();
4491 tc_stat.ready = check_all_headers();
4492 #endif
4493 }
4494
4495 #ifdef __PCTOOL__
4496 void tagcache_reverse_scan(void)
4497 {
4498 logf("Checking for deleted files");
4499 check_deleted_files();
4500 }
4501 #endif
4502
4503 bool tagcache_is_initialized(void)
4504 {
4505 return tc_stat.initialized;
4506 }
4507 bool tagcache_is_usable(void)
4508 {
4509 return tc_stat.initialized && tc_stat.ready;
4510 }
4511 int tagcache_get_commit_step(void)
4512 {
4513 return tc_stat.commit_step;
4514 }
4515 int tagcache_get_max_commit_step(void)
4516 {
4517 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4518 }
4519
My public Rockbox git repo