apps/tagcache.c

/***************************************************************************
 *             __________               __   ___.
 *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
 *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
 *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
 *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
 *                     \/            \/     \/    \/            \/
 * $Id$
 *
 * Copyright (C) 2005 by Miika Pekkarinen
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ****************************************************************************/

/*
 *                    TagCache API
 * 
 *       ----------x---------x------------------x-----
 *                 |         |                  |              External
 * +---------------x-------+ |       TagCache   |              Libraries
 * | Modification routines | |         Core     |           
 * +-x---------x-----------+ |                  |           
 *   | (R/W)   |             |                  |           |
 *   |  +------x-------------x-+  +-------------x-----+     |
 *   |  |                      x==x Filters & clauses |     |
 *   |  | Search routines      |  +-------------------+     |
 *   |  |                      x============================x DirCache
 *   |  +-x--------------------+                            | (optional)
 *   |    | (R)                                             |
 *   |    | +-------------------------------+  +---------+  |
 *   |    | | DB Commit (sort,unique,index) |  |         |  |
 *   |    | +-x--------------------------x--+  | Control |  |
 *   |    |   | (R/W)                    | (R) | Thread  |  |
 *   |    |   | +----------------------+ |     |         |  |
 *   |    |   | | TagCache DB Builder  | |     +---------+  |
 *   |    |   | +-x-------------x------+ |                  |
 *   |    |   |   | (R)         | (W)    |                  |
 *   |    |   |   |          +--x--------x---------+        |
 *   |    |   |   |          | Temporary Commit DB |        |
 *   |    |   |   |          +---------------------+        |
 * +-x----x-------x--+                                      |
 * | TagCache RAM DB x==\(W) +-----------------+            |
 * +-----------------+   \===x                 |            |
 *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
 * +-x----x---x---x---+   /==x                 |            | (optional)
 * | Tagcache Disk DB x==/   +-----------------+            |  
 * +------------------+                                     |
 * 
 */

#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include "config.h"
#include "ata_idle_notify.h"
#include "thread.h"
#include "kernel.h"
#include "system.h"
#include "logf.h"
#include "string.h"
#include "usb.h"
#include "metadata.h"
#include "id3.h"
#include "tagcache.h"
#include "buffer.h"
#include "crc32.h"
#include "misc.h"
#include "settings.h"
#include "dir.h"
#include "structec.h"
#include "tagcache.h"

#ifndef __PCTOOL__
#include "splash.h"
#include "lang.h"
#include "eeprom_settings.h"
#endif

#ifdef __PCTOOL__
#define yield() do { } while(0)
#define sim_sleep(timeout) do { } while(0)
#define do_timed_yield() do { } while(0)
#endif

#ifndef __PCTOOL__
/* Tag Cache thread. */
static struct event_queue tagcache_queue;
static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
static const char tagcache_thread_name[] = "tagcache";
#endif

#define UNTAGGED "<Untagged>"

/* Previous path when scanning directory tree recursively. */
static char curpath[TAG_MAXLEN+32];
static long curpath_size = sizeof(curpath);

/* Used when removing duplicates. */
static char *tempbuf;     /* Allocated when needed. */
static long tempbufidx;   /* Current location in buffer. */
static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
static long tempbuf_left; /* Buffer space left. */
static long tempbuf_pos;

/* Tags we want to get sorted (loaded to the tempbuf). */
static const int sorted_tags[] = { tag_artist, tag_album, tag_genre, 
    tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };

/* Uniqued tags (we can use these tags with filters and conditional clauses). */
static const int unique_tags[] = { tag_artist, tag_album, tag_genre, 
    tag_composer, tag_comment, tag_albumartist, tag_grouping };

/* Numeric tags (we can use these tags with conditional clauses). */
static const int numeric_tags[] = { tag_year, tag_discnumber, 
    tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
    tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
    tag_virt_length_min, tag_virt_length_sec,
    tag_virt_playtime_min, tag_virt_playtime_sec,
    tag_virt_entryage, tag_virt_autoscore };

/* String presentation of the tags defined in tagcache.h. Must be in correct order! */
static const char *tags_str[] = { "artist", "album", "genre", "title", 
    "filename", "composer", "comment", "albumartist", "grouping", "year", 
    "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating", 
    "playtime", "lastplayed", "commitid", "mtime" };

/* Status information of the tagcache. */
static struct tagcache_stat tc_stat;

/* Queue commands. */
enum tagcache_queue {
    Q_STOP_SCAN = 0,
    Q_START_SCAN,
    Q_IMPORT_CHANGELOG,
    Q_UPDATE,
    Q_REBUILD,
    
    /* Internal tagcache command queue. */
    CMD_UPDATE_MASTER_HEADER,
    CMD_UPDATE_NUMERIC,
};

struct tagcache_command_entry {
    int32_t command;
    int32_t idx_id;
    int32_t tag;
    int32_t data;
};

static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
static volatile int command_queue_widx = 0;
static volatile int command_queue_ridx = 0;
static struct mutex command_queue_mutex;

/* Tag database structures. */

/* Variable-length tag entry in tag files. */
struct tagfile_entry {
    short tag_length;  /* Length of the data in bytes including '\0' */
    short idx_id;      /* Corresponding entry location in index file of not unique tags */
    char tag_data[0];  /* Begin of the tag data */
};

/* Fixed-size tag entry in master db index. */
struct index_entry {
    int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
    int32_t flag;                /* Status flags */
};

/* Header is the same in every file. */
struct tagcache_header {
    int32_t magic;       /* Header version number */
    int32_t datasize;    /* Data size in bytes */
    int32_t entry_count; /* Number of entries in this file */
};

struct master_header {
    struct tagcache_header tch;
    int32_t serial; /* Increasing counting number */
    int32_t commitid; /* Number of commits so far */
    int32_t dirty;
};

/* For the endianess correction */
static const char *tagfile_entry_ec   = "ss";
static const char *index_entry_ec     = "lllllllllllllllllllll"; /* (1 + TAG_COUNT) * l */
static const char *tagcache_header_ec = "lll";
static const char *master_header_ec   = "llllll";

static struct master_header current_tcmh;

#ifdef HAVE_TC_RAMCACHE
/* Header is created when loading database to ram. */
struct ramcache_header {
    struct master_header h;      /* Header from the master index */
    struct index_entry *indices; /* Master index file content */
    char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
    int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
};

# ifdef HAVE_EEPROM_SETTINGS
struct statefile_header {
    struct ramcache_header *hdr;
    struct tagcache_stat tc_stat;
};
# endif

/* Pointer to allocated ramcache_header */
static struct ramcache_header *hdr;
#endif

/** 
 * Full tag entries stored in a temporary file waiting 
 * for commit to the cache. */
struct temp_file_entry {
    long tag_offset[TAG_COUNT];
    short tag_length[TAG_COUNT];
    long flag;
    
    long data_length;
};

struct tempbuf_id_list {
    long id;
    struct tempbuf_id_list *next;
};

struct tempbuf_searchidx {
    long idx_id;
    char *str;
    int seek;
    struct tempbuf_id_list idlist;
};

/* Lookup buffer for fixing messed up index while after sorting. */
static long commit_entry_count;
static long lookup_buffer_depth;
static struct tempbuf_searchidx **lookup;

/* Used when building the temporary file. */
static int cachefd = -1, filenametag_fd;
static int total_entry_count = 0;
static int data_size = 0;
static int processed_dir_count;

/* Thread safe locking */
static volatile int write_lock;
static volatile int read_lock;

static bool delete_entry(long idx_id);

const char* tagcache_tag_to_str(int tag)
{
    return tags_str[tag];
}

bool tagcache_is_numeric_tag(int type)
{
    int i;

    for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
    {
        if (type == numeric_tags[i])
            return true;
    }

    return false;
}

bool tagcache_is_unique_tag(int type)
{
    int i;

    for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
    {
        if (type == unique_tags[i])
            return true;
    }

    return false;
}

bool tagcache_is_sorted_tag(int type)
{
    int i;

    for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
    {
        if (type == sorted_tags[i])
            return true;
    }

    return false;
}

#ifdef HAVE_DIRCACHE
/**
 * Returns true if specified flag is still present, i.e., dircache
 * has not been reloaded.
 */
static bool is_dircache_intact(void)
{
    return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
}
#endif

static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
{
    int fd;
    char buf[MAX_PATH];
    int rc;
    
    if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
        return -1;
    
    snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
    
    fd = open(buf, write ? O_RDWR : O_RDONLY);
    if (fd < 0)
    {
        logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
        tc_stat.ready = false;
        return fd;
    }
    
    /* Check the header. */
    rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
    if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
    {
        logf("header error");
        tc_stat.ready = false;
        close(fd);
        return -2;
    }

    return fd;
}

static int open_master_fd(struct master_header *hdr, bool write)
{
    int fd;
    int rc;
    
    fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
    if (fd < 0)
    {
        logf("master file open failed for R/W");
        tc_stat.ready = false;
        return fd;
    }
    
    tc_stat.econ = false;
    
    /* Check the header. */
    rc = read(fd, hdr, sizeof(struct master_header));
    if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
    {
        /* Success. */
        return fd;
    }
    
    /* Trying to read again, this time with endianess correction enabled. */
    lseek(fd, 0, SEEK_SET);
    
    rc = ecread(fd, hdr, 1, master_header_ec, true);
    if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
    {
        logf("header error");
        tc_stat.ready = false;
        close(fd);
        return -2;
    }

    tc_stat.econ = true;
    
    return fd;
}

#ifndef __PCTOOL__
static bool do_timed_yield(void)
{
    /* Sorting can lock up for quite a while, so yield occasionally */
    static long wakeup_tick = 0;
    if (current_tick >= wakeup_tick)
    {
        wakeup_tick = current_tick + (HZ/4);
        yield();
        return true;
    }
    return false;
}
#endif

#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
static long find_entry_ram(const char *filename,
                           const struct dirent *dc)
{
    static long last_pos = 0;
    int i;
    
    /* Check if we tagcache is loaded into ram. */
    if (!tc_stat.ramcache)
        return -1;

    if (dc == NULL)
        dc = dircache_get_entry_ptr(filename);
    
    if (dc == NULL)
    {
        logf("tagcache: file not found.");
        return -1;
    }

    try_again:
            
    if (last_pos > 0)
        i = last_pos;
    else
        i = 0;
    
    for (; i < hdr->h.tch.entry_count; i++)
    {
        if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
        {
            last_pos = MAX(0, i - 3);
            return i;
        }
        
        do_timed_yield();
    }

    if (last_pos > 0)
    {
        last_pos = 0;
        goto try_again;
    }

    return -1;
}
#endif

static long find_entry_disk(const char *filename)
{
    struct tagcache_header tch;
    static long last_pos = -1;
    long pos_history[POS_HISTORY_COUNT];
    long pos_history_idx = 0;
    bool found = false;
    struct tagfile_entry tfe;
    int fd;
    char buf[TAG_MAXLEN+32];
    int i;
    int pos = -1;

    if (!tc_stat.ready)
        return -2;
    
    fd = filenametag_fd;
    if (fd < 0)
    {
        last_pos = -1;
        if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
            return -1;
    }
    
    check_again:
    
    if (last_pos > 0)
        lseek(fd, last_pos, SEEK_SET);
    else
        lseek(fd, sizeof(struct tagcache_header), SEEK_SET);

    while (true)
    {
        pos = lseek(fd, 0, SEEK_CUR);
        for (i = pos_history_idx-1; i >= 0; i--)
            pos_history[i+1] = pos_history[i];
        pos_history[0] = pos;

        if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ) 
            != sizeof(struct tagfile_entry))
        {
            break ;
        }
        
        if (tfe.tag_length >= (long)sizeof(buf))
        {
            logf("too long tag #1");
            close(fd);
            last_pos = -1;
            return -2;
        }
        
        if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
        {
            logf("read error #2");
            close(fd);
            last_pos = -1;
            return -3;
        }
        
        if (!strcasecmp(filename, buf))
        {
            last_pos = pos_history[pos_history_idx];
            found = true;
            break ;
        }
        
        if (pos_history_idx < POS_HISTORY_COUNT - 1)
            pos_history_idx++;
    }
    
    /* Not found? */
    if (!found)
    {
        if (last_pos > 0)
        {
            last_pos = -1;
            logf("seek again");
            goto check_again;
        }
        
        if (fd != filenametag_fd)
            close(fd);
        return -4;
    }
    
    if (fd != filenametag_fd)
        close(fd);
        
    return tfe.idx_id;
}

static int find_index(const char *filename)
{
    long idx_id = -1;
    
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
    if (tc_stat.ramcache && is_dircache_intact())
        idx_id = find_entry_ram(filename, NULL);
#endif
    
    if (idx_id < 0)
        idx_id = find_entry_disk(filename);
    
    return idx_id;
}

bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
{
    int idx_id;
    
    if (!tc_stat.ready)
        return false;
    
    idx_id = find_index(filename);
    if (idx_id < 0)
        return false;
    
    if (!tagcache_search(tcs, tag_filename))
        return false;
    
    tcs->entry_count = 0;
    tcs->idx_id = idx_id;
    
    return true;
}

static bool get_index(int masterfd, int idxid, 
                      struct index_entry *idx, bool use_ram)
{
    bool localfd = false;
    
    if (idxid < 0)
    {
        logf("Incorrect idxid: %d", idxid);
        return false;
    }
    
#ifdef HAVE_TC_RAMCACHE
    if (tc_stat.ramcache && use_ram)
    {
        if (hdr->indices[idxid].flag & FLAG_DELETED)
            return false;
        
        memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
        return true;
    }
#else
    (void)use_ram;
#endif
    
    if (masterfd < 0)
    {
        struct master_header tcmh;
        
        localfd = true;
        masterfd = open_master_fd(&tcmh, false);
        if (masterfd < 0)
            return false;
    }
    
    lseek(masterfd, idxid * sizeof(struct index_entry) 
          + sizeof(struct master_header), SEEK_SET);
    if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ) 
        != sizeof(struct index_entry))
    {
        logf("read error #3");
        if (localfd)
            close(masterfd);
        
        return false;
    }
    
    if (localfd)
        close(masterfd);
    
    if (idx->flag & FLAG_DELETED)
        return false;
    
    return true;
}

static bool write_index(int masterfd, int idxid, struct index_entry *idx)
{
    /* We need to exclude all memory only flags & tags when writing to disk. */
    if (idx->flag & FLAG_DIRCACHE)
    {
        logf("memory only flags!");
        return false;
    }
    
#ifdef HAVE_TC_RAMCACHE
    /* Only update numeric data. Writing the whole index to RAM by memcpy
     * destroys dircache pointers!
     */
    if (tc_stat.ramcache)
    {
        int tag;
        struct index_entry *idx_ram = &hdr->indices[idxid];
        
        for (tag = 0; tag < TAG_COUNT; tag++)
        {
            if (tagcache_is_numeric_tag(tag))
            {
                idx_ram->tag_seek[tag] = idx->tag_seek[tag];
            }
        }
        
        /* Don't touch the dircache flag. */
        idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
    }
#endif
    
    lseek(masterfd, idxid * sizeof(struct index_entry) 
          + sizeof(struct master_header), SEEK_SET);
    if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ) 
        != sizeof(struct index_entry))
    {
        logf("write error #3");
        logf("idxid: %d", idxid);
        return false;
    }
    
    return true;
}

static bool open_files(struct tagcache_search *tcs, int tag)
{
    if (tcs->idxfd[tag] < 0)
    {
        char fn[MAX_PATH];

        snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
        tcs->idxfd[tag] = open(fn, O_RDONLY);
    }
    
    if (tcs->idxfd[tag] < 0)
    {
        logf("File not open!");
        return false;
    }
    
    return true;
}

static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx, 
                     int tag, char *buf, long size)
{
    struct tagfile_entry tfe;
    long seek;
    
    *buf = '\0';
    
    if (tagcache_is_numeric_tag(tag))
        return false;
    
    seek = idx->tag_seek[tag];
    if (seek < 0)
    {
        logf("Retrieve failed");
        return false;
    }
    
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch)
    {
        struct tagfile_entry *ep;
        
# ifdef HAVE_DIRCACHE
        if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
            && is_dircache_intact())
        {
            dircache_copy_path((struct dirent *)seek,
                               buf, size);
            return true;
        }
        else
# endif
        if (tag != tag_filename)
        {
            ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
            strncpy(buf, ep->tag_data, size-1);
            
            return true;
        }
    }
#endif
    
    if (!open_files(tcs, tag))
        return false;
    
    lseek(tcs->idxfd[tag], seek, SEEK_SET);
    if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ) 
        != sizeof(struct tagfile_entry))
    {
        logf("read error #5");
        return false;
    }
    
    if (tfe.tag_length >= size)
    {
        logf("too small buffer");
        return false;
    }
    
    if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
        tfe.tag_length)
    {
        logf("read error #6");
        return false;
    }
    
    buf[tfe.tag_length] = '\0';
    
    return true;
}

static long check_virtual_tags(int tag, const struct index_entry *idx)
{
    long data = 0;
    
    switch (tag) 
    {
        case tag_virt_length_sec:
            data = (idx->tag_seek[tag_length]/1000) % 60;
            break;
        
        case tag_virt_length_min:
            data = (idx->tag_seek[tag_length]/1000) / 60;
            break;
        
        case tag_virt_playtime_sec:
            data = (idx->tag_seek[tag_playtime]/1000) % 60;
            break;
        
        case tag_virt_playtime_min:
            data = (idx->tag_seek[tag_playtime]/1000) / 60;
            break;
        
        case tag_virt_autoscore:
            if (idx->tag_seek[tag_length] == 0 
                || idx->tag_seek[tag_playcount] == 0)
            {
                data = 0;
            }
            else
            {
                data = 100 * idx->tag_seek[tag_playtime]
                    / idx->tag_seek[tag_length]
                    / idx->tag_seek[tag_playcount];
            }
            break;
        
        /* How many commits before the file has been added to the DB. */
        case tag_virt_entryage:
            data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
            break;
        
        default:
            data = idx->tag_seek[tag];
    }
    
    return data;
}

long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
{
    struct index_entry idx;
    
    if (!tc_stat.ready)
        return false;
    
    if (!tagcache_is_numeric_tag(tag))
        return -1;
    
    if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
        return -2;
    
    return check_virtual_tags(tag, &idx);
}

inline static bool str_ends_with(const char *str1, const char *str2)
{
    int str_len = strlen(str1);
    int clause_len = strlen(str2);
    
    if (clause_len > str_len)
        return false;
    
    return !strcasecmp(&str1[str_len - clause_len], str2);
}

inline static bool str_oneof(const char *str, const char *list)
{
    const char *sep;
    int l, len = strlen(str);
    
    while (*list)
    {
        sep = strchr(list, '|');
        l = sep ? (long)sep - (long)list : (int)strlen(list);
        if ((l==len) && !strncasecmp(str, list, len))
            return true;
        list += sep ? l + 1 : l;
    }

    return false;
}

static bool check_against_clause(long numeric, const char *str,
                                 const struct tagcache_search_clause *clause)
{
    if (clause->numeric)
    {
        switch (clause->type)
        {
            case clause_is:
                return numeric == clause->numeric_data;
            case clause_is_not:
                return numeric != clause->numeric_data;
            case clause_gt:
                return numeric > clause->numeric_data;
            case clause_gteq:
                return numeric >= clause->numeric_data;
            case clause_lt:
                return numeric < clause->numeric_data;
            case clause_lteq:
                return numeric <= clause->numeric_data;        
            default:
                logf("Incorrect numeric tag: %d", clause->type);
        }
    }
    else
    {
        switch (clause->type)
        {
            case clause_is:
                return !strcasecmp(clause->str, str);
            case clause_is_not:
                return strcasecmp(clause->str, str);
            case clause_gt:
                return 0>strcasecmp(clause->str, str);
            case clause_gteq:
                return 0>=strcasecmp(clause->str, str);
            case clause_lt:
                return 0<strcasecmp(clause->str, str);
            case clause_lteq:
                return 0<=strcasecmp(clause->str, str);
            case clause_contains:
                return (strcasestr(str, clause->str) != NULL);
            case clause_not_contains:
                return (strcasestr(str, clause->str) == NULL);
            case clause_begins_with:
                return (strcasestr(str, clause->str) == str);
            case clause_not_begins_with:
                return (strcasestr(str, clause->str) != str);
            case clause_ends_with:
                return str_ends_with(str, clause->str);
            case clause_not_ends_with:
                return !str_ends_with(str, clause->str);
            case clause_oneof:
                return str_oneof(str, clause->str);
            
            default:
                logf("Incorrect tag: %d", clause->type);
        }
    }

    return false;
}

static bool check_clauses(struct tagcache_search *tcs,
                          struct index_entry *idx,
                          struct tagcache_search_clause **clause, int count)
{
    int i;
    
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch)
    {
        /* Go through all conditional clauses. */
        for (i = 0; i < count; i++)
        {
            struct tagfile_entry *tfe;
            int seek;
            char buf[256];
            char *str = NULL;
            
            seek = check_virtual_tags(clause[i]->tag, idx);
            
            if (!tagcache_is_numeric_tag(clause[i]->tag))
            {
                if (clause[i]->tag == tag_filename)
                {
                    retrieve(tcs, idx, tag_filename, buf, sizeof buf);
                    str = buf;
                }
                else
                {
                    tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
                    str = tfe->tag_data;
                }
            }
        
            if (!check_against_clause(seek, str, clause[i]))
                return false;
        }
    }
    else
#endif
    {
        /* Check for conditions. */
        for (i = 0; i < count; i++)
        {
            struct tagfile_entry tfe;
            int seek;
            char str[256];
            
            seek = check_virtual_tags(clause[i]->tag, idx);
                
            memset(str, 0, sizeof str);
            if (!tagcache_is_numeric_tag(clause[i]->tag))
            {
                int fd = tcs->idxfd[clause[i]->tag];
                lseek(fd, seek, SEEK_SET);
                ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
                if (tfe.tag_length >= (int)sizeof(str))
                {
                    logf("Too long tag read!");
                    break ;
                }

                read(fd, str, tfe.tag_length);
                
                /* Check if entry has been deleted. */
                if (str[0] == '\0')
                    break;
            }
            
            if (!check_against_clause(seek, str, clause[i]))
                return false;
        }
    }
    
    return true;
}

bool tagcache_check_clauses(struct tagcache_search *tcs,
                            struct tagcache_search_clause **clause, int count)
{
    struct index_entry idx;
    
    if (count == 0)
        return true;
    
    if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
        return false;
    
    return check_clauses(tcs, &idx, clause, count);
}

static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
{
    int i;
    
    /* If uniq buffer is not defined we must return true for search to work. */
    if (tcs->unique_list == NULL 
        || (!tagcache_is_unique_tag(tcs->type)
            && !tagcache_is_numeric_tag(tcs->type)))
    {
        return true;
    }
    
    for (i = 0; i < tcs->unique_list_count; i++)
    {
        /* Return false if entry is found. */
        if (tcs->unique_list[i] == id)
            return false;
    }
    
    if (tcs->unique_list_count < tcs->unique_list_capacity)
    {
        tcs->unique_list[i] = id;
        tcs->unique_list_count++;
    }
    
    return true;
}

static bool build_lookup_list(struct tagcache_search *tcs)
{
    struct index_entry entry;
    int i;
    
    tcs->seek_list_count = 0;
    
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch)
    {
        int j;

        for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
        {
            struct index_entry *idx = &hdr->indices[i];
            if (tcs->seek_list_count == SEEK_LIST_SIZE)
                break ;
            
            /* Skip deleted files. */
            if (idx->flag & FLAG_DELETED)
                continue;
            
            /* Go through all filters.. */
            for (j = 0; j < tcs->filter_count; j++)
            {
                if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
                {
                    break ;
                }
            }
            
            if (j < tcs->filter_count)
                continue ;

            /* Check for conditions. */
            if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
                continue;
            
            /* Add to the seek list if not already in uniq buffer. */
            if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
                continue;
            
            /* Lets add it. */
            tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
            tcs->seek_flags[tcs->seek_list_count] = idx->flag;
            tcs->seek_list_count++;
        }
        
        tcs->seek_pos = i;

        return tcs->seek_list_count > 0;
    }
#endif
    
    lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
            sizeof(struct master_header), SEEK_SET);
    
    while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ) 
           == sizeof(struct index_entry))
    {
        if (tcs->seek_list_count == SEEK_LIST_SIZE)
            break ;

        tcs->seek_pos++;
        
        /* Check if entry has been deleted. */
        if (entry.flag & FLAG_DELETED)
            continue;
        
        /* Go through all filters.. */
        for (i = 0; i < tcs->filter_count; i++)
        {
            if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
                break ;
        }
        
        if (i < tcs->filter_count)
            continue ;
        
        /* Check for conditions. */
        if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
            continue;
            
        /* Add to the seek list if not already in uniq buffer. */
        if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
            continue;
            
        /* Lets add it. */
        tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
        tcs->seek_flags[tcs->seek_list_count] = entry.flag;
        tcs->seek_list_count++;
        
        yield();
    }

    return tcs->seek_list_count > 0;
}

        
static void remove_files(void)
{
    int i;
    char buf[MAX_PATH];
    
    tc_stat.ready = false;
    tc_stat.ramcache = false;
    tc_stat.econ = false;
    remove(TAGCACHE_FILE_MASTER);
    for (i = 0; i < TAG_COUNT; i++)
    {
        if (tagcache_is_numeric_tag(i))
            continue;
        
        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
        remove(buf);
    }
}


static bool check_all_headers(void)
{
    struct master_header myhdr;
    struct tagcache_header tch;
    int tag;
    int fd;
    
    if ( (fd = open_master_fd(&myhdr, false)) < 0)
        return false;
    
    close(fd);
    if (myhdr.dirty)
    {
        logf("tagcache is dirty!");
        return false;
    }
    
    memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
    
    for (tag = 0; tag < TAG_COUNT; tag++)
    {
        if (tagcache_is_numeric_tag(tag))
            continue;
        
        if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
            return false;
        
        close(fd);
    }
    
    return true;
}

bool tagcache_search(struct tagcache_search *tcs, int tag)
{
    struct tagcache_header tag_hdr;
    struct master_header   master_hdr;
    int i;

    if (tcs->initialized)
        tagcache_search_finish(tcs);
    
    while (read_lock)
        sleep(1);
    
    memset(tcs, 0, sizeof(struct tagcache_search));
    if (tc_stat.commit_step > 0 || !tc_stat.ready)
        return false;
    
    tcs->position = sizeof(struct tagcache_header);
    tcs->type = tag;
    tcs->seek_pos = 0;
    tcs->seek_list_count = 0;
    tcs->filter_count = 0;
    tcs->masterfd = -1;

    for (i = 0; i < TAG_COUNT; i++)
        tcs->idxfd[i] = -1;

#ifndef HAVE_TC_RAMCACHE
    tcs->ramsearch = false;
#else
    tcs->ramsearch = tc_stat.ramcache;
    if (tcs->ramsearch)
    {
        tcs->entry_count = hdr->entry_count[tcs->type];
    }
    else
#endif
    {
        if (!tagcache_is_numeric_tag(tcs->type))
        {
            tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
            if (tcs->idxfd[tcs->type] < 0)
                return false;
        }
        
        /* Always open as R/W so we can pass tcs to functions that modify data also
         * without failing. */
        tcs->masterfd = open_master_fd(&master_hdr, true);
        
        if (tcs->masterfd < 0)
            return false;
    }

    tcs->valid = true;
    tcs->initialized = true;
    write_lock++;
    
    return true;
}

void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
                                 void *buffer, long length)
{
    tcs->unique_list = (unsigned long *)buffer;
    tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
    tcs->unique_list_count = 0;
}

bool tagcache_search_add_filter(struct tagcache_search *tcs,
                                int tag, int seek)
{
    if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
        return false;

    if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
        return false;
    
    tcs->filter_tag[tcs->filter_count] = tag;
    tcs->filter_seek[tcs->filter_count] = seek;
    tcs->filter_count++;

    return true;
}

bool tagcache_search_add_clause(struct tagcache_search *tcs,
                                struct tagcache_search_clause *clause)
{
    int i;
    
    if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
    {
        logf("Too many clauses");
        return false;
    }

    /* Check if there is already a similar filter in present (filters are
     * much faster than clauses). 
     */
    for (i = 0; i < tcs->filter_count; i++)
    {
        if (tcs->filter_tag[i] == clause->tag)
            return true;
    }
    
    if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
    {
        char buf[MAX_PATH];

        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);        
        tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
    }
    
    tcs->clause[tcs->clause_count] = clause;
    tcs->clause_count++;
    
    return true;
}

/* TODO: Remove this mess. */
#ifdef HAVE_DIRCACHE
#define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
    ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
#else
#define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
#endif

static bool get_next(struct tagcache_search *tcs)
{
    static char buf[TAG_MAXLEN+32];
    struct tagfile_entry entry;
    long flag = 0;

    if (!tcs->valid || !tc_stat.ready)
        return false;
        
    if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
#ifdef HAVE_TC_RAMCACHE
        && !tcs->ramsearch
#endif
        )
        return false;
    
    /* Relative fetch. */
    if (tcs->filter_count > 0 || tcs->clause_count > 0
        || tagcache_is_numeric_tag(tcs->type))
    {
        /* Check for end of list. */
        if (tcs->seek_list_count == 0)
        {
            /* Try to fetch more. */
            if (!build_lookup_list(tcs))
            {
                tcs->valid = false;
                return false;
            }
        }
        
        tcs->seek_list_count--;
        flag = tcs->seek_flags[tcs->seek_list_count];
        
        /* Seek stream to the correct position and continue to direct fetch. */
        if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
            && !tagcache_is_numeric_tag(tcs->type))
        {
            if (!open_files(tcs, tcs->type))
                return false;
        
            lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
        }
        else
            tcs->position = tcs->seek_list[tcs->seek_list_count];
    }
    
    if (tagcache_is_numeric_tag(tcs->type))
    {
        snprintf(buf, sizeof(buf), "%d", tcs->position);
        tcs->result_seek = tcs->position;
        tcs->result = buf;
        tcs->result_len = strlen(buf) + 1;
        return true;
    }
    
    /* Direct fetch. */
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
    {
        struct tagfile_entry *ep;
        
        if (tcs->entry_count == 0)
        {
            tcs->valid = false;
            return false;
        }
        tcs->entry_count--;
        
        tcs->result_seek = tcs->position;

# ifdef HAVE_DIRCACHE
        if (tcs->type == tag_filename)
        {
            dircache_copy_path((struct dirent *)tcs->position,
                               buf, sizeof buf);
            tcs->result = buf;
            tcs->result_len = strlen(buf) + 1;
            tcs->idx_id = FLAG_GET_ATTR(flag);
            tcs->ramresult = false;
            
            return true;
        }
# endif
        
        ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
        tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
        tcs->result = ep->tag_data;
        tcs->result_len = strlen(tcs->result) + 1;
        tcs->idx_id = ep->idx_id;
        tcs->ramresult = true;
        
        return true;
    }
    else
#endif
    {
        if (!open_files(tcs, tcs->type))
            return false;
        
        tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
        if (ecread(tcs->idxfd[tcs->type], &entry, 1,
                 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
        {
            /* End of data. */
            tcs->valid = false;
            return false;
        }
    }
    
    if (entry.tag_length > (long)sizeof(buf))
    {
        tcs->valid = false;
        logf("too long tag #2");
        return false;
    }
    
    if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
    {
        tcs->valid = false;
        logf("read error #4");
        return false;
    }
    
    tcs->result = buf;
    tcs->result_len = strlen(tcs->result) + 1;
    tcs->idx_id = entry.idx_id;
    tcs->ramresult = false;

    return true;
}

bool tagcache_get_next(struct tagcache_search *tcs)
{
    while (get_next(tcs))
    {
        if (tcs->result_len > 1)
            return true;
    }
    
    return false;
}

bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, 
                       int tag, char *buf, long size)
{
    struct index_entry idx;
    
    *buf = '\0';
    if (!get_index(tcs->masterfd, idxid, &idx, true))
        return false;
    
    return retrieve(tcs, &idx, tag, buf, size);
}

static bool update_master_header(void)
{
    struct master_header myhdr;
    int fd;
    
    if (!tc_stat.ready)
        return false;
    
    if ( (fd = open_master_fd(&myhdr, true)) < 0)
        return false;
    
    myhdr.serial = current_tcmh.serial;
    myhdr.commitid = current_tcmh.commitid;
    
    /* Write it back */
    lseek(fd, 0, SEEK_SET);
    ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
    close(fd);
    
#ifdef HAVE_TC_RAMCACHE
    if (hdr)
    {
        hdr->h.serial = current_tcmh.serial;
        hdr->h.commitid = current_tcmh.commitid;
    }
#endif
    
    return true;
}

#if 0

void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
{
    struct tagentry *entry;
    
    if (tcs->type != tag_title)
        return ;

    /* We will need reserve buffer for this. */
    if (tcs->ramcache)
    {
        struct tagfile_entry *ep;

        ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
        tcs->seek_list[tcs->seek_list_count];
    }

    entry = find_entry_ram();
    
}
#endif

void tagcache_search_finish(struct tagcache_search *tcs)
{
    int i;
    
    if (!tcs->initialized)
        return;
    
    if (tcs->masterfd >= 0)
    {
        close(tcs->masterfd);
        tcs->masterfd = -1;
    }

    for (i = 0; i < TAG_COUNT; i++)
    {
        if (tcs->idxfd[i] >= 0)
        {
            close(tcs->idxfd[i]);
            tcs->idxfd[i] = -1;
        }
    }
    
    tcs->ramsearch = false;
    tcs->valid = false;
    tcs->initialized = 0;
    if (write_lock > 0)
        write_lock--;
}

#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
{
    return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
}

static long get_tag_numeric(const struct index_entry *entry, int tag)
{
    return check_virtual_tags(tag, entry);
}

static char* get_tag_string(const struct index_entry *entry, int tag)
{
    char* s = get_tag(entry, tag)->tag_data;
    return strcmp(s, UNTAGGED) ? s : NULL;
}

bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
{
    struct index_entry *entry;
    int idx_id;
    
    if (!tc_stat.ready)
        return false;
    
    /* Find the corresponding entry in tagcache. */
    idx_id = find_entry_ram(filename, NULL);
    if (idx_id < 0 || !tc_stat.ramcache)
        return false;
    
    entry = &hdr->indices[idx_id];
    
    id3->title        = get_tag_string(entry, tag_title);
    id3->artist       = get_tag_string(entry, tag_artist);
    id3->album        = get_tag_string(entry, tag_album);
    id3->genre_string = get_tag_string(entry, tag_genre);
    id3->composer     = get_tag_string(entry, tag_composer);
    id3->comment      = get_tag_string(entry, tag_comment);
    id3->albumartist  = get_tag_string(entry, tag_albumartist);
    id3->grouping     = get_tag_string(entry, tag_grouping);

    id3->playcount  = get_tag_numeric(entry, tag_playcount);
    id3->rating     = get_tag_numeric(entry, tag_rating);
    id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
    id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
    id3->year       = get_tag_numeric(entry, tag_year);

    id3->discnum = get_tag_numeric(entry, tag_discnumber);
    id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
    id3->bitrate = get_tag_numeric(entry, tag_bitrate);
    if (id3->bitrate == 0)
        id3->bitrate = 1;

    return true;
}
#endif

static inline void write_item(const char *item)
{
    int len = strlen(item) + 1;

    data_size += len;
    write(cachefd, item, len);
}

static int check_if_empty(char **tag)
{
    int length;

    if (*tag == NULL || **tag == '\0')
    {
        *tag = UNTAGGED;
        return sizeof(UNTAGGED); /* Tag length */
    }

    length = strlen(*tag);
    if (length > TAG_MAXLEN)
    {
        logf("over length tag: %s", *tag);
        length = TAG_MAXLEN;
        (*tag)[length] = '\0';
    }

    return length + 1;
}

#define ADD_TAG(entry,tag,data) \
    /* Adding tag */ \
    entry.tag_offset[tag] = offset; \
    entry.tag_length[tag] = check_if_empty(data); \
    offset += entry.tag_length[tag]
    
static void add_tagcache(char *path, unsigned long mtime
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
                         ,const struct dirent *dc
#endif
                         )
{
    struct mp3entry id3;
    struct temp_file_entry entry;
    bool ret;
    int fd;
    int idx_id = -1;
    char tracknumfix[3];
    int offset = 0;
    int path_length = strlen(path);
    bool has_albumartist;
    bool has_grouping;

    if (cachefd < 0)
        return ;

    /* Check for overlength file path. */
    if (path_length > TAG_MAXLEN)
    {
        /* Path can't be shortened. */
        logf("Too long path: %s", path);
        return ;
    }
    
    /* Check if the file is supported. */
    if (probe_file_format(path) == AFMT_UNKNOWN)
        return ;
    
    /* Check if the file is already cached. */
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
    if (tc_stat.ramcache && is_dircache_intact())
    {
        idx_id = find_entry_ram(path, dc);
    }
    else
#endif
    {
        if (filenametag_fd >= 0)
        {
            idx_id = find_entry_disk(path);
        }
    }
    
    /* Check if file has been modified. */
    if (idx_id >= 0)
    {
        struct index_entry idx;
        
        if (!get_index(-1, idx_id, &idx, true))
        {
            logf("failed to retrieve index entry");
            return ;
        }
        
        if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
        {
            /* No changes to file. */
            return ;
        }
        
        /* Metadata might have been changed. Delete the entry. */
        logf("Re-adding: %s", path);
        if (!delete_entry(idx_id))
        {
            logf("delete_entry failed: %d", idx_id);
            return ;
        }
    }
    
    fd = open(path, O_RDONLY);
    if (fd < 0)
    {
        logf("open fail: %s", path);
        return ;
    }

    memset(&id3, 0, sizeof(struct mp3entry));
    memset(&entry, 0, sizeof(struct temp_file_entry));
    memset(&tracknumfix, 0, sizeof(tracknumfix));
    ret = get_metadata(&id3, fd, path);
    close(fd);

    if (!ret)
        return ;

    logf("-> %s", path);
    
    /* Generate track number if missing. */
    if (id3.tracknum <= 0)
    {
        const char *p = strrchr(path, '.');
        
        if (p == NULL)
            p = &path[strlen(path)-1];
        
        while (*p != '/')
        {
            if (isdigit(*p) && isdigit(*(p-1)))
            {
                tracknumfix[1] = *p--;
                tracknumfix[0] = *p;
                break;
            }
            p--;
        }
        
        if (tracknumfix[0] != '\0')
        {
            id3.tracknum = atoi(tracknumfix);
            /* Set a flag to indicate track number has been generated. */
            entry.flag |= FLAG_TRKNUMGEN;
        }
        else
        {
            /* Unable to generate track number. */
            id3.tracknum = -1;
        }
    }
    
    /* Numeric tags */
    entry.tag_offset[tag_year] = id3.year;
    entry.tag_offset[tag_discnumber] = id3.discnum;
    entry.tag_offset[tag_tracknumber] = id3.tracknum;
    entry.tag_offset[tag_length] = id3.length;
    entry.tag_offset[tag_bitrate] = id3.bitrate;
    entry.tag_offset[tag_mtime] = mtime;
    
    /* String tags. */
    has_albumartist = id3.albumartist != NULL
        && strlen(id3.albumartist) > 0;
    has_grouping = id3.grouping != NULL
        && strlen(id3.grouping) > 0;

    ADD_TAG(entry, tag_filename, &path);
    ADD_TAG(entry, tag_title, &id3.title);
    ADD_TAG(entry, tag_artist, &id3.artist);
    ADD_TAG(entry, tag_album, &id3.album);
    ADD_TAG(entry, tag_genre, &id3.genre_string);
    ADD_TAG(entry, tag_composer, &id3.composer);
    ADD_TAG(entry, tag_comment, &id3.comment);
    if (has_albumartist)
    {
        ADD_TAG(entry, tag_albumartist, &id3.albumartist);
    }
    else
    {
        ADD_TAG(entry, tag_albumartist, &id3.artist);
    }
    if (has_grouping)
    {
        ADD_TAG(entry, tag_grouping, &id3.grouping);
    }
    else
    {
        ADD_TAG(entry, tag_grouping, &id3.title);
    }
    entry.data_length = offset;
    
    /* Write the header */
    write(cachefd, &entry, sizeof(struct temp_file_entry));
    
    /* And tags also... Correct order is critical */
    write_item(path);
    write_item(id3.title);
    write_item(id3.artist);
    write_item(id3.album);
    write_item(id3.genre_string);
    write_item(id3.composer);
    write_item(id3.comment);
    if (has_albumartist)
    {
        write_item(id3.albumartist);
    }
    else
    {
        write_item(id3.artist);
    }
    if (has_grouping)
    {
        write_item(id3.grouping);
    }
    else
    {
        write_item(id3.title);
    }
    total_entry_count++;    
}

static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
{
    struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
    int len = strlen(str)+1;
    int i;
    unsigned crc32;
    unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
    char buf[TAG_MAXLEN+32];
    
    for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
        buf[i] = tolower(str[i]);
    buf[i] = '\0';
    
    crc32 = crc_32(buf, i, 0xffffffff);
    
    if (unique)
    {
        /* Check if the crc does not exist -> entry does not exist for sure. */
        for (i = 0; i < tempbufidx; i++)
        {
            if (crcbuf[-i] != crc32)
                continue;
            
            if (!strcasecmp(str, index[i].str))
            {
                if (id < 0 || id >= lookup_buffer_depth)
                {
                    logf("lookup buf overf.: %d", id);
                    return false;
                }
                
                lookup[id] = &index[i];
                return true;
            }
        }
    }
    
    /* Insert to CRC buffer. */
    crcbuf[-tempbufidx] = crc32;
    tempbuf_left -= 4;
    
    /* Insert it to the buffer. */
    tempbuf_left -= len;
    if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
        return false;
    
    if (id >= lookup_buffer_depth)
    {
        logf("lookup buf overf. #2: %d", id);
        return false;
    }

    if (id >= 0)
    {
        lookup[id] = &index[tempbufidx];
        index[tempbufidx].idlist.id = id;
    }
    else
        index[tempbufidx].idlist.id = -1;
    
    index[tempbufidx].idlist.next = NULL;
    index[tempbufidx].idx_id = idx_id;
    index[tempbufidx].seek = -1;
    index[tempbufidx].str = &tempbuf[tempbuf_pos];
    memcpy(index[tempbufidx].str, str, len);
    tempbuf_pos += len;
    tempbufidx++;
    
    return true;
}

static int compare(const void *p1, const void *p2)
{
    do_timed_yield();

    struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
    struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
    
    if (strcmp(e1->str, UNTAGGED) == 0)
    {
        if (strcmp(e2->str, UNTAGGED) == 0)
            return 0;
        return -1;
    }
    else if (strcmp(e2->str, UNTAGGED) == 0)
        return 1;
    
    return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
}

static int tempbuf_sort(int fd)
{
    struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
    struct tagfile_entry fe;
    int i;
    int length;
    
    /* Generate reverse lookup entries. */
    for (i = 0; i < lookup_buffer_depth; i++)
    {
        struct tempbuf_id_list *idlist;
        
        if (!lookup[i])
            continue;
        
        if (lookup[i]->idlist.id == i)
            continue;
        
        idlist = &lookup[i]->idlist;
        while (idlist->next != NULL)
            idlist = idlist->next;
        
        tempbuf_left -= sizeof(struct tempbuf_id_list);
        if (tempbuf_left - 4 < 0)
            return -1;
        
        idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
        if (tempbuf_pos & 0x03)
        {
            tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
            tempbuf_left -= 3;
            idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
        }
        tempbuf_pos += sizeof(struct tempbuf_id_list);
        
        idlist = idlist->next;
        idlist->id = i;
        idlist->next = NULL;

        do_timed_yield();
    }
    
    qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx