Merge tag 'gpio-v3.13-3' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw...

[linux-2.6.git] / drivers / md / bcache / btree.h

#ifndef _BCACHE_BTREE_H
#define _BCACHE_BTREE_H

/*
 * THE BTREE:
 *
 * At a high level, bcache's btree is relatively standard b+ tree. All keys and
 * pointers are in the leaves; interior nodes only have pointers to the child
 * nodes.
 *
 * In the interior nodes, a struct bkey always points to a child btree node, and
 * the key is the highest key in the child node - except that the highest key in
 * an interior node is always MAX_KEY. The size field refers to the size on disk
 * of the child node - this would allow us to have variable sized btree nodes
 * (handy for keeping the depth of the btree 1 by expanding just the root).
 *
 * Btree nodes are themselves log structured, but this is hidden fairly
 * thoroughly. Btree nodes on disk will in practice have extents that overlap
 * (because they were written at different times), but in memory we never have
 * overlapping extents - when we read in a btree node from disk, the first thing
 * we do is resort all the sets of keys with a mergesort, and in the same pass
 * we check for overlapping extents and adjust them appropriately.
 *
 * struct btree_op is a central interface to the btree code. It's used for
 * specifying read vs. write locking, and the embedded closure is used for
 * waiting on IO or reserve memory.
 *
 * BTREE CACHE:
 *
 * Btree nodes are cached in memory; traversing the btree might require reading
 * in btree nodes which is handled mostly transparently.
 *
 * bch_btree_node_get() looks up a btree node in the cache and reads it in from
 * disk if necessary. This function is almost never called directly though - the
 * btree() macro is used to get a btree node, call some function on it, and
 * unlock the node after the function returns.
 *
 * The root is special cased - it's taken out of the cache's lru (thus pinning
 * it in memory), so we can find the root of the btree by just dereferencing a
 * pointer instead of looking it up in the cache. This makes locking a bit
 * tricky, since the root pointer is protected by the lock in the btree node it
 * points to - the btree_root() macro handles this.
 *
 * In various places we must be able to allocate memory for multiple btree nodes
 * in order to make forward progress. To do this we use the btree cache itself
 * as a reserve; if __get_free_pages() fails, we'll find a node in the btree
 * cache we can reuse. We can't allow more than one thread to be doing this at a
 * time, so there's a lock, implemented by a pointer to the btree_op closure -
 * this allows the btree_root() macro to implicitly release this lock.
 *
 * BTREE IO:
 *
 * Btree nodes never have to be explicitly read in; bch_btree_node_get() handles
 * this.
 *
 * For writing, we have two btree_write structs embeddded in struct btree - one
 * write in flight, and one being set up, and we toggle between them.
 *
 * Writing is done with a single function -  bch_btree_write() really serves two
 * different purposes and should be broken up into two different functions. When
 * passing now = false, it merely indicates that the node is now dirty - calling
 * it ensures that the dirty keys will be written at some point in the future.
 *
 * When passing now = true, bch_btree_write() causes a write to happen
 * "immediately" (if there was already a write in flight, it'll cause the write
 * to happen as soon as the previous write completes). It returns immediately
 * though - but it takes a refcount on the closure in struct btree_op you passed
 * to it, so a closure_sync() later can be used to wait for the write to
 * complete.
 *
 * This is handy because btree_split() and garbage collection can issue writes
 * in parallel, reducing the amount of time they have to hold write locks.
 *
 * LOCKING:
 *
 * When traversing the btree, we may need write locks starting at some level -
 * inserting a key into the btree will typically only require a write lock on
 * the leaf node.
 *
 * This is specified with the lock field in struct btree_op; lock = 0 means we
 * take write locks at level <= 0, i.e. only leaf nodes. bch_btree_node_get()
 * checks this field and returns the node with the appropriate lock held.
 *
 * If, after traversing the btree, the insertion code discovers it has to split
 * then it must restart from the root and take new locks - to do this it changes
 * the lock field and returns -EINTR, which causes the btree_root() macro to
 * loop.
 *
 * Handling cache misses require a different mechanism for upgrading to a write
 * lock. We do cache lookups with only a read lock held, but if we get a cache
 * miss and we wish to insert this data into the cache, we have to insert a
 * placeholder key to detect races - otherwise, we could race with a write and
 * overwrite the data that was just written to the cache with stale data from
 * the backing device.
 *
 * For this we use a sequence number that write locks and unlocks increment - to
 * insert the check key it unlocks the btree node and then takes a write lock,
 * and fails if the sequence number doesn't match.
 */

#include "bset.h"
#include "debug.h"

struct btree_write {
        atomic_t                *journal;

        /* If btree_split() frees a btree node, it writes a new pointer to that
         * btree node indicating it was freed; it takes a refcount on
         * c->prio_blocked because we can't write the gens until the new
         * pointer is on disk. This allows btree_write_endio() to release the
         * refcount that btree_split() took.
         */
        int                     prio_blocked;
};

struct btree {
        /* Hottest entries first */
        struct hlist_node       hash;

        /* Key/pointer for this btree node */
        BKEY_PADDED(key);

        /* Single bit - set when accessed, cleared by shrinker */
        unsigned long           accessed;
        unsigned long           seq;
        struct rw_semaphore     lock;
        struct cache_set        *c;
        struct btree            *parent;

        unsigned long           flags;
        uint16_t                written;        /* would be nice to kill */
        uint8_t                 level;
        uint8_t                 nsets;
        uint8_t                 page_order;

        /*
         * Set of sorted keys - the real btree node - plus a binary search tree
         *
         * sets[0] is special; set[0]->tree, set[0]->prev and set[0]->data point
         * to the memory we have allocated for this btree node. Additionally,
         * set[0]->data points to the entire btree node as it exists on disk.
         */
        struct bset_tree        sets[MAX_BSETS];

        /* For outstanding btree writes, used as a lock - protects write_idx */
        struct closure_with_waitlist    io;

        struct list_head        list;
        struct delayed_work     work;

        struct btree_write      writes[2];
        struct bio              *bio;
};

#define BTREE_FLAG(flag)                                                \
static inline bool btree_node_ ## flag(struct btree *b)                 \
{       return test_bit(BTREE_NODE_ ## flag, &b->flags); }              \
                                                                        \
static inline void set_btree_node_ ## flag(struct btree *b)             \
{       set_bit(BTREE_NODE_ ## flag, &b->flags); }                      \

enum btree_flags {
        BTREE_NODE_io_error,
        BTREE_NODE_dirty,
        BTREE_NODE_write_idx,
};

BTREE_FLAG(io_error);
BTREE_FLAG(dirty);
BTREE_FLAG(write_idx);

static inline struct btree_write *btree_current_write(struct btree *b)
{
        return b->writes + btree_node_write_idx(b);
}

static inline struct btree_write *btree_prev_write(struct btree *b)
{
        return b->writes + (btree_node_write_idx(b) ^ 1);
}

static inline unsigned bset_offset(struct btree *b, struct bset *i)
{
        return (((size_t) i) - ((size_t) b->sets->data)) >> 9;
}

static inline struct bset *write_block(struct btree *b)
{
        return ((void *) b->sets[0].data) + b->written * block_bytes(b->c);
}

static inline bool bset_written(struct btree *b, struct bset_tree *t)
{
        return t->data < write_block(b);
}

static inline bool bkey_written(struct btree *b, struct bkey *k)
{
        return k < write_block(b)->start;
}

static inline void set_gc_sectors(struct cache_set *c)
{
        atomic_set(&c->sectors_to_gc, c->sb.bucket_size * c->nbuckets / 16);
}

static inline struct bkey *bch_btree_iter_init(struct btree *b,
                                               struct btree_iter *iter,
                                               struct bkey *search)
{
        return __bch_btree_iter_init(b, iter, search, b->sets);
}

static inline bool bch_ptr_invalid(struct btree *b, const struct bkey *k)
{
        if (b->level)
                return bch_btree_ptr_invalid(b->c, k);
        else
                return bch_extent_ptr_invalid(b->c, k);
}

void bkey_put(struct cache_set *c, struct bkey *k);

/* Looping macros */

#define for_each_cached_btree(b, c, iter)                               \
        for (iter = 0;                                                  \
             iter < ARRAY_SIZE((c)->bucket_hash);                       \
             iter++)                                                    \
                hlist_for_each_entry_rcu((b), (c)->bucket_hash + iter, hash)

#define for_each_key_filter(b, k, iter, filter)                         \
        for (bch_btree_iter_init((b), (iter), NULL);                    \
             ((k) = bch_btree_iter_next_filter((iter), b, filter));)

#define for_each_key(b, k, iter)                                        \
        for (bch_btree_iter_init((b), (iter), NULL);                    \
             ((k) = bch_btree_iter_next(iter));)

/* Recursing down the btree */

struct btree_op {
        /* Btree level at which we start taking write locks */
        short                   lock;

        unsigned                insert_collision:1;
};

static inline void bch_btree_op_init(struct btree_op *op, int write_lock_level)
{
        memset(op, 0, sizeof(struct btree_op));
        op->lock = write_lock_level;
}

static inline void rw_lock(bool w, struct btree *b, int level)
{
        w ? down_write_nested(&b->lock, level + 1)
          : down_read_nested(&b->lock, level + 1);
        if (w)
                b->seq++;
}

static inline void rw_unlock(bool w, struct btree *b)
{
        if (w)
                b->seq++;
        (w ? up_write : up_read)(&b->lock);
}

void bch_btree_node_read(struct btree *);
void bch_btree_node_write(struct btree *, struct closure *);

void bch_btree_set_root(struct btree *);
struct btree *bch_btree_node_alloc(struct cache_set *, int, bool);
struct btree *bch_btree_node_get(struct cache_set *, struct bkey *, int, bool);

int bch_btree_insert_check_key(struct btree *, struct btree_op *,
                               struct bkey *);
int bch_btree_insert(struct cache_set *, struct keylist *,
                     atomic_t *, struct bkey *);

int bch_gc_thread_start(struct cache_set *);
size_t bch_btree_gc_finish(struct cache_set *);
void bch_moving_gc(struct cache_set *);
int bch_btree_check(struct cache_set *);
uint8_t __bch_btree_mark_key(struct cache_set *, int, struct bkey *);

static inline void wake_up_gc(struct cache_set *c)
{
        if (c->gc_thread)
                wake_up_process(c->gc_thread);
}

#define MAP_DONE        0
#define MAP_CONTINUE    1

#define MAP_ALL_NODES   0
#define MAP_LEAF_NODES  1

#define MAP_END_KEY     1

typedef int (btree_map_nodes_fn)(struct btree_op *, struct btree *);
int __bch_btree_map_nodes(struct btree_op *, struct cache_set *,
                          struct bkey *, btree_map_nodes_fn *, int);

static inline int bch_btree_map_nodes(struct btree_op *op, struct cache_set *c,
                                      struct bkey *from, btree_map_nodes_fn *fn)
{
        return __bch_btree_map_nodes(op, c, from, fn, MAP_ALL_NODES);
}

static inline int bch_btree_map_leaf_nodes(struct btree_op *op,
                                           struct cache_set *c,
                                           struct bkey *from,
                                           btree_map_nodes_fn *fn)
{
        return __bch_btree_map_nodes(op, c, from, fn, MAP_LEAF_NODES);
}

typedef int (btree_map_keys_fn)(struct btree_op *, struct btree *,
                                struct bkey *);
int bch_btree_map_keys(struct btree_op *, struct cache_set *,
                       struct bkey *, btree_map_keys_fn *, int);

typedef bool (keybuf_pred_fn)(struct keybuf *, struct bkey *);

void bch_keybuf_init(struct keybuf *);
void bch_refill_keybuf(struct cache_set *, struct keybuf *,
                       struct bkey *, keybuf_pred_fn *);
bool bch_keybuf_check_overlapping(struct keybuf *, struct bkey *,
                                  struct bkey *);
void bch_keybuf_del(struct keybuf *, struct keybuf_key *);
struct keybuf_key *bch_keybuf_next(struct keybuf *);
struct keybuf_key *bch_keybuf_next_rescan(struct cache_set *, struct keybuf *,
                                          struct bkey *, keybuf_pred_fn *);

#endif