sys/vfs/hammer/hammer_inode.c

   1 /*
   2  * Copyright (c) 2007-2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_inode.c,v 1.86 2008/06/27 20:56:59 dillon Exp $
  35  */
  36
  37 #include "hammer.h"
  38 #include <vm/vm_extern.h>
  39 #include <sys/buf.h>
  40 #include <sys/buf2.h>
  41
  42 static int      hammer_unload_inode(struct hammer_inode *ip);
  43 static void     hammer_flush_inode_core(hammer_inode_t ip, int flags);
  44 static int      hammer_setup_child_callback(hammer_record_t rec, void *data);
  45 static int      hammer_setup_parent_inodes(hammer_inode_t ip);
  46 static int      hammer_setup_parent_inodes_helper(hammer_record_t record);
  47 static void     hammer_inode_wakereclaims(hammer_inode_t ip);
  48
  49 #ifdef DEBUG_TRUNCATE
  50 extern struct hammer_inode *HammerTruncIp;
  51 #endif
  52
  53 /*
  54  * Red-Black tree support for inode structures.
  55  *
  56  * Insertions
  57  */
  58 int
  59 hammer_ino_rb_compare(hammer_inode_t ip1, hammer_inode_t ip2)
  60 {
  61         if (ip1->obj_localization < ip2->obj_localization)
  62                 return(-1);
  63         if (ip1->obj_localization > ip2->obj_localization)
  64                 return(1);
  65         if (ip1->obj_id < ip2->obj_id)
  66                 return(-1);
  67         if (ip1->obj_id > ip2->obj_id)
  68                 return(1);
  69         if (ip1->obj_asof < ip2->obj_asof)
  70                 return(-1);
  71         if (ip1->obj_asof > ip2->obj_asof)
  72                 return(1);
  73         return(0);
  74 }
  75
  76 /*
  77  * LOOKUP_INFO
  78  */
  79 static int
  80 hammer_inode_info_cmp(hammer_inode_info_t info, hammer_inode_t ip)
  81 {
  82         if (info->obj_localization < ip->obj_localization)
  83                 return(-1);
  84         if (info->obj_localization > ip->obj_localization)
  85                 return(1);
  86         if (info->obj_id < ip->obj_id)
  87                 return(-1);
  88         if (info->obj_id > ip->obj_id)
  89                 return(1);
  90         if (info->obj_asof < ip->obj_asof)
  91                 return(-1);
  92         if (info->obj_asof > ip->obj_asof)
  93                 return(1);
  94         return(0);
  95 }
  96
  97 /*
  98  * Used by hammer_scan_inode_snapshots() to locate all of an object's
  99  * snapshots.  Note that the asof field is not tested, which we can get
 100  * away with because it is the lowest-priority field.
 101  */
 102 static int
 103 hammer_inode_info_cmp_all_history(hammer_inode_t ip, void *data)
 104 {
 105         hammer_inode_info_t info = data;
 106
 107         if (ip->obj_localization > info->obj_localization)
 108                 return(1);
 109         if (ip->obj_localization < info->obj_localization)
 110                 return(-1);
 111         if (ip->obj_id > info->obj_id)
 112                 return(1);
 113         if (ip->obj_id < info->obj_id)
 114                 return(-1);
 115         return(0);
 116 }
 117
 118 RB_GENERATE(hammer_ino_rb_tree, hammer_inode, rb_node, hammer_ino_rb_compare);
 119 RB_GENERATE_XLOOKUP(hammer_ino_rb_tree, INFO, hammer_inode, rb_node,
 120                 hammer_inode_info_cmp, hammer_inode_info_t);
 121
 122 /*
 123  * The kernel is not actively referencing this vnode but is still holding
 124  * it cached.
 125  *
 126  * This is called from the frontend.
 127  */
 128 int
 129 hammer_vop_inactive(struct vop_inactive_args *ap)
 130 {
 131         struct hammer_inode *ip = VTOI(ap->a_vp);
 132
 133         /*
 134          * Degenerate case
 135          */
 136         if (ip == NULL) {
 137                 vrecycle(ap->a_vp);
 138                 return(0);
 139         }
 140
 141         /*
 142          * If the inode no longer has visibility in the filesystem try to
 143          * recycle it immediately, even if the inode is dirty.  Recycling
 144          * it quickly allows the system to reclaim buffer cache and VM
 145          * resources which can matter a lot in a heavily loaded system.
 146          *
 147          * This can deadlock in vfsync() if we aren't careful.
 148          *
 149          * Do not queue the inode to the flusher if we still have visibility,
 150          * otherwise namespace calls such as chmod will unnecessarily generate
 151          * multiple inode updates.
 152          */
 153         hammer_inode_unloadable_check(ip, 0);
 154         if (ip->ino_data.nlinks == 0) {
 155                 if (ip->flags & HAMMER_INODE_MODMASK)
 156                         hammer_flush_inode(ip, 0);
 157                 vrecycle(ap->a_vp);
 158         }
 159         return(0);
 160 }
 161
 162 /*
 163  * Release the vnode association.  This is typically (but not always)
 164  * the last reference on the inode.
 165  *
 166  * Once the association is lost we are on our own with regards to
 167  * flushing the inode.
 168  */
 169 int
 170 hammer_vop_reclaim(struct vop_reclaim_args *ap)
 171 {
 172         struct hammer_inode *ip;
 173         hammer_mount_t hmp;
 174         struct vnode *vp;
 175
 176         vp = ap->a_vp;
 177
 178         if ((ip = vp->v_data) != NULL) {
 179                 hmp = ip->hmp;
 180                 vp->v_data = NULL;
 181                 ip->vp = NULL;
 182
 183                 if ((ip->flags & HAMMER_INODE_RECLAIM) == 0) {
 184                         ++hammer_count_reclaiming;
 185                         ++hmp->inode_reclaims;
 186                         ip->flags |= HAMMER_INODE_RECLAIM;
 187                         if (hmp->inode_reclaims > HAMMER_RECLAIM_FLUSH &&
 188                             (hmp->inode_reclaims & 255) == 0) {
 189                                 hammer_flusher_async(hmp);
 190                         }
 191                 }
 192                 hammer_rel_inode(ip, 1);
 193         }
 194         return(0);
 195 }
 196
 197 /*
 198  * Return a locked vnode for the specified inode.  The inode must be
 199  * referenced but NOT LOCKED on entry and will remain referenced on
 200  * return.
 201  *
 202  * Called from the frontend.
 203  */
 204 int
 205 hammer_get_vnode(struct hammer_inode *ip, struct vnode **vpp)
 206 {
 207         hammer_mount_t hmp;
 208         struct vnode *vp;
 209         int error = 0;
 210
 211         hmp = ip->hmp;
 212
 213         for (;;) {
 214                 if ((vp = ip->vp) == NULL) {
 215                         error = getnewvnode(VT_HAMMER, hmp->mp, vpp, 0, 0);
 216                         if (error)
 217                                 break;
 218                         hammer_lock_ex(&ip->lock);
 219                         if (ip->vp != NULL) {
 220                                 hammer_unlock(&ip->lock);
 221                                 vp->v_type = VBAD;
 222                                 vx_put(vp);
 223                                 continue;
 224                         }
 225                         hammer_ref(&ip->lock);
 226                         vp = *vpp;
 227                         ip->vp = vp;
 228                         vp->v_type =
 229                                 hammer_get_vnode_type(ip->ino_data.obj_type);
 230
 231                         hammer_inode_wakereclaims(ip);
 232
 233                         switch(ip->ino_data.obj_type) {
 234                         case HAMMER_OBJTYPE_CDEV:
 235                         case HAMMER_OBJTYPE_BDEV:
 236                                 vp->v_ops = &hmp->mp->mnt_vn_spec_ops;
 237                                 addaliasu(vp, ip->ino_data.rmajor,
 238                                           ip->ino_data.rminor);
 239                                 break;
 240                         case HAMMER_OBJTYPE_FIFO:
 241                                 vp->v_ops = &hmp->mp->mnt_vn_fifo_ops;
 242                                 break;
 243                         default:
 244                                 break;
 245                         }
 246
 247                         /*
 248                          * Only mark as the root vnode if the ip is not
 249                          * historical, otherwise the VFS cache will get
 250                          * confused.  The other half of the special handling
 251                          * is in hammer_vop_nlookupdotdot().
 252                          *
 253                          * Pseudo-filesystem roots also do not count.
 254                          */
 255                         if (ip->obj_id == HAMMER_OBJID_ROOT &&
 256                             ip->obj_asof == hmp->asof &&
 257                             ip->obj_localization == 0) {
 258                                 vp->v_flag |= VROOT;
 259                         }
 260
 261                         vp->v_data = (void *)ip;
 262                         /* vnode locked by getnewvnode() */
 263                         /* make related vnode dirty if inode dirty? */
 264                         hammer_unlock(&ip->lock);
 265                         if (vp->v_type == VREG)
 266                                 vinitvmio(vp, ip->ino_data.size);
 267                         break;
 268                 }
 269
 270                 /*
 271                  * loop if the vget fails (aka races), or if the vp
 272                  * no longer matches ip->vp.
 273                  */
 274                 if (vget(vp, LK_EXCLUSIVE) == 0) {
 275                         if (vp == ip->vp)
 276                                 break;
 277                         vput(vp);
 278                 }
 279         }
 280         *vpp = vp;
 281         return(error);
 282 }
 283
 284 /*
 285  * Locate all copies of the inode for obj_id compatible with the specified
 286  * asof, reference, and issue the related call-back.  This routine is used
 287  * for direct-io invalidation and does not create any new inodes.
 288  */
 289 void
 290 hammer_scan_inode_snapshots(hammer_mount_t hmp, hammer_inode_info_t iinfo,
 291                             int (*callback)(hammer_inode_t ip, void *data),
 292                             void *data)
 293 {
 294         hammer_ino_rb_tree_RB_SCAN(&hmp->rb_inos_root,
 295                                    hammer_inode_info_cmp_all_history,
 296                                    callback, iinfo);
 297 }
 298
 299 /*
 300  * Acquire a HAMMER inode.  The returned inode is not locked.  These functions
 301  * do not attach or detach the related vnode (use hammer_get_vnode() for
 302  * that).
 303  *
 304  * The flags argument is only applied for newly created inodes, and only
 305  * certain flags are inherited.
 306  *
 307  * Called from the frontend.
 308  */
 309 struct hammer_inode *
 310 hammer_get_inode(hammer_transaction_t trans, hammer_inode_t dip,
 311                  u_int64_t obj_id, hammer_tid_t asof, u_int32_t localization,
 312                  int flags, int *errorp)
 313 {
 314         hammer_mount_t hmp = trans->hmp;
 315         struct hammer_inode_info iinfo;
 316         struct hammer_cursor cursor;
 317         struct hammer_inode *ip;
 318
 319         /*
 320          * Determine if we already have an inode cached.  If we do then
 321          * we are golden.
 322          */
 323         iinfo.obj_id = obj_id;
 324         iinfo.obj_asof = asof;
 325         iinfo.obj_localization = localization;
 326 loop:
 327         ip = hammer_ino_rb_tree_RB_LOOKUP_INFO(&hmp->rb_inos_root, &iinfo);
 328         if (ip) {
 329                 hammer_ref(&ip->lock);
 330                 *errorp = 0;
 331                 return(ip);
 332         }
 333
 334         /*
 335          * Allocate a new inode structure and deal with races later.
 336          */
 337         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 338         ++hammer_count_inodes;
 339         ++hmp->count_inodes;
 340         ip->obj_id = obj_id;
 341         ip->obj_asof = iinfo.obj_asof;
 342         ip->obj_localization = localization;
 343         ip->hmp = hmp;
 344         ip->flags = flags & HAMMER_INODE_RO;
 345         ip->cache[0].ip = ip;
 346         ip->cache[1].ip = ip;
 347         if (hmp->ronly || (hmp->hflags & HMNT_SLAVE))
 348                 ip->flags |= HAMMER_INODE_RO;
 349         ip->sync_trunc_off = ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 350         RB_INIT(&ip->rec_tree);
 351         TAILQ_INIT(&ip->target_list);
 352
 353         /*
 354          * Locate the on-disk inode.
 355          */
 356 retry:
 357         hammer_init_cursor(trans, &cursor, (dip ? &dip->cache[0] : NULL), NULL);
 358         cursor.key_beg.localization = localization + HAMMER_LOCALIZE_INODE;
 359         cursor.key_beg.obj_id = ip->obj_id;
 360         cursor.key_beg.key = 0;
 361         cursor.key_beg.create_tid = 0;
 362         cursor.key_beg.delete_tid = 0;
 363         cursor.key_beg.rec_type = HAMMER_RECTYPE_INODE;
 364         cursor.key_beg.obj_type = 0;
 365         cursor.asof = iinfo.obj_asof;
 366         cursor.flags = HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_GET_DATA |
 367                        HAMMER_CURSOR_ASOF;
 368
 369         *errorp = hammer_btree_lookup(&cursor);
 370         if (*errorp == EDEADLK) {
 371                 hammer_done_cursor(&cursor);
 372                 goto retry;
 373         }
 374
 375         /*
 376          * On success the B-Tree lookup will hold the appropriate
 377          * buffer cache buffers and provide a pointer to the requested
 378          * information.  Copy the information to the in-memory inode
 379          * and cache the B-Tree node to improve future operations.
 380          */
 381         if (*errorp == 0) {
 382                 ip->ino_leaf = cursor.node->ondisk->elms[cursor.index].leaf;
 383                 ip->ino_data = cursor.data->inode;
 384
 385                 /*
 386                  * cache[0] tries to cache the location of the object inode.
 387                  * The assumption is that it is near the directory inode.
 388                  *
 389                  * cache[1] tries to cache the location of the object data.
 390                  * The assumption is that it is near the directory data.
 391                  */
 392                 hammer_cache_node(&ip->cache[0], cursor.node);
 393                 if (dip && dip->cache[1].node)
 394                         hammer_cache_node(&ip->cache[1], dip->cache[1].node);
 395
 396                 /*
 397                  * The file should not contain any data past the file size
 398                  * stored in the inode.  Setting sync_trunc_off to the
 399                  * file size instead of max reduces B-Tree lookup overheads
 400                  * on append by allowing the flusher to avoid checking for
 401                  * record overwrites.
 402                  */
 403                 ip->sync_trunc_off = ip->ino_data.size;
 404         }
 405
 406         /*
 407          * The inode is placed on the red-black tree and will be synced to
 408          * the media when flushed or by the filesystem sync.  If this races
 409          * another instantiation/lookup the insertion will fail.
 410          */
 411         if (*errorp == 0) {
 412                 hammer_ref(&ip->lock);
 413                 if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 414                         hammer_uncache_node(&ip->cache[0]);
 415                         hammer_uncache_node(&ip->cache[1]);
 416                         KKASSERT(ip->lock.refs == 1);
 417                         --hammer_count_inodes;
 418                         --hmp->count_inodes;
 419                         kfree(ip, M_HAMMER);
 420                         hammer_done_cursor(&cursor);
 421                         goto loop;
 422                 }
 423                 ip->flags |= HAMMER_INODE_ONDISK;
 424         } else {
 425                 if (ip->flags & HAMMER_INODE_RSV_INODES) {
 426                         ip->flags &= ~HAMMER_INODE_RSV_INODES; /* sanity */
 427                         --hmp->rsv_inodes;
 428                 }
 429                 hmp->rsv_databufs -= ip->rsv_databufs;
 430                 ip->rsv_databufs = 0;                          /* sanity */
 431
 432                 --hammer_count_inodes;
 433                 --hmp->count_inodes;
 434                 kfree(ip, M_HAMMER);
 435                 ip = NULL;
 436         }
 437         hammer_done_cursor(&cursor);
 438         return (ip);
 439 }
 440
 441 /*
 442  * Create a new filesystem object, returning the inode in *ipp.  The
 443  * returned inode will be referenced.
 444  *
 445  * The inode is created in-memory.
 446  */
 447 int
 448 hammer_create_inode(hammer_transaction_t trans, struct vattr *vap,
 449                     struct ucred *cred, hammer_inode_t dip,
 450                     int pseudofs, struct hammer_inode **ipp)
 451 {
 452         hammer_mount_t hmp;
 453         hammer_inode_t ip;
 454         uid_t xuid;
 455         u_int32_t localization;
 456         int error;
 457
 458         hmp = trans->hmp;
 459
 460         /*
 461          * Assign the localization domain.  If if dip is NULL we are creating
 462          * a pseudo-fs and must locate an unused localization domain.
 463          */
 464         if (pseudofs) {
 465                 for (localization = HAMMER_DEF_LOCALIZATION;
 466                      localization < HAMMER_LOCALIZE_PSEUDOFS_MASK;
 467                      localization += HAMMER_LOCALIZE_PSEUDOFS_INC) {
 468                         ip = hammer_get_inode(trans, NULL, HAMMER_OBJID_ROOT,
 469                                               hmp->asof, localization,
 470                                               0, &error);
 471                         if (ip == NULL) {
 472                                 if (error != ENOENT)
 473                                         return(error);
 474                                 break;
 475                         }
 476                         if (ip)
 477                                 hammer_rel_inode(ip, 0);
 478                 }
 479         } else {
 480                 localization = dip->obj_localization;
 481         }
 482
 483         ip = kmalloc(sizeof(*ip), M_HAMMER, M_WAITOK|M_ZERO);
 484         ++hammer_count_inodes;
 485         ++hmp->count_inodes;
 486
 487         /*
 488          * Allocate a new object id.  If creating a new pseudo-fs the
 489          * obj_id is 1.
 490          */
 491         if (pseudofs)
 492                 ip->obj_id = HAMMER_OBJID_ROOT;
 493         else
 494                 ip->obj_id = hammer_alloc_objid(hmp, dip);
 495         ip->obj_localization = localization;
 496
 497         KKASSERT(ip->obj_id != 0);
 498         ip->obj_asof = hmp->asof;
 499         ip->hmp = hmp;
 500         ip->flush_state = HAMMER_FST_IDLE;
 501         ip->flags = HAMMER_INODE_DDIRTY |
 502                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME;
 503         ip->cache[0].ip = ip;
 504         ip->cache[1].ip = ip;
 505
 506         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
 507         RB_INIT(&ip->rec_tree);
 508         TAILQ_INIT(&ip->target_list);
 509
 510         ip->ino_data.atime = trans->time;
 511         ip->ino_data.mtime = trans->time;
 512         ip->ino_data.size = 0;
 513         ip->ino_data.nlinks = 0;
 514
 515         /*
 516          * A nohistory designator on the parent directory is inherited by
 517          * the child.  We will do this even for pseudo-fs creation... the
 518          * sysad can turn it off.
 519          */
 520         ip->ino_data.uflags = dip->ino_data.uflags &
 521                               (SF_NOHISTORY|UF_NOHISTORY|UF_NODUMP);
 522
 523         ip->ino_leaf.base.btype = HAMMER_BTREE_TYPE_RECORD;
 524         ip->ino_leaf.base.localization = ip->obj_localization +
 525                                          HAMMER_LOCALIZE_INODE;
 526         ip->ino_leaf.base.obj_id = ip->obj_id;
 527         ip->ino_leaf.base.key = 0;
 528         ip->ino_leaf.base.create_tid = 0;
 529         ip->ino_leaf.base.delete_tid = 0;
 530         ip->ino_leaf.base.rec_type = HAMMER_RECTYPE_INODE;
 531         ip->ino_leaf.base.obj_type = hammer_get_obj_type(vap->va_type);
 532
 533         ip->ino_data.obj_type = ip->ino_leaf.base.obj_type;
 534         ip->ino_data.version = HAMMER_INODE_DATA_VERSION;
 535         ip->ino_data.mode = vap->va_mode;
 536         ip->ino_data.ctime = trans->time;
 537
 538         /*
 539          * Setup the ".." pointer.  This only needs to be done for directories
 540          * but we do it for all objects as a recovery aid.
 541          *
 542          * The parent_obj_localization field only applies to pseudo-fs roots.
 543          */
 544         ip->ino_data.parent_obj_id = dip->ino_leaf.base.obj_id;
 545         if (ip->ino_data.obj_type == HAMMER_OBJTYPE_DIRECTORY &&
 546             ip->obj_id == HAMMER_OBJID_ROOT) {
 547                 ip->ino_data.ext.obj.parent_obj_localization =
 548                                                 dip->obj_localization;
 549         }
 550
 551         switch(ip->ino_leaf.base.obj_type) {
 552         case HAMMER_OBJTYPE_CDEV:
 553         case HAMMER_OBJTYPE_BDEV:
 554                 ip->ino_data.rmajor = vap->va_rmajor;
 555                 ip->ino_data.rminor = vap->va_rminor;
 556                 break;
 557         default:
 558                 break;
 559         }
 560
 561         /*
 562          * Calculate default uid/gid and overwrite with information from
 563          * the vap.
 564          */
 565         xuid = hammer_to_unix_xid(&dip->ino_data.uid);
 566         xuid = vop_helper_create_uid(hmp->mp, dip->ino_data.mode, xuid, cred,
 567                                      &vap->va_mode);
 568         ip->ino_data.mode = vap->va_mode;
 569
 570         if (vap->va_vaflags & VA_UID_UUID_VALID)
 571                 ip->ino_data.uid = vap->va_uid_uuid;
 572         else if (vap->va_uid != (uid_t)VNOVAL)
 573                 hammer_guid_to_uuid(&ip->ino_data.uid, vap->va_uid);
 574         else
 575                 hammer_guid_to_uuid(&ip->ino_data.uid, xuid);
 576
 577         if (vap->va_vaflags & VA_GID_UUID_VALID)
 578                 ip->ino_data.gid = vap->va_gid_uuid;
 579         else if (vap->va_gid != (gid_t)VNOVAL)
 580                 hammer_guid_to_uuid(&ip->ino_data.gid, vap->va_gid);
 581         else
 582                 ip->ino_data.gid = dip->ino_data.gid;
 583
 584         hammer_ref(&ip->lock);
 585         if (RB_INSERT(hammer_ino_rb_tree, &hmp->rb_inos_root, ip)) {
 586                 hammer_unref(&ip->lock);
 587                 panic("hammer_create_inode: duplicate obj_id %llx", ip->obj_id);
 588         }
 589         *ipp = ip;
 590         return(0);
 591 }
 592
 593 /*
 594  * Called by hammer_sync_inode().
 595  */
 596 static int
 597 hammer_update_inode(hammer_cursor_t cursor, hammer_inode_t ip)
 598 {
 599         hammer_transaction_t trans = cursor->trans;
 600         hammer_record_t record;
 601         int error;
 602         int redirty;
 603
 604 retry:
 605         error = 0;
 606
 607         /*
 608          * If the inode has a presence on-disk then locate it and mark
 609          * it deleted, setting DELONDISK.
 610          *
 611          * The record may or may not be physically deleted, depending on
 612          * the retention policy.
 613          */
 614         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) ==
 615             HAMMER_INODE_ONDISK) {
 616                 hammer_normalize_cursor(cursor);
 617                 cursor->key_beg.localization = ip->obj_localization +
 618                                                HAMMER_LOCALIZE_INODE;
 619                 cursor->key_beg.obj_id = ip->obj_id;
 620                 cursor->key_beg.key = 0;
 621                 cursor->key_beg.create_tid = 0;
 622                 cursor->key_beg.delete_tid = 0;
 623                 cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 624                 cursor->key_beg.obj_type = 0;
 625                 cursor->asof = ip->obj_asof;
 626                 cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 627                 cursor->flags |= HAMMER_CURSOR_GET_LEAF | HAMMER_CURSOR_ASOF;
 628                 cursor->flags |= HAMMER_CURSOR_BACKEND;
 629
 630                 error = hammer_btree_lookup(cursor);
 631                 if (hammer_debug_inode)
 632                         kprintf("IPDEL %p %08x %d", ip, ip->flags, error);
 633                 if (error) {
 634                         kprintf("error %d\n", error);
 635                         Debugger("hammer_update_inode");
 636                 }
 637
 638                 if (error == 0) {
 639                         error = hammer_ip_delete_record(cursor, ip, trans->tid);
 640                         if (hammer_debug_inode)
 641                                 kprintf(" error %d\n", error);
 642                         if (error && error != EDEADLK) {
 643                                 kprintf("error %d\n", error);
 644                                 Debugger("hammer_update_inode2");
 645                         }
 646                         if (error == 0) {
 647                                 ip->flags |= HAMMER_INODE_DELONDISK;
 648                         }
 649                         if (cursor->node)
 650                                 hammer_cache_node(&ip->cache[0], cursor->node);
 651                 }
 652                 if (error == EDEADLK) {
 653                         hammer_done_cursor(cursor);
 654                         error = hammer_init_cursor(trans, cursor,
 655                                                    &ip->cache[0], ip);
 656                         if (hammer_debug_inode)
 657                                 kprintf("IPDED %p %d\n", ip, error);
 658                         if (error == 0)
 659                                 goto retry;
 660                 }
 661         }
 662
 663         /*
 664          * Ok, write out the initial record or a new record (after deleting
 665          * the old one), unless the DELETED flag is set.  This routine will
 666          * clear DELONDISK if it writes out a record.
 667          *
 668          * Update our inode statistics if this is the first application of
 669          * the inode on-disk.
 670          */
 671         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED) == 0) {
 672                 /*
 673                  * Generate a record and write it to the media
 674                  */
 675                 record = hammer_alloc_mem_record(ip, 0);
 676                 record->type = HAMMER_MEM_RECORD_INODE;
 677                 record->flush_state = HAMMER_FST_FLUSH;
 678                 record->leaf = ip->sync_ino_leaf;
 679                 record->leaf.base.create_tid = trans->tid;
 680                 record->leaf.data_len = sizeof(ip->sync_ino_data);
 681                 record->leaf.create_ts = trans->time32;
 682                 record->data = (void *)&ip->sync_ino_data;
 683                 record->flags |= HAMMER_RECF_INTERLOCK_BE;
 684
 685                 /*
 686                  * If this flag is set we cannot sync the new file size
 687                  * because we haven't finished related truncations.  The
 688                  * inode will be flushed in another flush group to finish
 689                  * the job.
 690                  */
 691                 if ((ip->flags & HAMMER_INODE_WOULDBLOCK) &&
 692                     ip->sync_ino_data.size != ip->ino_data.size) {
 693                         redirty = 1;
 694                         ip->sync_ino_data.size = ip->ino_data.size;
 695                 } else {
 696                         redirty = 0;
 697                 }
 698
 699                 for (;;) {
 700                         error = hammer_ip_sync_record_cursor(cursor, record);
 701                         if (hammer_debug_inode)
 702                                 kprintf("GENREC %p rec %08x %d\n",
 703                                         ip, record->flags, error);
 704                         if (error != EDEADLK)
 705                                 break;
 706                         hammer_done_cursor(cursor);
 707                         error = hammer_init_cursor(trans, cursor,
 708                                                    &ip->cache[0], ip);
 709                         if (hammer_debug_inode)
 710                                 kprintf("GENREC reinit %d\n", error);
 711                         if (error)
 712                                 break;
 713                 }
 714                 if (error) {
 715                         kprintf("error %d\n", error);
 716                         Debugger("hammer_update_inode3");
 717                 }
 718
 719                 /*
 720                  * The record isn't managed by the inode's record tree,
 721                  * destroy it whether we succeed or fail.
 722                  */
 723                 record->flags &= ~HAMMER_RECF_INTERLOCK_BE;
 724                 record->flags |= HAMMER_RECF_DELETED_FE;
 725                 record->flush_state = HAMMER_FST_IDLE;
 726                 hammer_rel_mem_record(record);
 727
 728                 /*
 729                  * Finish up.
 730                  */
 731                 if (error == 0) {
 732                         if (hammer_debug_inode)
 733                                 kprintf("CLEANDELOND %p %08x\n", ip, ip->flags);
 734                         ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 735                                             HAMMER_INODE_ATIME |
 736                                             HAMMER_INODE_MTIME);
 737                         ip->flags &= ~HAMMER_INODE_DELONDISK;
 738                         if (redirty)
 739                                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
 740
 741                         /*
 742                          * Root volume count of inodes
 743                          */
 744                         if ((ip->flags & HAMMER_INODE_ONDISK) == 0) {
 745                                 hammer_modify_volume_field(trans,
 746                                                            trans->rootvol,
 747                                                            vol0_stat_inodes);
 748                                 ++ip->hmp->rootvol->ondisk->vol0_stat_inodes;
 749                                 hammer_modify_volume_done(trans->rootvol);
 750                                 ip->flags |= HAMMER_INODE_ONDISK;
 751                                 if (hammer_debug_inode)
 752                                         kprintf("NOWONDISK %p\n", ip);
 753                         }
 754                 }
 755         }
 756
 757         /*
 758          * If the inode has been destroyed, clean out any left-over flags
 759          * that may have been set by the frontend.
 760          */
 761         if (error == 0 && (ip->flags & HAMMER_INODE_DELETED)) {
 762                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY |
 763                                     HAMMER_INODE_ATIME |
 764                                     HAMMER_INODE_MTIME);
 765         }
 766         return(error);
 767 }
 768
 769 /*
 770  * Update only the itimes fields.
 771  *
 772  * ATIME can be updated without generating any UNDO.  MTIME is updated
 773  * with UNDO so it is guaranteed to be synchronized properly in case of
 774  * a crash.
 775  *
 776  * Neither field is included in the B-Tree leaf element's CRC, which is how
 777  * we can get away with updating ATIME the way we do.
 778  */
 779 static int
 780 hammer_update_itimes(hammer_cursor_t cursor, hammer_inode_t ip)
 781 {
 782         hammer_transaction_t trans = cursor->trans;
 783         int error;
 784
 785 retry:
 786         if ((ip->flags & (HAMMER_INODE_ONDISK|HAMMER_INODE_DELONDISK)) !=
 787             HAMMER_INODE_ONDISK) {
 788                 return(0);
 789         }
 790
 791         hammer_normalize_cursor(cursor);
 792         cursor->key_beg.localization = ip->obj_localization +
 793                                        HAMMER_LOCALIZE_INODE;
 794         cursor->key_beg.obj_id = ip->obj_id;
 795         cursor->key_beg.key = 0;
 796         cursor->key_beg.create_tid = 0;
 797         cursor->key_beg.delete_tid = 0;
 798         cursor->key_beg.rec_type = HAMMER_RECTYPE_INODE;
 799         cursor->key_beg.obj_type = 0;
 800         cursor->asof = ip->obj_asof;
 801         cursor->flags &= ~HAMMER_CURSOR_INITMASK;
 802         cursor->flags |= HAMMER_CURSOR_ASOF;
 803         cursor->flags |= HAMMER_CURSOR_GET_LEAF;
 804         cursor->flags |= HAMMER_CURSOR_GET_DATA;
 805         cursor->flags |= HAMMER_CURSOR_BACKEND;
 806
 807         error = hammer_btree_lookup(cursor);
 808         if (error) {
 809                 kprintf("error %d\n", error);
 810                 Debugger("hammer_update_itimes1");
 811         }
 812         if (error == 0) {
 813                 hammer_cache_node(&ip->cache[0], cursor->node);
 814                 if (ip->sync_flags & HAMMER_INODE_MTIME) {
 815                         /*
 816                          * Updating MTIME requires an UNDO.  Just cover
 817                          * both atime and mtime.
 818                          */
 819                         hammer_modify_buffer(trans, cursor->data_buffer,
 820                                      HAMMER_ITIMES_BASE(&cursor->data->inode),
 821                                      HAMMER_ITIMES_BYTES);
 822                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 823                         cursor->data->inode.mtime = ip->sync_ino_data.mtime;
 824                         hammer_modify_buffer_done(cursor->data_buffer);
 825                 } else if (ip->sync_flags & HAMMER_INODE_ATIME) {
 826                         /*
 827                          * Updating atime only can be done in-place with
 828                          * no UNDO.
 829                          */
 830                         hammer_modify_buffer(trans, cursor->data_buffer,
 831                                              NULL, 0);
 832                         cursor->data->inode.atime = ip->sync_ino_data.atime;
 833                         hammer_modify_buffer_done(cursor->data_buffer);
 834                 }
 835                 ip->sync_flags &= ~(HAMMER_INODE_ATIME | HAMMER_INODE_MTIME);
 836         }
 837         if (error == EDEADLK) {
 838                 hammer_done_cursor(cursor);
 839                 error = hammer_init_cursor(trans, cursor,
 840                                            &ip->cache[0], ip);
 841                 if (error == 0)
 842                         goto retry;
 843         }
 844         return(error);
 845 }
 846
 847 /*
 848  * Release a reference on an inode, flush as requested.
 849  *
 850  * On the last reference we queue the inode to the flusher for its final
 851  * disposition.
 852  */
 853 void
 854 hammer_rel_inode(struct hammer_inode *ip, int flush)
 855 {
 856         hammer_mount_t hmp = ip->hmp;
 857
 858         /*
 859          * Handle disposition when dropping the last ref.
 860          */
 861         for (;;) {
 862                 if (ip->lock.refs == 1) {
 863                         /*
 864                          * Determine whether on-disk action is needed for
 865                          * the inode's final disposition.
 866                          */
 867                         KKASSERT(ip->vp == NULL);
 868                         hammer_inode_unloadable_check(ip, 0);
 869                         if (ip->flags & HAMMER_INODE_MODMASK) {
 870                                 if (hmp->rsv_inodes > desiredvnodes) {
 871                                         hammer_flush_inode(ip,
 872                                                            HAMMER_FLUSH_SIGNAL);
 873                                 } else {
 874                                         hammer_flush_inode(ip, 0);
 875                                 }
 876                         } else if (ip->lock.refs == 1) {
 877                                 hammer_unload_inode(ip);
 878                                 break;
 879                         }
 880                 } else {
 881                         if (flush)
 882                                 hammer_flush_inode(ip, 0);
 883
 884                         /*
 885                          * The inode still has multiple refs, try to drop
 886                          * one ref.
 887                          */
 888                         KKASSERT(ip->lock.refs >= 1);
 889                         if (ip->lock.refs > 1) {
 890                                 hammer_unref(&ip->lock);
 891                                 break;
 892                         }
 893                 }
 894         }
 895 }
 896
 897 /*
 898  * Unload and destroy the specified inode.  Must be called with one remaining
 899  * reference.  The reference is disposed of.
 900  *
 901  * This can only be called in the context of the flusher.
 902  */
 903 static int
 904 hammer_unload_inode(struct hammer_inode *ip)
 905 {
 906         hammer_mount_t hmp = ip->hmp;
 907
 908         KASSERT(ip->lock.refs == 1,
 909                 ("hammer_unload_inode: %d refs\n", ip->lock.refs));
 910         KKASSERT(ip->vp == NULL);
 911         KKASSERT(ip->flush_state == HAMMER_FST_IDLE);
 912         KKASSERT(ip->cursor_ip_refs == 0);
 913         KKASSERT(ip->lock.lockcount == 0);
 914         KKASSERT((ip->flags & HAMMER_INODE_MODMASK) == 0);
 915
 916         KKASSERT(RB_EMPTY(&ip->rec_tree));
 917         KKASSERT(TAILQ_EMPTY(&ip->target_list));
 918
 919         RB_REMOVE(hammer_ino_rb_tree, &hmp->rb_inos_root, ip);
 920
 921         hammer_uncache_node(&ip->cache[0]);
 922         hammer_uncache_node(&ip->cache[1]);
 923         if (ip->objid_cache)
 924                 hammer_clear_objid(ip);
 925         --hammer_count_inodes;
 926         --hmp->count_inodes;
 927
 928         hammer_inode_wakereclaims(ip);
 929         kfree(ip, M_HAMMER);
 930
 931         return(0);
 932 }
 933
 934 /*
 935  * Called on mount -u when switching from RW to RO or vise-versa.  Adjust
 936  * the read-only flag for cached inodes.
 937  *
 938  * This routine is called from a RB_SCAN().
 939  */
 940 int
 941 hammer_reload_inode(hammer_inode_t ip, void *arg __unused)
 942 {
 943         hammer_mount_t hmp = ip->hmp;
 944
 945         if (hmp->ronly || hmp->asof != HAMMER_MAX_TID)
 946                 ip->flags |= HAMMER_INODE_RO;
 947         else
 948                 ip->flags &= ~HAMMER_INODE_RO;
 949         return(0);
 950 }
 951
 952 /*
 953  * A transaction has modified an inode, requiring updates as specified by
 954  * the passed flags.
 955  *
 956  * HAMMER_INODE_DDIRTY: Inode data has been updated
 957  * HAMMER_INODE_XDIRTY: Dirty in-memory records
 958  * HAMMER_INODE_BUFS:   Dirty buffer cache buffers
 959  * HAMMER_INODE_DELETED: Inode record/data must be deleted
 960  * HAMMER_INODE_ATIME/MTIME: mtime/atime has been updated
 961  */
 962 void
 963 hammer_modify_inode(hammer_inode_t ip, int flags)
 964 {
 965         KKASSERT ((ip->flags & HAMMER_INODE_RO) == 0 ||
 966                   (flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
 967                             HAMMER_INODE_BUFS | HAMMER_INODE_DELETED |
 968                             HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) == 0);
 969         if ((ip->flags & HAMMER_INODE_RSV_INODES) == 0) {
 970                 ip->flags |= HAMMER_INODE_RSV_INODES;
 971                 ++ip->hmp->rsv_inodes;
 972         }
 973
 974         ip->flags |= flags;
 975 }
 976
 977 /*
 978  * Request that an inode be flushed.  This whole mess cannot block and may
 979  * recurse (if not synchronous).  Once requested HAMMER will attempt to
 980  * actively flush the inode until the flush can be done.
 981  *
 982  * The inode may already be flushing, or may be in a setup state.  We can
 983  * place the inode in a flushing state if it is currently idle and flag it
 984  * to reflush if it is currently flushing.
 985  *
 986  * If the HAMMER_FLUSH_SYNCHRONOUS flag is specified we will attempt to
 987  * flush the indoe synchronously using the caller's context.
 988  */
 989 void
 990 hammer_flush_inode(hammer_inode_t ip, int flags)
 991 {
 992         int good;
 993
 994         /*
 995          * Trivial 'nothing to flush' case.  If the inode is ina SETUP
 996          * state we have to put it back into an IDLE state so we can
 997          * drop the extra ref.
 998          */
 999         if ((ip->flags & HAMMER_INODE_MODMASK) == 0) {
1000                 if (ip->flush_state == HAMMER_FST_SETUP) {
1001                         ip->flush_state = HAMMER_FST_IDLE;
1002                         hammer_rel_inode(ip, 0);
1003                 }
1004                 return;
1005         }
1006
1007         /*
1008          * Our flush action will depend on the current state.
1009          */
1010         switch(ip->flush_state) {
1011         case HAMMER_FST_IDLE:
1012                 /*
1013                  * We have no dependancies and can flush immediately.  Some
1014                  * our children may not be flushable so we have to re-test
1015                  * with that additional knowledge.
1016                  */
1017                 hammer_flush_inode_core(ip, flags);
1018                 break;
1019         case HAMMER_FST_SETUP:
1020                 /*
1021                  * Recurse upwards through dependancies via target_list
1022                  * and start their flusher actions going if possible.
1023                  *
1024                  * 'good' is our connectivity.  -1 means we have none and
1025                  * can't flush, 0 means there weren't any dependancies, and
1026                  * 1 means we have good connectivity.
1027                  */
1028                 good = hammer_setup_parent_inodes(ip);
1029
1030                 /*
1031                  * We can continue if good >= 0.  Determine how many records
1032                  * under our inode can be flushed (and mark them).
1033                  */
1034                 if (good >= 0) {
1035                         hammer_flush_inode_core(ip, flags);
1036                 } else {
1037                         ip->flags |= HAMMER_INODE_REFLUSH;
1038                         if (flags & HAMMER_FLUSH_SIGNAL) {
1039                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1040                                 hammer_flusher_async(ip->hmp);
1041                         }
1042                 }
1043                 break;
1044         default:
1045                 /*
1046                  * We are already flushing, flag the inode to reflush
1047                  * if needed after it completes its current flush.
1048                  */
1049                 if ((ip->flags & HAMMER_INODE_REFLUSH) == 0)
1050                         ip->flags |= HAMMER_INODE_REFLUSH;
1051                 if (flags & HAMMER_FLUSH_SIGNAL) {
1052                         ip->flags |= HAMMER_INODE_RESIGNAL;
1053                         hammer_flusher_async(ip->hmp);
1054                 }
1055                 break;
1056         }
1057 }
1058
1059 /*
1060  * Scan ip->target_list, which is a list of records owned by PARENTS to our
1061  * ip which reference our ip.
1062  *
1063  * XXX This is a huge mess of recursive code, but not one bit of it blocks
1064  *     so for now do not ref/deref the structures.  Note that if we use the
1065  *     ref/rel code later, the rel CAN block.
1066  */
1067 static int
1068 hammer_setup_parent_inodes(hammer_inode_t ip)
1069 {
1070         hammer_record_t depend;
1071 #if 0
1072         hammer_record_t next;
1073         hammer_inode_t  pip;
1074 #endif
1075         int good;
1076         int r;
1077
1078         good = 0;
1079         TAILQ_FOREACH(depend, &ip->target_list, target_entry) {
1080                 r = hammer_setup_parent_inodes_helper(depend);
1081                 KKASSERT(depend->target_ip == ip);
1082                 if (r < 0 && good == 0)
1083                         good = -1;
1084                 if (r > 0)
1085                         good = 1;
1086         }
1087         return(good);
1088
1089 #if 0
1090 retry:
1091         good = 0;
1092         next = TAILQ_FIRST(&ip->target_list);
1093         if (next) {
1094                 hammer_ref(&next->lock);
1095                 hammer_ref(&next->ip->lock);
1096         }
1097         while ((depend = next) != NULL) {
1098                 if (depend->target_ip == NULL) {
1099                         pip = depend->ip;
1100                         hammer_rel_mem_record(depend);
1101                         hammer_rel_inode(pip, 0);
1102                         goto retry;
1103                 }
1104                 KKASSERT(depend->target_ip == ip);
1105                 next = TAILQ_NEXT(depend, target_entry);
1106                 if (next) {
1107                         hammer_ref(&next->lock);
1108                         hammer_ref(&next->ip->lock);
1109                 }
1110                 r = hammer_setup_parent_inodes_helper(depend);
1111                 if (r < 0 && good == 0)
1112                         good = -1;
1113                 if (r > 0)
1114                         good = 1;
1115                 pip = depend->ip;
1116                 hammer_rel_mem_record(depend);
1117                 hammer_rel_inode(pip, 0);
1118         }
1119         return(good);
1120 #endif
1121 }
1122
1123 /*
1124  * This helper function takes a record representing the dependancy between
1125  * the parent inode and child inode.
1126  *
1127  * record->ip           = parent inode
1128  * record->target_ip    = child inode
1129  *
1130  * We are asked to recurse upwards and convert the record from SETUP
1131  * to FLUSH if possible.
1132  *
1133  * Return 1 if the record gives us connectivity
1134  *
1135  * Return 0 if the record is not relevant
1136  *
1137  * Return -1 if we can't resolve the dependancy and there is no connectivity.
1138  */
1139 static int
1140 hammer_setup_parent_inodes_helper(hammer_record_t record)
1141 {
1142         hammer_mount_t hmp;
1143         hammer_inode_t pip;
1144         int good;
1145
1146         KKASSERT(record->flush_state != HAMMER_FST_IDLE);
1147         pip = record->ip;
1148         hmp = pip->hmp;
1149
1150         /*
1151          * If the record is already flushing, is it in our flush group?
1152          *
1153          * If it is in our flush group but it is a general record or a
1154          * delete-on-disk, it does not improve our connectivity (return 0),
1155          * and if the target inode is not trying to destroy itself we can't
1156          * allow the operation yet anyway (the second return -1).
1157          */
1158         if (record->flush_state == HAMMER_FST_FLUSH) {
1159                 if (record->flush_group != hmp->flusher.next) {
1160                         pip->flags |= HAMMER_INODE_REFLUSH;
1161                         return(-1);
1162                 }
1163                 if (record->type == HAMMER_MEM_RECORD_ADD)
1164                         return(1);
1165                 /* GENERAL or DEL */
1166                 return(0);
1167         }
1168
1169         /*
1170          * It must be a setup record.  Try to resolve the setup dependancies
1171          * by recursing upwards so we can place ip on the flush list.
1172          */
1173         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1174
1175         good = hammer_setup_parent_inodes(pip);
1176
1177         /*
1178          * We can't flush ip because it has no connectivity (XXX also check
1179          * nlinks for pre-existing connectivity!).  Flag it so any resolution
1180          * recurses back down.
1181          */
1182         if (good < 0) {
1183                 pip->flags |= HAMMER_INODE_REFLUSH;
1184                 return(good);
1185         }
1186
1187         /*
1188          * We are go, place the parent inode in a flushing state so we can
1189          * place its record in a flushing state.  Note that the parent
1190          * may already be flushing.  The record must be in the same flush
1191          * group as the parent.
1192          */
1193         if (pip->flush_state != HAMMER_FST_FLUSH)
1194                 hammer_flush_inode_core(pip, HAMMER_FLUSH_RECURSION);
1195         KKASSERT(pip->flush_state == HAMMER_FST_FLUSH);
1196         KKASSERT(record->flush_state == HAMMER_FST_SETUP);
1197
1198 #if 0
1199         if (record->type == HAMMER_MEM_RECORD_DEL &&
1200             (record->target_ip->flags & (HAMMER_INODE_DELETED|HAMMER_INODE_DELONDISK)) == 0) {
1201                 /*
1202                  * Regardless of flushing state we cannot sync this path if the
1203                  * record represents a delete-on-disk but the target inode
1204                  * is not ready to sync its own deletion.
1205                  *
1206                  * XXX need to count effective nlinks to determine whether
1207                  * the flush is ok, otherwise removing a hardlink will
1208                  * just leave the DEL record to rot.
1209                  */
1210                 record->target_ip->flags |= HAMMER_INODE_REFLUSH;
1211                 return(-1);
1212         } else
1213 #endif
1214         if (pip->flush_group == pip->hmp->flusher.next) {
1215                 /*
1216                  * This is the record we wanted to synchronize.  If the
1217                  * record went into a flush state while we blocked it
1218                  * had better be in the correct flush group.
1219                  */
1220                 if (record->flush_state != HAMMER_FST_FLUSH) {
1221                         record->flush_state = HAMMER_FST_FLUSH;
1222                         record->flush_group = pip->flush_group;
1223                         hammer_ref(&record->lock);
1224                 } else {
1225                         KKASSERT(record->flush_group == pip->flush_group);
1226                 }
1227                 if (record->type == HAMMER_MEM_RECORD_ADD)
1228                         return(1);
1229
1230                 /*
1231                  * A general or delete-on-disk record does not contribute
1232                  * to our visibility.  We can still flush it, however.
1233                  */
1234                 return(0);
1235         } else {
1236                 /*
1237                  * We couldn't resolve the dependancies, request that the
1238                  * inode be flushed when the dependancies can be resolved.
1239                  */
1240                 pip->flags |= HAMMER_INODE_REFLUSH;
1241                 return(-1);
1242         }
1243 }
1244
1245 /*
1246  * This is the core routine placing an inode into the FST_FLUSH state.
1247  */
1248 static void
1249 hammer_flush_inode_core(hammer_inode_t ip, int flags)
1250 {
1251         int go_count;
1252
1253         /*
1254          * Set flush state and prevent the flusher from cycling into
1255          * the next flush group.  Do not place the ip on the list yet.
1256          * Inodes not in the idle state get an extra reference.
1257          */
1258         KKASSERT(ip->flush_state != HAMMER_FST_FLUSH);
1259         if (ip->flush_state == HAMMER_FST_IDLE)
1260                 hammer_ref(&ip->lock);
1261         ip->flush_state = HAMMER_FST_FLUSH;
1262         ip->flush_group = ip->hmp->flusher.next;
1263         ++ip->hmp->flusher.group_lock;
1264         ++ip->hmp->count_iqueued;
1265         ++hammer_count_iqueued;
1266
1267         /*
1268          * We need to be able to vfsync/truncate from the backend.
1269          */
1270         KKASSERT((ip->flags & HAMMER_INODE_VHELD) == 0);
1271         if (ip->vp && (ip->vp->v_flag & VINACTIVE) == 0) {
1272                 ip->flags |= HAMMER_INODE_VHELD;
1273                 vref(ip->vp);
1274         }
1275
1276         /*
1277          * Figure out how many in-memory records we can actually flush
1278          * (not including inode meta-data, buffers, etc).
1279          */
1280         if (flags & HAMMER_FLUSH_RECURSION) {
1281                 go_count = 1;
1282         } else {
1283                 go_count = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1284                                    hammer_setup_child_callback, NULL);
1285         }
1286
1287         /*
1288          * This is a more involved test that includes go_count.  If we
1289          * can't flush, flag the inode and return.  If go_count is 0 we
1290          * were are unable to flush any records in our rec_tree and
1291          * must ignore the XDIRTY flag.
1292          */
1293         if (go_count == 0) {
1294                 if ((ip->flags & HAMMER_INODE_MODMASK_NOXDIRTY) == 0) {
1295                         ip->flags |= HAMMER_INODE_REFLUSH;
1296
1297                         --ip->hmp->count_iqueued;
1298                         --hammer_count_iqueued;
1299
1300                         ip->flush_state = HAMMER_FST_SETUP;
1301                         if (ip->flags & HAMMER_INODE_VHELD) {
1302                                 ip->flags &= ~HAMMER_INODE_VHELD;
1303                                 vrele(ip->vp);
1304                         }
1305                         if (flags & HAMMER_FLUSH_SIGNAL) {
1306                                 ip->flags |= HAMMER_INODE_RESIGNAL;
1307                                 hammer_flusher_async(ip->hmp);
1308                         }
1309                         if (--ip->hmp->flusher.group_lock == 0)
1310                                 wakeup(&ip->hmp->flusher.group_lock);
1311                         return;
1312                 }
1313         }
1314
1315         /*
1316          * Snapshot the state of the inode for the backend flusher.
1317          *
1318          * The truncation must be retained in the frontend until after
1319          * we've actually performed the record deletion.
1320          *
1321          * We continue to retain sync_trunc_off even when all truncations
1322          * have been resolved as an optimization to determine if we can
1323          * skip the B-Tree lookup for overwrite deletions.
1324          *
1325          * NOTE: The DELETING flag is a mod flag, but it is also sticky,
1326          * and stays in ip->flags.  Once set, it stays set until the
1327          * inode is destroyed.
1328          */
1329         ip->sync_flags = (ip->flags & HAMMER_INODE_MODMASK);
1330         if (ip->sync_flags & HAMMER_INODE_TRUNCATED)
1331                 ip->sync_trunc_off = ip->trunc_off;
1332         ip->sync_ino_leaf = ip->ino_leaf;
1333         ip->sync_ino_data = ip->ino_data;
1334         ip->trunc_off = 0x7FFFFFFFFFFFFFFFLL;
1335         ip->flags &= ~HAMMER_INODE_MODMASK;
1336 #ifdef DEBUG_TRUNCATE
1337         if ((ip->sync_flags & HAMMER_INODE_TRUNCATED) && ip == HammerTruncIp)
1338                 kprintf("truncateS %016llx\n", ip->sync_trunc_off);
1339 #endif
1340
1341         /*
1342          * The flusher list inherits our inode and reference.
1343          */
1344         TAILQ_INSERT_TAIL(&ip->hmp->flush_list, ip, flush_entry);
1345         if (--ip->hmp->flusher.group_lock == 0)
1346                 wakeup(&ip->hmp->flusher.group_lock);
1347
1348         if (flags & HAMMER_FLUSH_SIGNAL) {
1349                 hammer_flusher_async(ip->hmp);
1350         }
1351 }
1352
1353 /*
1354  * Callback for scan of ip->rec_tree.  Try to include each record in our
1355  * flush.  ip->flush_group has been set but the inode has not yet been
1356  * moved into a flushing state.
1357  *
1358  * If we get stuck on a record we have to set HAMMER_INODE_REFLUSH on
1359  * both inodes.
1360  *
1361  * We return 1 for any record placed or found in FST_FLUSH, which prevents
1362  * the caller from shortcutting the flush.
1363  */
1364 static int
1365 hammer_setup_child_callback(hammer_record_t rec, void *data)
1366 {
1367         hammer_inode_t target_ip;
1368         hammer_inode_t ip;
1369         int r;
1370
1371         /*
1372          * Deleted records are ignored.  Note that the flush detects deleted
1373          * front-end records at multiple points to deal with races.  This is
1374          * just the first line of defense.  The only time DELETED_FE cannot
1375          * be set is when HAMMER_RECF_INTERLOCK_BE is set.
1376          *
1377          * Don't get confused between record deletion and, say, directory
1378          * entry deletion.  The deletion of a directory entry that is on
1379          * the media has nothing to do with the record deletion flags.
1380          */
1381         if (rec->flags & (HAMMER_RECF_DELETED_FE|HAMMER_RECF_DELETED_BE))
1382                 return(0);
1383
1384         /*
1385          * If the record is in an idle state it has no dependancies and
1386          * can be flushed.
1387          */
1388         ip = rec->ip;
1389         r = 0;
1390
1391         switch(rec->flush_state) {
1392         case HAMMER_FST_IDLE:
1393                 /*
1394                  * Record has no setup dependancy, we can flush it.
1395                  */
1396                 KKASSERT(rec->target_ip == NULL);
1397                 rec->flush_state = HAMMER_FST_FLUSH;
1398                 rec->flush_group = ip->flush_group;
1399                 hammer_ref(&rec->lock);
1400                 r = 1;
1401                 break;
1402         case HAMMER_FST_SETUP:
1403                 /*
1404                  * Record has a setup dependancy.  Try to include the
1405                  * target ip in the flush.
1406                  *
1407                  * We have to be careful here, if we do not do the right
1408                  * thing we can lose track of dirty inodes and the system
1409                  * will lockup trying to allocate buffers.
1410                  */
1411                 target_ip = rec->target_ip;
1412                 KKASSERT(target_ip != NULL);
1413                 KKASSERT(target_ip->flush_state != HAMMER_FST_IDLE);
1414                 if (target_ip->flush_state == HAMMER_FST_FLUSH) {
1415                         /*
1416                          * If the target IP is already flushing in our group
1417                          * we are golden, otherwise make sure the target
1418                          * reflushes.
1419                          */
1420                         if (target_ip->flush_group == ip->flush_group) {
1421                                 rec->flush_state = HAMMER_FST_FLUSH;
1422                                 rec->flush_group = ip->flush_group;
1423                                 hammer_ref(&rec->lock);
1424                                 r = 1;
1425                         } else {
1426                                 target_ip->flags |= HAMMER_INODE_REFLUSH;
1427                         }
1428                 } else if (rec->type == HAMMER_MEM_RECORD_ADD) {
1429                         /*
1430                          * If the target IP is not flushing we can force
1431                          * it to flush, even if it is unable to write out
1432                          * any of its own records we have at least one in
1433                          * hand that we CAN deal with.
1434                          */
1435                         rec->flush_state = HAMMER_FST_FLUSH;
1436                         rec->flush_group = ip->flush_group;
1437                         hammer_ref(&rec->lock);
1438                         hammer_flush_inode_core(target_ip,
1439                                                 HAMMER_FLUSH_RECURSION);
1440                         r = 1;
1441                 } else {
1442                         /*
1443                          * General or delete-on-disk record.
1444                          *
1445                          * XXX this needs help.  If a delete-on-disk we could
1446                          * disconnect the target.  If the target has its own
1447                          * dependancies they really need to be flushed.
1448                          *
1449                          * XXX
1450                          */
1451                         rec->flush_state = HAMMER_FST_FLUSH;
1452                         rec->flush_group = ip->flush_group;
1453                         hammer_ref(&rec->lock);
1454                         hammer_flush_inode_core(target_ip,
1455                                                 HAMMER_FLUSH_RECURSION);
1456                         r = 1;
1457                 }
1458                 break;
1459         case HAMMER_FST_FLUSH:
1460                 /*
1461                  * If the WOULDBLOCK flag is set records may have been left
1462                  * over from a previous flush attempt and should be moved
1463                  * to the current flush group.  If it is not set then all
1464                  * such records had better have been flushed already or
1465                  * already associated with the current flush group.
1466                  */
1467                 if (ip->flags & HAMMER_INODE_WOULDBLOCK) {
1468                         kprintf("b");
1469                         rec->flush_group = ip->flush_group;
1470                 } else {
1471                         KKASSERT(rec->flush_group == ip->flush_group);
1472                 }
1473                 r = 1;
1474                 break;
1475         }
1476         return(r);
1477 }
1478
1479 /*
1480  * Wait for a previously queued flush to complete.  Not only do we need to
1481  * wait for the inode to sync out, we also may have to run the flusher again
1482  * to get it past the UNDO position pertaining to the flush so a crash does
1483  * not 'undo' our flush.
1484  */
1485 void
1486 hammer_wait_inode(hammer_inode_t ip)
1487 {
1488         hammer_mount_t hmp = ip->hmp;
1489         int sync_group;
1490         int waitcount;
1491
1492         sync_group = ip->flush_group;
1493         waitcount = (ip->flags & HAMMER_INODE_REFLUSH) ? 2 : 1;
1494
1495         if (ip->flush_state == HAMMER_FST_SETUP) {
1496                 hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1497         }
1498         /* XXX can we make this != FST_IDLE ? check SETUP depends */
1499         while (ip->flush_state == HAMMER_FST_FLUSH &&
1500                (ip->flush_group - sync_group) < waitcount) {
1501                 ip->flags |= HAMMER_INODE_FLUSHW;
1502                 tsleep(&ip->flags, 0, "hmrwin", 0);
1503         }
1504         while (hmp->flusher.done - sync_group < waitcount) {
1505                 kprintf("Y");
1506                 hammer_flusher_sync(hmp);
1507         }
1508 }
1509
1510 /*
1511  * Called by the backend code when a flush has been completed.
1512  * The inode has already been removed from the flush list.
1513  *
1514  * A pipelined flush can occur, in which case we must re-enter the
1515  * inode on the list and re-copy its fields.
1516  */
1517 void
1518 hammer_flush_inode_done(hammer_inode_t ip)
1519 {
1520         hammer_mount_t hmp;
1521         int dorel;
1522
1523         KKASSERT(ip->flush_state == HAMMER_FST_FLUSH);
1524
1525         hmp = ip->hmp;
1526
1527         /*
1528          * Merge left-over flags back into the frontend and fix the state.
1529          */
1530         ip->flags |= ip->sync_flags;
1531
1532         /*
1533          * The backend may have adjusted nlinks, so if the adjusted nlinks
1534          * does not match the fronttend set the frontend's RDIRTY flag again.
1535          */
1536         if (ip->ino_data.nlinks != ip->sync_ino_data.nlinks)
1537                 ip->flags |= HAMMER_INODE_DDIRTY;
1538
1539         /*
1540          * Fix up the dirty buffer status.  IO completions will also
1541          * try to clean up rsv_databufs.
1542          */
1543         if (ip->vp && RB_ROOT(&ip->vp->v_rbdirty_tree)) {
1544                 ip->flags |= HAMMER_INODE_BUFS;
1545         } else {
1546                 hmp->rsv_databufs -= ip->rsv_databufs;
1547                 ip->rsv_databufs = 0;
1548         }
1549
1550         /*
1551          * Re-set the XDIRTY flag if some of the inode's in-memory records
1552          * could not be flushed.
1553          */
1554         KKASSERT((RB_EMPTY(&ip->rec_tree) &&
1555                   (ip->flags & HAMMER_INODE_XDIRTY) == 0) ||
1556                  (!RB_EMPTY(&ip->rec_tree) &&
1557                   (ip->flags & HAMMER_INODE_XDIRTY) != 0));
1558
1559         /*
1560          * Do not lose track of inodes which no longer have vnode
1561          * assocations, otherwise they may never get flushed again.
1562          */
1563         if ((ip->flags & HAMMER_INODE_MODMASK) && ip->vp == NULL)
1564                 ip->flags |= HAMMER_INODE_REFLUSH;
1565
1566         /*
1567          * Adjust flush_state.  The target state (idle or setup) shouldn't
1568          * be terribly important since we will reflush if we really need
1569          * to do anything.
1570          *
1571          * If the WOULDBLOCK flag is set we must re-flush immediately
1572          * to continue a potentially large deletion.  The flag also causes
1573          * the hammer_setup_child_callback() to move records in the old
1574          * flush group to the new one.
1575          */
1576         if (ip->flags & HAMMER_INODE_WOULDBLOCK) {
1577                 kprintf("B");
1578                 ip->flush_state = HAMMER_FST_IDLE;
1579                 hammer_flush_inode_core(ip, HAMMER_FLUSH_SIGNAL);
1580                 ip->flags &= ~HAMMER_INODE_WOULDBLOCK;
1581                 dorel = 1;
1582         } else if (TAILQ_EMPTY(&ip->target_list) && RB_EMPTY(&ip->rec_tree)) {
1583                 ip->flush_state = HAMMER_FST_IDLE;
1584                 dorel = 1;
1585         } else {
1586                 ip->flush_state = HAMMER_FST_SETUP;
1587                 dorel = 0;
1588         }
1589
1590         --hmp->count_iqueued;
1591         --hammer_count_iqueued;
1592
1593         /*
1594          * Clean up the vnode ref
1595          */
1596         if (ip->flags & HAMMER_INODE_VHELD) {
1597                 ip->flags &= ~HAMMER_INODE_VHELD;
1598                 vrele(ip->vp);
1599         }
1600
1601         /*
1602          * If the frontend made more changes and requested another flush,
1603          * then try to get it running.
1604          */
1605         if (ip->flags & HAMMER_INODE_REFLUSH) {
1606                 ip->flags &= ~HAMMER_INODE_REFLUSH;
1607                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
1608                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
1609                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
1610                 } else {
1611                         hammer_flush_inode(ip, 0);
1612                 }
1613         }
1614
1615         /*
1616          * If the inode is now clean drop the space reservation.
1617          */
1618         if ((ip->flags & HAMMER_INODE_MODMASK) == 0 &&
1619             (ip->flags & HAMMER_INODE_RSV_INODES)) {
1620                 ip->flags &= ~HAMMER_INODE_RSV_INODES;
1621                 --hmp->rsv_inodes;
1622         }
1623
1624         /*
1625          * Finally, if the frontend is waiting for a flush to complete,
1626          * wake it up.
1627          */
1628         if (ip->flush_state != HAMMER_FST_FLUSH) {
1629                 if (ip->flags & HAMMER_INODE_FLUSHW) {
1630                         ip->flags &= ~HAMMER_INODE_FLUSHW;
1631                         wakeup(&ip->flags);
1632                 }
1633         }
1634         if (dorel)
1635                 hammer_rel_inode(ip, 0);
1636 }
1637
1638 /*
1639  * Called from hammer_sync_inode() to synchronize in-memory records
1640  * to the media.
1641  */
1642 static int
1643 hammer_sync_record_callback(hammer_record_t record, void *data)
1644 {
1645         hammer_cursor_t cursor = data;
1646         hammer_transaction_t trans = cursor->trans;
1647         int error;
1648
1649         /*
1650          * Skip records that do not belong to the current flush.
1651          */
1652         ++hammer_stats_record_iterations;
1653         if (record->flush_state != HAMMER_FST_FLUSH)
1654                 return(0);
1655
1656 #if 1
1657         if (record->flush_group != record->ip->flush_group) {
1658                 kprintf("sync_record %p ip %p bad flush group %d %d\n", record, record->ip, record->flush_group ,record->ip->flush_group);
1659                 Debugger("blah2");
1660                 return(0);
1661         }
1662 #endif
1663         KKASSERT(record->flush_group == record->ip->flush_group);
1664
1665         /*
1666          * Interlock the record using the BE flag.  Once BE is set the
1667          * frontend cannot change the state of FE.
1668          *
1669          * NOTE: If FE is set prior to us setting BE we still sync the
1670          * record out, but the flush completion code converts it to
1671          * a delete-on-disk record instead of destroying it.
1672          */
1673         KKASSERT((record->flags & HAMMER_RECF_INTERLOCK_BE) == 0);
1674         record->flags |= HAMMER_RECF_INTERLOCK_BE;
1675
1676         /*
1677          * The backend may have already disposed of the record.
1678          */
1679         if (record->flags & HAMMER_RECF_DELETED_BE) {
1680                 error = 0;
1681                 goto done;
1682         }
1683
1684         /*
1685          * If the whole inode is being deleting all on-disk records will
1686          * be deleted very soon, we can't sync any new records to disk
1687          * because they will be deleted in the same transaction they were
1688          * created in (delete_tid == create_tid), which will assert.
1689          *
1690          * XXX There may be a case with RECORD_ADD with DELETED_FE set
1691          * that we currently panic on.
1692          */
1693         if (record->ip->sync_flags & HAMMER_INODE_DELETING) {
1694                 switch(record->type) {
1695                 case HAMMER_MEM_RECORD_DATA:
1696                         /*
1697                          * We don't have to do anything, if the record was
1698                          * committed the space will have been accounted for
1699                          * in the blockmap.
1700                          */
1701                         /* fall through */
1702                 case HAMMER_MEM_RECORD_GENERAL:
1703                         record->flags |= HAMMER_RECF_DELETED_FE;
1704                         record->flags |= HAMMER_RECF_DELETED_BE;
1705                         error = 0;
1706                         goto done;
1707                 case HAMMER_MEM_RECORD_ADD:
1708                         panic("hammer_sync_record_callback: illegal add "
1709                               "during inode deletion record %p", record);
1710                         break; /* NOT REACHED */
1711                 case HAMMER_MEM_RECORD_INODE:
1712                         panic("hammer_sync_record_callback: attempt to "
1713                               "sync inode record %p?", record);
1714                         break; /* NOT REACHED */
1715                 case HAMMER_MEM_RECORD_DEL:
1716                         /*
1717                          * Follow through and issue the on-disk deletion
1718                          */
1719                         break;
1720                 }
1721         }
1722
1723         /*
1724          * If DELETED_FE is set special handling is needed for directory
1725          * entries.  Dependant pieces related to the directory entry may
1726          * have already been synced to disk.  If this occurs we have to
1727          * sync the directory entry and then change the in-memory record
1728          * from an ADD to a DELETE to cover the fact that it's been
1729          * deleted by the frontend.
1730          *
1731          * A directory delete covering record (MEM_RECORD_DEL) can never
1732          * be deleted by the frontend.
1733          *
1734          * Any other record type (aka DATA) can be deleted by the frontend.
1735          * XXX At the moment the flusher must skip it because there may
1736          * be another data record in the flush group for the same block,
1737          * meaning that some frontend data changes can leak into the backend's
1738          * synchronization point.
1739          */
1740         if (record->flags & HAMMER_RECF_DELETED_FE) {
1741                 if (record->type == HAMMER_MEM_RECORD_ADD) {
1742                         record->flags |= HAMMER_RECF_CONVERT_DELETE;
1743                 } else {
1744                         KKASSERT(record->type != HAMMER_MEM_RECORD_DEL);
1745                         record->flags |= HAMMER_RECF_DELETED_BE;
1746                         error = 0;
1747                         goto done;
1748                 }
1749         }
1750
1751         /*
1752          * Assign the create_tid for new records.  Deletions already
1753          * have the record's entire key properly set up.
1754          */
1755         if (record->type != HAMMER_MEM_RECORD_DEL)
1756                 record->leaf.base.create_tid = trans->tid;
1757                 record->leaf.create_ts = trans->time32;
1758         for (;;) {
1759                 error = hammer_ip_sync_record_cursor(cursor, record);
1760                 if (error != EDEADLK)
1761                         break;
1762                 hammer_done_cursor(cursor);
1763                 error = hammer_init_cursor(trans, cursor, &record->ip->cache[0],
1764                                            record->ip);
1765                 if (error)
1766                         break;
1767         }
1768         record->flags &= ~HAMMER_RECF_CONVERT_DELETE;
1769
1770         if (error) {
1771                 error = -error;
1772                 if (error != -ENOSPC) {
1773                         kprintf("hammer_sync_record_callback: sync failed rec "
1774                                 "%p, error %d\n", record, error);
1775                         Debugger("sync failed rec");
1776                 }
1777         }
1778 done:
1779         hammer_flush_record_done(record, error);
1780         return(error);
1781 }
1782
1783 /*
1784  * XXX error handling
1785  */
1786 int
1787 hammer_sync_inode(hammer_inode_t ip)
1788 {
1789         struct hammer_transaction trans;
1790         struct hammer_cursor cursor;
1791         hammer_node_t tmp_node;
1792         hammer_record_t depend;
1793         hammer_record_t next;
1794         int error, tmp_error;
1795         u_int64_t nlinks;
1796
1797         if ((ip->sync_flags & HAMMER_INODE_MODMASK) == 0)
1798                 return(0);
1799
1800         hammer_start_transaction_fls(&trans, ip->hmp);
1801         error = hammer_init_cursor(&trans, &cursor, &ip->cache[1], ip);
1802         if (error)
1803                 goto done;
1804
1805         /*
1806          * Any directory records referencing this inode which are not in
1807          * our current flush group must adjust our nlink count for the
1808          * purposes of synchronization to disk.
1809          *
1810          * Records which are in our flush group can be unlinked from our
1811          * inode now, potentially allowing the inode to be physically
1812          * deleted.
1813          *
1814          * This cannot block.
1815          */
1816         nlinks = ip->ino_data.nlinks;
1817         next = TAILQ_FIRST(&ip->target_list);
1818         while ((depend = next) != NULL) {
1819                 next = TAILQ_NEXT(depend, target_entry);
1820                 if (depend->flush_state == HAMMER_FST_FLUSH &&
1821                     depend->flush_group == ip->hmp->flusher.act) {
1822                         /*
1823                          * If this is an ADD that was deleted by the frontend
1824                          * the frontend nlinks count will have already been
1825                          * decremented, but the backend is going to sync its
1826                          * directory entry and must account for it.  The
1827                          * record will be converted to a delete-on-disk when
1828                          * it gets synced.
1829                          *
1830                          * If the ADD was not deleted by the frontend we
1831                          * can remove the dependancy from our target_list.
1832                          */
1833                         if (depend->flags & HAMMER_RECF_DELETED_FE) {
1834                                 ++nlinks;
1835                         } else {
1836                                 TAILQ_REMOVE(&ip->target_list, depend,
1837                                              target_entry);
1838                                 depend->target_ip = NULL;
1839                         }
1840                 } else if ((depend->flags & HAMMER_RECF_DELETED_FE) == 0) {
1841                         /*
1842                          * Not part of our flush group
1843                          */
1844                         KKASSERT((depend->flags & HAMMER_RECF_DELETED_BE) == 0);
1845                         switch(depend->type) {
1846                         case HAMMER_MEM_RECORD_ADD:
1847                                 --nlinks;
1848                                 break;
1849                         case HAMMER_MEM_RECORD_DEL:
1850                                 ++nlinks;
1851                                 break;
1852                         default:
1853                                 break;
1854                         }
1855                 }
1856         }
1857
1858         /*
1859          * Set dirty if we had to modify the link count.
1860          */
1861         if (ip->sync_ino_data.nlinks != nlinks) {
1862                 KKASSERT((int64_t)nlinks >= 0);
1863                 ip->sync_ino_data.nlinks = nlinks;
1864                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
1865         }
1866
1867         /*
1868          * If there is a trunction queued destroy any data past the (aligned)
1869          * truncation point.  Userland will have dealt with the buffer
1870          * containing the truncation point for us.
1871          *
1872          * We don't flush pending frontend data buffers until after we've
1873          * dealt with the truncation.
1874          */
1875         if (ip->sync_flags & HAMMER_INODE_TRUNCATED) {
1876                 /*
1877                  * Interlock trunc_off.  The VOP front-end may continue to
1878                  * make adjustments to it while we are blocked.
1879                  */
1880                 off_t trunc_off;
1881                 off_t aligned_trunc_off;
1882                 int blkmask;
1883
1884                 trunc_off = ip->sync_trunc_off;
1885                 blkmask = hammer_blocksize(trunc_off) - 1;
1886                 aligned_trunc_off = (trunc_off + blkmask) & ~(int64_t)blkmask;
1887
1888                 /*
1889                  * Delete any whole blocks on-media.  The front-end has
1890                  * already cleaned out any partial block and made it
1891                  * pending.  The front-end may have updated trunc_off
1892                  * while we were blocked so we only use sync_trunc_off.
1893                  *
1894                  * This operation can blow out the buffer cache, EWOULDBLOCK
1895                  * means we were unable to complete the deletion.
1896                  */
1897                 error = hammer_ip_delete_range(&cursor, ip,
1898                                                 aligned_trunc_off,
1899                                                 0x7FFFFFFFFFFFFFFFLL, 2);
1900                 if (error == EWOULDBLOCK) {
1901                         ip->flags |= HAMMER_INODE_WOULDBLOCK;
1902                         error = 0;
1903                         goto defer_buffer_flush;
1904                 }
1905
1906                 if (error)
1907                         Debugger("hammer_ip_delete_range errored");
1908
1909                 /*
1910                  * Clear the truncation flag on the backend after we have
1911                  * complete the deletions.  Backend data is now good again
1912                  * (including new records we are about to sync, below).
1913                  *
1914                  * Leave sync_trunc_off intact.  As we write additional
1915                  * records the backend will update sync_trunc_off.  This
1916                  * tells the backend whether it can skip the overwrite
1917                  * test.  This should work properly even when the backend
1918                  * writes full blocks where the truncation point straddles
1919                  * the block because the comparison is against the base
1920                  * offset of the record.
1921                  */
1922                 ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1923                 /* ip->sync_trunc_off = 0x7FFFFFFFFFFFFFFFLL; */
1924         } else {
1925                 error = 0;
1926         }
1927
1928         /*
1929          * Now sync related records.  These will typically be directory
1930          * entries or delete-on-disk records.
1931          *
1932          * Not all records will be flushed, but clear XDIRTY anyway.  We
1933          * will set it again in the frontend hammer_flush_inode_done()
1934          * if records remain.
1935          */
1936         if (error == 0) {
1937                 tmp_error = RB_SCAN(hammer_rec_rb_tree, &ip->rec_tree, NULL,
1938                                     hammer_sync_record_callback, &cursor);
1939                 if (tmp_error < 0)
1940                         tmp_error = -error;
1941                 if (tmp_error)
1942                         error = tmp_error;
1943         }
1944         hammer_cache_node(&ip->cache[1], cursor.node);
1945
1946         /*
1947          * Re-seek for inode update, assuming our cache hasn't been ripped
1948          * out from under us.
1949          */
1950         if (error == 0) {
1951                 tmp_node = hammer_ref_node_safe(ip->hmp, &ip->cache[0], &error);
1952                 if (tmp_node) {
1953                         if ((tmp_node->flags & HAMMER_NODE_DELETED) == 0)
1954                                 hammer_cursor_seek(&cursor, tmp_node, 0);
1955                         hammer_rel_node(tmp_node);
1956                 }
1957                 error = 0;
1958         }
1959
1960         /*
1961          * If we are deleting the inode the frontend had better not have
1962          * any active references on elements making up the inode.
1963          */
1964         if (error == 0 && ip->sync_ino_data.nlinks == 0 &&
1965                 RB_EMPTY(&ip->rec_tree)  &&
1966             (ip->sync_flags & HAMMER_INODE_DELETING) &&
1967             (ip->flags & HAMMER_INODE_DELETED) == 0) {
1968                 int count1 = 0;
1969
1970                 error = hammer_ip_delete_range_all(&cursor, ip, &count1);
1971                 if (error == 0) {
1972                         ip->flags |= HAMMER_INODE_DELETED;
1973                         ip->sync_flags &= ~HAMMER_INODE_DELETING;
1974                         ip->sync_flags &= ~HAMMER_INODE_TRUNCATED;
1975                         KKASSERT(RB_EMPTY(&ip->rec_tree));
1976
1977                         /*
1978                          * Set delete_tid in both the frontend and backend
1979                          * copy of the inode record.  The DELETED flag handles
1980                          * this, do not set RDIRTY.
1981                          */
1982                         ip->ino_leaf.base.delete_tid = trans.tid;
1983                         ip->sync_ino_leaf.base.delete_tid = trans.tid;
1984                         ip->ino_leaf.delete_ts = trans.time32;
1985                         ip->sync_ino_leaf.delete_ts = trans.time32;
1986
1987
1988                         /*
1989                          * Adjust the inode count in the volume header
1990                          */
1991                         if (ip->flags & HAMMER_INODE_ONDISK) {
1992                                 hammer_modify_volume_field(&trans,
1993                                                            trans.rootvol,
1994                                                            vol0_stat_inodes);
1995                                 --ip->hmp->rootvol->ondisk->vol0_stat_inodes;
1996                                 hammer_modify_volume_done(trans.rootvol);
1997                         }
1998                 } else if (error == EWOULDBLOCK) {
1999                         ip->flags |= HAMMER_INODE_WOULDBLOCK;
2000                         error = 0;
2001                         goto defer_buffer_flush;
2002                 } else {
2003                         Debugger("hammer_ip_delete_range_all errored");
2004                 }
2005         }
2006
2007         ip->sync_flags &= ~HAMMER_INODE_BUFS;
2008
2009         if (error)
2010                 Debugger("RB_SCAN errored");
2011
2012 defer_buffer_flush:
2013         /*
2014          * Now update the inode's on-disk inode-data and/or on-disk record.
2015          * DELETED and ONDISK are managed only in ip->flags.
2016          *
2017          * In the case of a defered buffer flush we still update the on-disk
2018          * inode to satisfy visibility requirements if there happen to be
2019          * directory dependancies.
2020          */
2021         switch(ip->flags & (HAMMER_INODE_DELETED | HAMMER_INODE_ONDISK)) {
2022         case HAMMER_INODE_DELETED|HAMMER_INODE_ONDISK:
2023                 /*
2024                  * If deleted and on-disk, don't set any additional flags.
2025                  * the delete flag takes care of things.
2026                  *
2027                  * Clear flags which may have been set by the frontend.
2028                  */
2029                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
2030                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
2031                                     HAMMER_INODE_DELETING);
2032                 break;
2033         case HAMMER_INODE_DELETED:
2034                 /*
2035                  * Take care of the case where a deleted inode was never
2036                  * flushed to the disk in the first place.
2037                  *
2038                  * Clear flags which may have been set by the frontend.
2039                  */
2040                 ip->sync_flags &= ~(HAMMER_INODE_DDIRTY | HAMMER_INODE_XDIRTY |
2041                                     HAMMER_INODE_ATIME | HAMMER_INODE_MTIME |
2042                                     HAMMER_INODE_DELETING);
2043                 while (RB_ROOT(&ip->rec_tree)) {
2044                         hammer_record_t record = RB_ROOT(&ip->rec_tree);
2045                         hammer_ref(&record->lock);
2046                         KKASSERT(record->lock.refs == 1);
2047                         record->flags |= HAMMER_RECF_DELETED_FE;
2048                         record->flags |= HAMMER_RECF_DELETED_BE;
2049                         hammer_rel_mem_record(record);
2050                 }
2051                 break;
2052         case HAMMER_INODE_ONDISK:
2053                 /*
2054                  * If already on-disk, do not set any additional flags.
2055                  */
2056                 break;
2057         default:
2058                 /*
2059                  * If not on-disk and not deleted, set DDIRTY to force
2060                  * an initial record to be written.
2061                  *
2062                  * Also set the create_tid in both the frontend and backend
2063                  * copy of the inode record.
2064                  */
2065                 ip->ino_leaf.base.create_tid = trans.tid;
2066                 ip->ino_leaf.create_ts = trans.time32;
2067                 ip->sync_ino_leaf.base.create_tid = trans.tid;
2068                 ip->sync_ino_leaf.create_ts = trans.time32;
2069                 ip->sync_flags |= HAMMER_INODE_DDIRTY;
2070                 break;
2071         }
2072
2073         /*
2074          * If RDIRTY or DDIRTY is set, write out a new record.  If the inode
2075          * is already on-disk the old record is marked as deleted.
2076          *
2077          * If DELETED is set hammer_update_inode() will delete the existing
2078          * record without writing out a new one.
2079          *
2080          * If *ONLY* the ITIMES flag is set we can update the record in-place.
2081          */
2082         if (ip->flags & HAMMER_INODE_DELETED) {
2083                 error = hammer_update_inode(&cursor, ip);
2084         } else
2085         if ((ip->sync_flags & HAMMER_INODE_DDIRTY) == 0 &&
2086             (ip->sync_flags & (HAMMER_INODE_ATIME | HAMMER_INODE_MTIME))) {
2087                 error = hammer_update_itimes(&cursor, ip);
2088         } else
2089         if (ip->sync_flags & (HAMMER_INODE_DDIRTY | HAMMER_INODE_ATIME | HAMMER_INODE_MTIME)) {
2090                 error = hammer_update_inode(&cursor, ip);
2091         }
2092         if (error)
2093                 Debugger("hammer_update_itimes/inode errored");
2094 done:
2095         /*
2096          * Save the TID we used to sync the inode with to make sure we
2097          * do not improperly reuse it.
2098          */
2099         hammer_done_cursor(&cursor);
2100         hammer_done_transaction(&trans);
2101         return(error);
2102 }
2103
2104 /*
2105  * This routine is called when the OS is no longer actively referencing
2106  * the inode (but might still be keeping it cached), or when releasing
2107  * the last reference to an inode.
2108  *
2109  * At this point if the inode's nlinks count is zero we want to destroy
2110  * it, which may mean destroying it on-media too.
2111  */
2112 void
2113 hammer_inode_unloadable_check(hammer_inode_t ip, int getvp)
2114 {
2115         struct vnode *vp;
2116
2117         /*
2118          * Set the DELETING flag when the link count drops to 0 and the
2119          * OS no longer has any opens on the inode.
2120          *
2121          * The backend will clear DELETING (a mod flag) and set DELETED
2122          * (a state flag) when it is actually able to perform the
2123          * operation.
2124          */
2125         if (ip->ino_data.nlinks == 0 &&
2126             (ip->flags & (HAMMER_INODE_DELETING|HAMMER_INODE_DELETED)) == 0) {
2127                 ip->flags |= HAMMER_INODE_DELETING;
2128                 ip->flags |= HAMMER_INODE_TRUNCATED;
2129                 ip->trunc_off = 0;
2130                 vp = NULL;
2131                 if (getvp) {
2132                         if (hammer_get_vnode(ip, &vp) != 0)
2133                                 return;
2134                 }
2135
2136                 /*
2137                  * Final cleanup
2138                  */
2139                 if (ip->vp) {
2140                         vtruncbuf(ip->vp, 0, HAMMER_BUFSIZE);
2141                         vnode_pager_setsize(ip->vp, 0);
2142                 }
2143                 if (getvp) {
2144                         vput(vp);
2145                 }
2146         }
2147 }
2148
2149 /*
2150  * Re-test an inode when a dependancy had gone away to see if we
2151  * can chain flush it.
2152  */
2153 void
2154 hammer_test_inode(hammer_inode_t ip)
2155 {
2156         if (ip->flags & HAMMER_INODE_REFLUSH) {
2157                 ip->flags &= ~HAMMER_INODE_REFLUSH;
2158                 hammer_ref(&ip->lock);
2159                 if (ip->flags & HAMMER_INODE_RESIGNAL) {
2160                         ip->flags &= ~HAMMER_INODE_RESIGNAL;
2161                         hammer_flush_inode(ip, HAMMER_FLUSH_SIGNAL);
2162                 } else {
2163                         hammer_flush_inode(ip, 0);
2164                 }
2165                 hammer_rel_inode(ip, 0);
2166         }
2167 }
2168
2169 /*
2170  * Clear the RECLAIM flag on an inode.  This occurs when the inode is
2171  * reassociated with a vp or just before it gets freed.
2172  *
2173  * Wakeup one thread blocked waiting on reclaims to complete.  Note that
2174  * the inode the thread is waiting on behalf of is a different inode then
2175  * the inode we are called with.  This is to create a pipeline.
2176  */
2177 static void
2178 hammer_inode_wakereclaims(hammer_inode_t ip)
2179 {
2180         struct hammer_reclaim *reclaim;
2181         hammer_mount_t hmp = ip->hmp;
2182
2183         if ((ip->flags & HAMMER_INODE_RECLAIM) == 0)
2184                 return;
2185
2186         --hammer_count_reclaiming;
2187         --hmp->inode_reclaims;
2188         ip->flags &= ~HAMMER_INODE_RECLAIM;
2189
2190         if ((reclaim = TAILQ_FIRST(&hmp->reclaim_list)) != NULL) {
2191                 TAILQ_REMOVE(&hmp->reclaim_list, reclaim, entry);
2192                 reclaim->okydoky = 1;
2193                 wakeup(reclaim);
2194         }
2195 }
2196
2197 /*
2198  * Setup our reclaim pipeline.  We only let so many detached (and dirty)
2199  * inodes build up before we start blocking.
2200  *
2201  * When we block we don't care *which* inode has finished reclaiming,
2202  * as lone as one does.  This is somewhat heuristical... we also put a
2203  * cap on how long we are willing to wait.
2204  */
2205 void
2206 hammer_inode_waitreclaims(hammer_mount_t hmp)
2207 {
2208         struct hammer_reclaim reclaim;
2209         int delay;
2210
2211         if (hmp->inode_reclaims > HAMMER_RECLAIM_WAIT) {
2212                 reclaim.okydoky = 0;
2213                 TAILQ_INSERT_TAIL(&hmp->reclaim_list,
2214                                   &reclaim, entry);
2215         } else {
2216                 reclaim.okydoky = 1;
2217         }
2218
2219         if (reclaim.okydoky == 0) {
2220                 delay = (hmp->inode_reclaims - HAMMER_RECLAIM_WAIT) * hz /
2221                         HAMMER_RECLAIM_WAIT;
2222                 if (delay >= 0)
2223                         tsleep(&reclaim, 0, "hmrrcm", delay + 1);
2224                 if (reclaim.okydoky == 0)
2225                         TAILQ_REMOVE(&hmp->reclaim_list, &reclaim, entry);
2226         }
2227 }
2228