?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hammer2 - More involved refactoring of chain_repparent, cleanup
[dragonfly.git] / sys / vfs / hammer2 / hammer2_strategy.c
blobdafd2a0b30194d9aa235a04080f3ec07281607aa
1 /*
2 * Copyright (c) 2011-2018 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
7 * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
18 * distribution.
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36 /*
37 * This module handles low level logical file I/O (strategy) which backs
38 * the logical buffer cache.
39 *
40 * [De]compression, zero-block, check codes, and buffer cache operations
41 * for file data is handled here.
42 *
43 * Live dedup makes its home here as well.
44 */
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/kernel.h>
49 #include <sys/fcntl.h>
50 #include <sys/buf.h>
51 #include <sys/proc.h>
52 #include <sys/namei.h>
53 #include <sys/mount.h>
54 #include <sys/vnode.h>
55 #include <sys/mountctl.h>
56 #include <sys/dirent.h>
57 #include <sys/uio.h>
58 #include <sys/objcache.h>
59 #include <sys/event.h>
60 #include <sys/file.h>
61 #include <vfs/fifofs/fifo.h>
62
63 #include "hammer2.h"
64 #include "hammer2_lz4.h"
65
66 #include "zlib/hammer2_zlib.h"
67
68 struct objcache *cache_buffer_read;
69 struct objcache *cache_buffer_write;
70
71 /*
72 * Strategy code (async logical file buffer I/O from system)
73 *
74 * Except for the transaction init (which should normally not block),
75 * we essentially run the strategy operation asynchronously via a XOP.
76 *
77 * WARNING! The XOP deals with buffer synchronization. It is not synchronized
78 * to the current cpu.
79 *
80 * XXX This isn't supposed to be able to deadlock against vfs_sync vfsync()
81 * calls but it has in the past when multiple flushes are queued.
82 *
83 * XXX We currently terminate the transaction once we get a quorum, otherwise
84 * the frontend can stall, but this can leave the remaining nodes with
85 * a potential flush conflict. We need to delay flushes on those nodes
86 * until running transactions complete separately from the normal
87 * transaction sequencing. FIXME TODO.
88 */
89 static void hammer2_strategy_xop_read(hammer2_thread_t *thr,
90 hammer2_xop_t *arg);
91 static void hammer2_strategy_xop_write(hammer2_thread_t *thr,
92 hammer2_xop_t *arg);
93 static int hammer2_strategy_read(struct vop_strategy_args *ap);
94 static int hammer2_strategy_write(struct vop_strategy_args *ap);
95 static void hammer2_strategy_read_completion(hammer2_chain_t *chain,
96 char *data, struct bio *bio);
97
98 static hammer2_off_t hammer2_dedup_lookup(hammer2_dev_t *hmp,
99 char **datap, int pblksize);
100
101 int
102 hammer2_vop_strategy(struct vop_strategy_args *ap)
103 {
104 struct bio *biop;
105 struct buf *bp;
106 int error;
107
108 biop = ap->a_bio;
109 bp = biop->bio_buf;
110
111 switch(bp->b_cmd) {
112 case BUF_CMD_READ:
113 error = hammer2_strategy_read(ap);
114 ++hammer2_iod_file_read;
115 break;
116 case BUF_CMD_WRITE:
117 error = hammer2_strategy_write(ap);
118 ++hammer2_iod_file_write;
119 break;
120 default:
121 bp->b_error = error = EINVAL;
122 bp->b_flags |= B_ERROR;
123 biodone(biop);
124 break;
125 }
126 return (error);
127 }
128
129 /*
130 * Return the largest contiguous physical disk range for the logical
131 * request, in bytes.
132 *
133 * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
134 *
135 * Basically disabled, the logical buffer write thread has to deal with
136 * buffers one-at-a-time. Note that this should not prevent cluster_read()
137 * from reading-ahead, it simply prevents it from trying form a single
138 * cluster buffer for the logical request. H2 already uses 64KB buffers!
139 */
140 int
141 hammer2_vop_bmap(struct vop_bmap_args *ap)
142 {
143 *ap->a_doffsetp = NOOFFSET;
144 if (ap->a_runp)
145 *ap->a_runp = 0;
146 if (ap->a_runb)
147 *ap->a_runb = 0;
148 return (EOPNOTSUPP);
149 }
150
151 /****************************************************************************
152 * READ SUPPORT *
153 ****************************************************************************/
154 /*
155 * Callback used in read path in case that a block is compressed with LZ4.
156 */
157 static
158 void
159 hammer2_decompress_LZ4_callback(const char *data, u_int bytes, struct bio *bio)
160 {
161 struct buf *bp;
162 char *compressed_buffer;
163 int compressed_size;
164 int result;
165
166 bp = bio->bio_buf;
167
168 #if 0
169 if bio->bio_caller_info2.index &&
170 bio->bio_caller_info1.uvalue32 !=
171 crc32(bp->b_data, bp->b_bufsize) --- return error
172 #endif
173
174 KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
175 compressed_size = *(const int *)data;
176 KKASSERT((uint32_t)compressed_size <= bytes - sizeof(int));
177
178 compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
179 result = LZ4_decompress_safe(__DECONST(char *, &data[sizeof(int)]),
180 compressed_buffer,
181 compressed_size,
182 bp->b_bufsize);
183 if (result < 0) {
184 kprintf("READ PATH: Error during decompression."
185 "bio %016jx/%d\n",
186 (intmax_t)bio->bio_offset, bytes);
187 /* make sure it isn't random garbage */
188 bzero(compressed_buffer, bp->b_bufsize);
189 }
190 KKASSERT(result <= bp->b_bufsize);
191 bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
192 if (result < bp->b_bufsize)
193 bzero(bp->b_data + result, bp->b_bufsize - result);
194 objcache_put(cache_buffer_read, compressed_buffer);
195 bp->b_resid = 0;
196 bp->b_flags |= B_AGE;
197 }
198
199 /*
200 * Callback used in read path in case that a block is compressed with ZLIB.
201 * It is almost identical to LZ4 callback, so in theory they can be unified,
202 * but we didn't want to make changes in bio structure for that.
203 */
204 static
205 void
206 hammer2_decompress_ZLIB_callback(const char *data, u_int bytes, struct bio *bio)
207 {
208 struct buf *bp;
209 char *compressed_buffer;
210 z_stream strm_decompress;
211 int result;
212 int ret;
213
214 bp = bio->bio_buf;
215
216 KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
217 strm_decompress.avail_in = 0;
218 strm_decompress.next_in = Z_NULL;
219
220 ret = inflateInit(&strm_decompress);
221
222 if (ret != Z_OK)
223 kprintf("HAMMER2 ZLIB: Fatal error in inflateInit.\n");
224
225 compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
226 strm_decompress.next_in = __DECONST(char *, data);
227
228 /* XXX supply proper size, subset of device bp */
229 strm_decompress.avail_in = bytes;
230 strm_decompress.next_out = compressed_buffer;
231 strm_decompress.avail_out = bp->b_bufsize;
232
233 ret = inflate(&strm_decompress, Z_FINISH);
234 if (ret != Z_STREAM_END) {
235 kprintf("HAMMER2 ZLIB: Fatar error during decompression.\n");
236 bzero(compressed_buffer, bp->b_bufsize);
237 }
238 bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
239 result = bp->b_bufsize - strm_decompress.avail_out;
240 if (result < bp->b_bufsize)
241 bzero(bp->b_data + result, strm_decompress.avail_out);
242 objcache_put(cache_buffer_read, compressed_buffer);
243 ret = inflateEnd(&strm_decompress);
244
245 bp->b_resid = 0;
246 bp->b_flags |= B_AGE;
247 }
248
249 /*
250 * Logical buffer I/O, async read.
251 */
252 static
253 int
254 hammer2_strategy_read(struct vop_strategy_args *ap)
255 {
256 hammer2_xop_strategy_t *xop;
257 struct buf *bp;
258 struct bio *bio;
259 struct bio *nbio;
260 hammer2_inode_t *ip;
261 hammer2_key_t lbase;
262
263 bio = ap->a_bio;
264 bp = bio->bio_buf;
265 ip = VTOI(ap->a_vp);
266 nbio = push_bio(bio);
267
268 lbase = bio->bio_offset;
269 KKASSERT(((int)lbase & HAMMER2_PBUFMASK) == 0);
270
271 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_STRATEGY);
272 xop->finished = 0;
273 xop->bio = bio;
274 xop->lbase = lbase;
275 hammer2_mtx_init(&xop->lock, "h2bior");
276 hammer2_xop_start(&xop->head, hammer2_strategy_xop_read);
277 /* asynchronous completion */
278
279 return(0);
280 }
281
282 /*
283 * Per-node XOP (threaded), do a synchronous lookup of the chain and
284 * its data. The frontend is asynchronous, so we are also responsible
285 * for racing to terminate the frontend.
286 */
287 static
288 void
289 hammer2_strategy_xop_read(hammer2_thread_t *thr, hammer2_xop_t *arg)
290 {
291 hammer2_xop_strategy_t *xop = &arg->xop_strategy;
292 hammer2_chain_t *parent;
293 hammer2_chain_t *chain;
294 hammer2_key_t key_dummy;
295 hammer2_key_t lbase;
296 struct bio *bio;
297 struct buf *bp;
298 int error;
299
300 /*
301 * Note that we can race completion of the bio supplied by
302 * the front-end so we cannot access it until we determine
303 * that we are the ones finishing it up.
304 */
305 lbase = xop->lbase;
306
307 /*
308 * This is difficult to optimize. The logical buffer might be
309 * partially dirty (contain dummy zero-fill pages), which would
310 * mess up our crc calculation if we were to try a direct read.
311 * So for now we always double-buffer through the underlying
312 * storage.
313 *
314 * If not for the above problem we could conditionalize on
315 * (1) 64KB buffer, (2) one chain (not multi-master) and
316 * (3) !hammer2_double_buffer, and issue a direct read into the
317 * logical buffer.
318 */
319 parent = hammer2_inode_chain(xop->head.ip1, thr->clindex,
320 HAMMER2_RESOLVE_ALWAYS |
321 HAMMER2_RESOLVE_SHARED);
322 if (parent) {
323 chain = hammer2_chain_lookup(&parent, &key_dummy,
324 lbase, lbase,
325 &error,
326 HAMMER2_LOOKUP_ALWAYS |
327 HAMMER2_LOOKUP_SHARED);
328 if (chain)
329 error = chain->error;
330 } else {
331 error = HAMMER2_ERROR_EIO;
332 chain = NULL;
333 }
334 error = hammer2_xop_feed(&xop->head, chain, thr->clindex, error);
335 if (chain) {
336 hammer2_chain_unlock(chain);
337 hammer2_chain_drop(chain);
338 }
339 if (parent) {
340 hammer2_chain_unlock(parent);
341 hammer2_chain_drop(parent);
342 }
343 chain = NULL; /* safety */
344 parent = NULL; /* safety */
345
346 /*
347 * Race to finish the frontend. First-to-complete. bio is only
348 * valid if we are determined to be the ones able to complete
349 * the operation.
350 */
351 if (xop->finished)
352 return;
353 hammer2_mtx_ex(&xop->lock);
354 if (xop->finished) {
355 hammer2_mtx_unlock(&xop->lock);
356 return;
357 }
358 bio = xop->bio;
359 bp = bio->bio_buf;
360 bkvasync(bp);
361
362 /*
363 * Async operation has not completed and we now own the lock.
364 * Determine if we can complete the operation by issuing the
365 * frontend collection non-blocking.
366 *
367 * H2 double-buffers the data, setting B_NOTMETA on the logical
368 * buffer hints to the OS that the logical buffer should not be
369 * swapcached (since the device buffer can be).
370 *
371 * Also note that even for compressed data we would rather the
372 * kernel cache/swapcache device buffers more and (decompressed)
373 * logical buffers less, since that will significantly improve
374 * the amount of end-user data that can be cached.
375 *
376 * NOTE: The chain->data for xop->head.cluster.focus will be
377 * synchronized to the current cpu by xop_collect(),
378 * but other chains in the cluster might not be.
379 */
380 error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
381
382 switch(error) {
383 case 0:
384 xop->finished = 1;
385 hammer2_mtx_unlock(&xop->lock);
386 bp->b_flags |= B_NOTMETA;
387 chain = xop->head.cluster.focus;
388 hammer2_strategy_read_completion(chain, (char *)chain->data,
389 xop->bio);
390 biodone(bio);
391 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
392 break;
393 case HAMMER2_ERROR_ENOENT:
394 xop->finished = 1;
395 hammer2_mtx_unlock(&xop->lock);
396 bp->b_flags |= B_NOTMETA;
397 bp->b_resid = 0;
398 bp->b_error = 0;
399 bzero(bp->b_data, bp->b_bcount);
400 biodone(bio);
401 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
402 break;
403 case HAMMER2_ERROR_EINPROGRESS:
404 hammer2_mtx_unlock(&xop->lock);
405 break;
406 default:
407 kprintf("strategy_xop_read: error %08x loff=%016jx\n",
408 error, bp->b_loffset);
409 xop->finished = 1;
410 hammer2_mtx_unlock(&xop->lock);
411 bp->b_flags |= B_ERROR;
412 bp->b_error = EIO;
413 biodone(bio);
414 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
415 break;
416 }
417 }
418
419 static
420 void
421 hammer2_strategy_read_completion(hammer2_chain_t *chain, char *data,
422 struct bio *bio)
423 {
424 struct buf *bp = bio->bio_buf;
425
426 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
427 /*
428 * Copy from in-memory inode structure.
429 */
430 bcopy(((hammer2_inode_data_t *)data)->u.data,
431 bp->b_data, HAMMER2_EMBEDDED_BYTES);
432 bzero(bp->b_data + HAMMER2_EMBEDDED_BYTES,
433 bp->b_bcount - HAMMER2_EMBEDDED_BYTES);
434 bp->b_resid = 0;
435 bp->b_error = 0;
436 } else if (chain->bref.type == HAMMER2_BREF_TYPE_DATA) {
437 /*
438 * Data is on-media, record for live dedup. Release the
439 * chain (try to free it) when done. The data is still
440 * cached by both the buffer cache in front and the
441 * block device behind us. This leaves more room in the
442 * LRU chain cache for meta-data chains which we really
443 * want to retain.
444 *
445 * NOTE: Deduplication cannot be safely recorded for
446 * records without a check code.
447 */
448 hammer2_dedup_record(chain, NULL, data);
449 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
450
451 /*
452 * Decompression and copy.
453 */
454 switch (HAMMER2_DEC_COMP(chain->bref.methods)) {
455 case HAMMER2_COMP_LZ4:
456 hammer2_decompress_LZ4_callback(data, chain->bytes,
457 bio);
458 /* b_resid set by call */
459 break;
460 case HAMMER2_COMP_ZLIB:
461 hammer2_decompress_ZLIB_callback(data, chain->bytes,
462 bio);
463 /* b_resid set by call */
464 break;
465 case HAMMER2_COMP_NONE:
466 KKASSERT(chain->bytes <= bp->b_bcount);
467 bcopy(data, bp->b_data, chain->bytes);
468 if (chain->bytes < bp->b_bcount) {
469 bzero(bp->b_data + chain->bytes,
470 bp->b_bcount - chain->bytes);
471 }
472 bp->b_resid = 0;
473 bp->b_error = 0;
474 break;
475 default:
476 panic("hammer2_strategy_read: "
477 "unknown compression type");
478 }
479 } else {
480 panic("hammer2_strategy_read: unknown bref type");
481 }
482 }
483
484 /****************************************************************************
485 * WRITE SUPPORT *
486 ****************************************************************************/
487
488 /*
489 * Functions for compression in threads,
490 * from hammer2_vnops.c
491 */
492 static void hammer2_write_file_core(char *data, hammer2_inode_t *ip,
493 hammer2_chain_t **parentp,
494 hammer2_key_t lbase, int ioflag, int pblksize,
495 hammer2_tid_t mtid, int *errorp);
496 static void hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
497 hammer2_chain_t **parentp,
498 hammer2_key_t lbase, int ioflag, int pblksize,
499 hammer2_tid_t mtid, int *errorp,
500 int comp_algo, int check_algo);
501 static void hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
502 hammer2_chain_t **parentp,
503 hammer2_key_t lbase, int ioflag, int pblksize,
504 hammer2_tid_t mtid, int *errorp,
505 int check_algo);
506 static int test_block_zeros(const char *buf, size_t bytes);
507 static void zero_write(char *data, hammer2_inode_t *ip,
508 hammer2_chain_t **parentp,
509 hammer2_key_t lbase,
510 hammer2_tid_t mtid, int *errorp);
511 static void hammer2_write_bp(hammer2_chain_t *chain, char *data,
512 int ioflag, int pblksize,
513 hammer2_tid_t mtid, int *errorp,
514 int check_algo);
515
516 static
517 int
518 hammer2_strategy_write(struct vop_strategy_args *ap)
519 {
520 hammer2_xop_strategy_t *xop;
521 hammer2_pfs_t *pmp;
522 struct bio *bio;
523 struct buf *bp;
524 hammer2_inode_t *ip;
525
526 bio = ap->a_bio;
527 bp = bio->bio_buf;
528 ip = VTOI(ap->a_vp);
529 pmp = ip->pmp;
530
531 atomic_set_int(&ip->flags, HAMMER2_INODE_DIRTYDATA);
532 hammer2_lwinprog_ref(pmp);
533 hammer2_trans_assert_strategy(pmp);
534 hammer2_trans_init(pmp, HAMMER2_TRANS_BUFCACHE);
535
536 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
537 HAMMER2_XOP_STRATEGY);
538 xop->finished = 0;
539 xop->bio = bio;
540 xop->lbase = bio->bio_offset;
541 hammer2_mtx_init(&xop->lock, "h2biow");
542 hammer2_xop_start(&xop->head, hammer2_strategy_xop_write);
543 /* asynchronous completion */
544
545 hammer2_lwinprog_wait(pmp, hammer2_flush_pipe);
546
547 return(0);
548 }
549
550 /*
551 * Per-node XOP (threaded). Write the logical buffer to the media.
552 *
553 * This is a bit problematic because there may be multiple target and
554 * any of them may be able to release the bp. In addition, if our
555 * particulr target is offline we don't want to block the bp (and thus
556 * the frontend). To accomplish this we copy the data to the per-thr
557 * scratch buffer.
558 */
559 static
560 void
561 hammer2_strategy_xop_write(hammer2_thread_t *thr, hammer2_xop_t *arg)
562 {
563 hammer2_xop_strategy_t *xop = &arg->xop_strategy;
564 hammer2_chain_t *parent;
565 hammer2_key_t lbase;
566 hammer2_inode_t *ip;
567 struct bio *bio;
568 struct buf *bp;
569 int error;
570 int lblksize;
571 int pblksize;
572 hammer2_off_t bio_offset;
573 char *bio_data;
574
575 /*
576 * We can only access the bp/bio if the frontend has not yet
577 * completed.
578 */
579 if (xop->finished)
580 return;
581 hammer2_mtx_sh(&xop->lock);
582 if (xop->finished) {
583 hammer2_mtx_unlock(&xop->lock);
584 return;
585 }
586
587 lbase = xop->lbase;
588 bio = xop->bio; /* ephermal */
589 bp = bio->bio_buf; /* ephermal */
590 ip = xop->head.ip1; /* retained by ref */
591 bio_offset = bio->bio_offset;
592 bio_data = thr->scratch;
593
594 /* hammer2_trans_init(parent->hmp->spmp, HAMMER2_TRANS_BUFCACHE); */
595
596 lblksize = hammer2_calc_logical(ip, bio->bio_offset, &lbase, NULL);
597 pblksize = hammer2_calc_physical(ip, lbase);
598 bkvasync(bp);
599 KKASSERT(lblksize <= MAXPHYS);
600 bcopy(bp->b_data, bio_data, lblksize);
601
602 hammer2_mtx_unlock(&xop->lock);
603 bp = NULL; /* safety, illegal to access after unlock */
604 bio = NULL; /* safety, illegal to access after unlock */
605
606 /*
607 * Actual operation
608 */
609 parent = hammer2_inode_chain(ip, thr->clindex, HAMMER2_RESOLVE_ALWAYS);
610 hammer2_write_file_core(bio_data, ip, &parent,
611 lbase, IO_ASYNC, pblksize,
612 xop->head.mtid, &error);
613 if (parent) {
614 hammer2_chain_unlock(parent);
615 hammer2_chain_drop(parent);
616 parent = NULL; /* safety */
617 }
618 hammer2_xop_feed(&xop->head, NULL, thr->clindex, error);
619
620 /*
621 * Try to complete the operation on behalf of the front-end.
622 */
623 if (xop->finished)
624 return;
625 hammer2_mtx_ex(&xop->lock);
626 if (xop->finished) {
627 hammer2_mtx_unlock(&xop->lock);
628 return;
629 }
630
631 /*
632 * Async operation has not completed and we now own the lock.
633 * Determine if we can complete the operation by issuing the
634 * frontend collection non-blocking.
635 *
636 * H2 double-buffers the data, setting B_NOTMETA on the logical
637 * buffer hints to the OS that the logical buffer should not be
638 * swapcached (since the device buffer can be).
639 */
640 error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
641
642 if (error == HAMMER2_ERROR_EINPROGRESS) {
643 hammer2_mtx_unlock(&xop->lock);
644 return;
645 }
646
647 /*
648 * Async operation has completed.
649 */
650 xop->finished = 1;
651 hammer2_mtx_unlock(&xop->lock);
652
653 bio = xop->bio; /* now owned by us */
654 bp = bio->bio_buf; /* now owned by us */
655
656 if (error == HAMMER2_ERROR_ENOENT || error == 0) {
657 bp->b_flags |= B_NOTMETA;
658 bp->b_resid = 0;
659 bp->b_error = 0;
660 biodone(bio);
661 } else {
662 kprintf("strategy_xop_write: error %d loff=%016jx\n",
663 error, bp->b_loffset);
664 bp->b_flags |= B_ERROR;
665 bp->b_error = EIO;
666 biodone(bio);
667 }
668 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
669 hammer2_trans_assert_strategy(ip->pmp);
670 hammer2_lwinprog_drop(ip->pmp);
671 hammer2_trans_done(ip->pmp);
672 }
673
674 /*
675 * Wait for pending I/O to complete
676 */
677 void
678 hammer2_bioq_sync(hammer2_pfs_t *pmp)
679 {
680 hammer2_lwinprog_wait(pmp, 0);
681 }
682
683 /*
684 * Assign physical storage at (cparent, lbase), returning a suitable chain
685 * and setting *errorp appropriately.
686 *
687 * If no error occurs, the returned chain will be in a modified state.
688 *
689 * If an error occurs, the returned chain may or may not be NULL. If
690 * not-null any chain->error (if not 0) will also be rolled up into *errorp.
691 * So the caller only needs to test *errorp.
692 *
693 * cparent can wind up being anything.
694 *
695 * If datap is not NULL, *datap points to the real data we intend to write.
696 * If we can dedup the storage location we set *datap to NULL to indicate
697 * to the caller that a dedup occurred.
698 *
699 * NOTE: Special case for data embedded in inode.
700 */
701 static
702 hammer2_chain_t *
703 hammer2_assign_physical(hammer2_inode_t *ip, hammer2_chain_t **parentp,
704 hammer2_key_t lbase, int pblksize,
705 hammer2_tid_t mtid, char **datap, int *errorp)
706 {
707 hammer2_chain_t *chain;
708 hammer2_key_t key_dummy;
709 hammer2_off_t dedup_off;
710 int pradix = hammer2_getradix(pblksize);
711
712 /*
713 * Locate the chain associated with lbase, return a locked chain.
714 * However, do not instantiate any data reference (which utilizes a
715 * device buffer) because we will be using direct IO via the
716 * logical buffer cache buffer.
717 */
718 KKASSERT(pblksize >= HAMMER2_ALLOC_MIN);
719
720 chain = hammer2_chain_lookup(parentp, &key_dummy,
721 lbase, lbase,
722 errorp,
723 HAMMER2_LOOKUP_NODATA);
724
725 /*
726 * The lookup code should not return a DELETED chain to us, unless
727 * its a short-file embedded in the inode. Then it is possible for
728 * the lookup to return a deleted inode.
729 */
730 if (chain && (chain->flags & HAMMER2_CHAIN_DELETED) &&
731 chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
732 kprintf("assign physical deleted chain @ "
733 "%016jx (%016jx.%02x) ip %016jx\n",
734 lbase, chain->bref.data_off, chain->bref.type,
735 ip->meta.inum);
736 Debugger("bleh");
737 }
738
739 if (chain == NULL) {
740 /*
741 * We found a hole, create a new chain entry.
742 *
743 * NOTE: DATA chains are created without device backing
744 * store (nor do we want any).
745 */
746 dedup_off = hammer2_dedup_lookup((*parentp)->hmp, datap,
747 pblksize);
748 *errorp |= hammer2_chain_create(parentp, &chain,
749 ip->pmp,
750 HAMMER2_ENC_CHECK(ip->meta.check_algo) |
751 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE),
752 lbase, HAMMER2_PBUFRADIX,
753 HAMMER2_BREF_TYPE_DATA,
754 pblksize, mtid,
755 dedup_off, 0);
756 if (chain == NULL)
757 goto failed;
758 /*ip->delta_dcount += pblksize;*/
759 } else if (chain->error == 0) {
760 switch (chain->bref.type) {
761 case HAMMER2_BREF_TYPE_INODE:
762 /*
763 * The data is embedded in the inode, which requires
764 * a bit more finess.
765 */
766 *errorp |= hammer2_chain_modify_ip(ip, chain, mtid, 0);
767 break;
768 case HAMMER2_BREF_TYPE_DATA:
769 dedup_off = hammer2_dedup_lookup(chain->hmp, datap,
770 pblksize);
771 if (chain->bytes != pblksize) {
772 *errorp |= hammer2_chain_resize(chain,
773 mtid, dedup_off,
774 pradix,
775 HAMMER2_MODIFY_OPTDATA);
776 if (*errorp)
777 break;
778 }
779
780 /*
781 * DATA buffers must be marked modified whether the
782 * data is in a logical buffer or not. We also have
783 * to make this call to fixup the chain data pointers
784 * after resizing in case this is an encrypted or
785 * compressed buffer.
786 */
787 *errorp |= hammer2_chain_modify(chain, mtid, dedup_off,
788 HAMMER2_MODIFY_OPTDATA);
789 break;
790 default:
791 panic("hammer2_assign_physical: bad type");
792 /* NOT REACHED */
793 break;
794 }
795 } else {
796 *errorp = chain->error;
797 }
798 atomic_set_int(&ip->flags, HAMMER2_INODE_DIRTYDATA);
799 failed:
800 return (chain);
801 }
802
803 /*
804 * hammer2_write_file_core() - hammer2_write_thread() helper
805 *
806 * The core write function which determines which path to take
807 * depending on compression settings. We also have to locate the
808 * related chains so we can calculate and set the check data for
809 * the blockref.
810 */
811 static
812 void
813 hammer2_write_file_core(char *data, hammer2_inode_t *ip,
814 hammer2_chain_t **parentp,
815 hammer2_key_t lbase, int ioflag, int pblksize,
816 hammer2_tid_t mtid, int *errorp)
817 {
818 hammer2_chain_t *chain;
819 char *bdata;
820
821 *errorp = 0;
822
823 switch(HAMMER2_DEC_ALGO(ip->meta.comp_algo)) {
824 case HAMMER2_COMP_NONE:
825 /*
826 * We have to assign physical storage to the buffer
827 * we intend to dirty or write now to avoid deadlocks
828 * in the strategy code later.
829 *
830 * This can return NOOFFSET for inode-embedded data.
831 * The strategy code will take care of it in that case.
832 */
833 bdata = data;
834 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
835 mtid, &bdata, errorp);
836 if (*errorp) {
837 /* skip modifications */
838 } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
839 hammer2_inode_data_t *wipdata;
840
841 wipdata = &chain->data->ipdata;
842 KKASSERT(wipdata->meta.op_flags &
843 HAMMER2_OPFLAG_DIRECTDATA);
844 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
845 ++hammer2_iod_file_wembed;
846 } else if (bdata == NULL) {
847 /*
848 * Copy of data already present on-media.
849 */
850 chain->bref.methods =
851 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
852 HAMMER2_ENC_CHECK(ip->meta.check_algo);
853 hammer2_chain_setcheck(chain, data);
854 } else {
855 hammer2_write_bp(chain, data, ioflag, pblksize,
856 mtid, errorp, ip->meta.check_algo);
857 }
858 if (chain) {
859 hammer2_chain_unlock(chain);
860 hammer2_chain_drop(chain);
861 }
862 break;
863 case HAMMER2_COMP_AUTOZERO:
864 /*
865 * Check for zero-fill only
866 */
867 hammer2_zero_check_and_write(data, ip, parentp,
868 lbase, ioflag, pblksize,
869 mtid, errorp,
870 ip->meta.check_algo);
871 break;
872 case HAMMER2_COMP_LZ4:
873 case HAMMER2_COMP_ZLIB:
874 default:
875 /*
876 * Check for zero-fill and attempt compression.
877 */
878 hammer2_compress_and_write(data, ip, parentp,
879 lbase, ioflag, pblksize,
880 mtid, errorp,
881 ip->meta.comp_algo,
882 ip->meta.check_algo);
883 break;
884 }
885 }
886
887 /*
888 * Helper
889 *
890 * Generic function that will perform the compression in compression
891 * write path. The compression algorithm is determined by the settings
892 * obtained from inode.
893 */
894 static
895 void
896 hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
897 hammer2_chain_t **parentp,
898 hammer2_key_t lbase, int ioflag, int pblksize,
899 hammer2_tid_t mtid, int *errorp, int comp_algo, int check_algo)
900 {
901 hammer2_chain_t *chain;
902 int comp_size;
903 int comp_block_size;
904 char *comp_buffer;
905 char *bdata;
906
907 /*
908 * An all-zeros write creates a hole unless the check code
909 * is disabled. When the check code is disabled all writes
910 * are done in-place, including any all-zeros writes.
911 *
912 * NOTE: A snapshot will still force a copy-on-write
913 * (see the HAMMER2_CHECK_NONE in hammer2_chain.c).
914 */
915 if (check_algo != HAMMER2_CHECK_NONE &&
916 test_block_zeros(data, pblksize)) {
917 zero_write(data, ip, parentp, lbase, mtid, errorp);
918 return;
919 }
920
921 /*
922 * Compression requested. Try to compress the block. We store
923 * the data normally if we cannot sufficiently compress it.
924 *
925 * We have a heuristic to detect files which are mostly
926 * uncompressable and avoid the compression attempt in that
927 * case. If the compression heuristic is turned off, we always
928 * try to compress.
929 */
930 comp_size = 0;
931 comp_buffer = NULL;
932
933 KKASSERT(pblksize / 2 <= 32768);
934
935 if (ip->comp_heuristic < 8 || (ip->comp_heuristic & 7) == 0 ||
936 hammer2_always_compress) {
937 z_stream strm_compress;
938 int comp_level;
939 int ret;
940
941 switch(HAMMER2_DEC_ALGO(comp_algo)) {
942 case HAMMER2_COMP_LZ4:
943 /*
944 * We need to prefix with the size, LZ4
945 * doesn't do it for us. Add the related
946 * overhead.
947 *
948 * NOTE: The LZ4 code seems to assume at least an
949 * 8-byte buffer size granularity and may
950 * overrun the buffer if given a 4-byte
951 * granularity.
952 */
953 comp_buffer = objcache_get(cache_buffer_write,
954 M_INTWAIT);
955 comp_size = LZ4_compress_limitedOutput(
956 data,
957 &comp_buffer[sizeof(int)],
958 pblksize,
959 pblksize / 2 - sizeof(int64_t));
960 *(int *)comp_buffer = comp_size;
961 if (comp_size)
962 comp_size += sizeof(int);
963 break;
964 case HAMMER2_COMP_ZLIB:
965 comp_level = HAMMER2_DEC_LEVEL(comp_algo);
966 if (comp_level == 0)
967 comp_level = 6; /* default zlib compression */
968 else if (comp_level < 6)
969 comp_level = 6;
970 else if (comp_level > 9)
971 comp_level = 9;
972 ret = deflateInit(&strm_compress, comp_level);
973 if (ret != Z_OK) {
974 kprintf("HAMMER2 ZLIB: fatal error "
975 "on deflateInit.\n");
976 }
977
978 comp_buffer = objcache_get(cache_buffer_write,
979 M_INTWAIT);
980 strm_compress.next_in = data;
981 strm_compress.avail_in = pblksize;
982 strm_compress.next_out = comp_buffer;
983 strm_compress.avail_out = pblksize / 2;
984 ret = deflate(&strm_compress, Z_FINISH);
985 if (ret == Z_STREAM_END) {
986 comp_size = pblksize / 2 -
987 strm_compress.avail_out;
988 } else {
989 comp_size = 0;
990 }
991 ret = deflateEnd(&strm_compress);
992 break;
993 default:
994 kprintf("Error: Unknown compression method.\n");
995 kprintf("Comp_method = %d.\n", comp_algo);
996 break;
997 }
998 }
999
1000 if (comp_size == 0) {
1001 /*
1002 * compression failed or turned off
1003 */
1004 comp_block_size = pblksize; /* safety */
1005 if (++ip->comp_heuristic > 128)
1006 ip->comp_heuristic = 8;
1007 } else {
1008 /*
1009 * compression succeeded
1010 */
1011 ip->comp_heuristic = 0;
1012 if (comp_size <= 1024) {
1013 comp_block_size = 1024;
1014 } else if (comp_size <= 2048) {
1015 comp_block_size = 2048;
1016 } else if (comp_size <= 4096) {
1017 comp_block_size = 4096;
1018 } else if (comp_size <= 8192) {
1019 comp_block_size = 8192;
1020 } else if (comp_size <= 16384) {
1021 comp_block_size = 16384;
1022 } else if (comp_size <= 32768) {
1023 comp_block_size = 32768;
1024 } else {
1025 panic("hammer2: WRITE PATH: "
1026 "Weird comp_size value.");
1027 /* NOT REACHED */
1028 comp_block_size = pblksize;
1029 }
1030
1031 /*
1032 * Must zero the remainder or dedup (which operates on a
1033 * physical block basis) will not find matches.
1034 */
1035 if (comp_size < comp_block_size) {
1036 bzero(comp_buffer + comp_size,
1037 comp_block_size - comp_size);
1038 }
1039 }
1040
1041 /*
1042 * Assign physical storage, data will be set to NULL if a live-dedup
1043 * was successful.
1044 */
1045 bdata = comp_size ? comp_buffer : data;
1046 chain = hammer2_assign_physical(ip, parentp, lbase, comp_block_size,
1047 mtid, &bdata, errorp);
1048
1049 if (*errorp) {
1050 goto done;
1051 }
1052
1053 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1054 hammer2_inode_data_t *wipdata;
1055
1056 *errorp = hammer2_chain_modify_ip(ip, chain, mtid, 0);
1057 if (*errorp == 0) {
1058 wipdata = &chain->data->ipdata;
1059 KKASSERT(wipdata->meta.op_flags &
1060 HAMMER2_OPFLAG_DIRECTDATA);
1061 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1062 ++hammer2_iod_file_wembed;
1063 }
1064 } else if (bdata == NULL) {
1065 /*
1066 * Live deduplication, a copy of the data is already present
1067 * on the media.
1068 */
1069 if (comp_size) {
1070 chain->bref.methods =
1071 HAMMER2_ENC_COMP(comp_algo) +
1072 HAMMER2_ENC_CHECK(check_algo);
1073 } else {
1074 chain->bref.methods =
1075 HAMMER2_ENC_COMP(
1076 HAMMER2_COMP_NONE) +
1077 HAMMER2_ENC_CHECK(check_algo);
1078 }
1079 bdata = comp_size ? comp_buffer : data;
1080 hammer2_chain_setcheck(chain, bdata);
1081 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1082 } else {
1083 hammer2_io_t *dio;
1084
1085 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1086
1087 switch(chain->bref.type) {
1088 case HAMMER2_BREF_TYPE_INODE:
1089 panic("hammer2_write_bp: unexpected inode\n");
1090 break;
1091 case HAMMER2_BREF_TYPE_DATA:
1092 /*
1093 * Optimize out the read-before-write
1094 * if possible.
1095 */
1096 *errorp = hammer2_io_newnz(chain->hmp,
1097 chain->bref.type,
1098 chain->bref.data_off,
1099 chain->bytes,
1100 &dio);
1101 if (*errorp) {
1102 hammer2_io_brelse(&dio);
1103 kprintf("hammer2: WRITE PATH: "
1104 "dbp bread error\n");
1105 break;
1106 }
1107 bdata = hammer2_io_data(dio, chain->bref.data_off);
1108
1109 /*
1110 * When loading the block make sure we don't
1111 * leave garbage after the compressed data.
1112 */
1113 if (comp_size) {
1114 chain->bref.methods =
1115 HAMMER2_ENC_COMP(comp_algo) +
1116 HAMMER2_ENC_CHECK(check_algo);
1117 bcopy(comp_buffer, bdata, comp_size);
1118 } else {
1119 chain->bref.methods =
1120 HAMMER2_ENC_COMP(
1121 HAMMER2_COMP_NONE) +
1122 HAMMER2_ENC_CHECK(check_algo);
1123 bcopy(data, bdata, pblksize);
1124 }
1125
1126 /*
1127 * The flush code doesn't calculate check codes for
1128 * file data (doing so can result in excessive I/O),
1129 * so we do it here.
1130 */
1131 hammer2_chain_setcheck(chain, bdata);
1132
1133 /*
1134 * Device buffer is now valid, chain is no longer in
1135 * the initial state.
1136 *
1137 * (No blockref table worries with file data)
1138 */
1139 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1140 hammer2_dedup_record(chain, dio, bdata);
1141
1142 /* Now write the related bdp. */
1143 if (ioflag & IO_SYNC) {
1144 /*
1145 * Synchronous I/O requested.
1146 */
1147 hammer2_io_bwrite(&dio);
1148 /*
1149 } else if ((ioflag & IO_DIRECT) &&
1150 loff + n == pblksize) {
1151 hammer2_io_bdwrite(&dio);
1152 */
1153 } else if (ioflag & IO_ASYNC) {
1154 hammer2_io_bawrite(&dio);
1155 } else {
1156 hammer2_io_bdwrite(&dio);
1157 }
1158 break;
1159 default:
1160 panic("hammer2_write_bp: bad chain type %d\n",
1161 chain->bref.type);
1162 /* NOT REACHED */
1163 break;
1164 }
1165 }
1166 done:
1167 if (chain) {
1168 hammer2_chain_unlock(chain);
1169 hammer2_chain_drop(chain);
1170 }
1171 if (comp_buffer)
1172 objcache_put(cache_buffer_write, comp_buffer);
1173 }
1174
1175 /*
1176 * Helper
1177 *
1178 * Function that performs zero-checking and writing without compression,
1179 * it corresponds to default zero-checking path.
1180 */
1181 static
1182 void
1183 hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
1184 hammer2_chain_t **parentp,
1185 hammer2_key_t lbase, int ioflag, int pblksize,
1186 hammer2_tid_t mtid, int *errorp,
1187 int check_algo)
1188 {
1189 hammer2_chain_t *chain;
1190 char *bdata;
1191
1192 if (check_algo != HAMMER2_CHECK_NONE &&
1193 test_block_zeros(data, pblksize)) {
1194 /*
1195 * An all-zeros write creates a hole unless the check code
1196 * is disabled. When the check code is disabled all writes
1197 * are done in-place, including any all-zeros writes.
1198 *
1199 * NOTE: A snapshot will still force a copy-on-write
1200 * (see the HAMMER2_CHECK_NONE in hammer2_chain.c).
1201 */
1202 zero_write(data, ip, parentp, lbase, mtid, errorp);
1203 } else {
1204 /*
1205 * Normal write
1206 */
1207 bdata = data;
1208 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
1209 mtid, &bdata, errorp);
1210 if (*errorp) {
1211 /* do nothing */
1212 } else if (bdata) {
1213 hammer2_write_bp(chain, data, ioflag, pblksize,
1214 mtid, errorp, check_algo);
1215 } else {
1216 /* dedup occurred */
1217 chain->bref.methods =
1218 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1219 HAMMER2_ENC_CHECK(check_algo);
1220 hammer2_chain_setcheck(chain, data);
1221 }
1222 if (chain) {
1223 hammer2_chain_unlock(chain);
1224 hammer2_chain_drop(chain);
1225 }
1226 }
1227 }
1228
1229 /*
1230 * Helper
1231 *
1232 * A function to test whether a block of data contains only zeros,
1233 * returns TRUE (non-zero) if the block is all zeros.
1234 */
1235 static
1236 int
1237 test_block_zeros(const char *buf, size_t bytes)
1238 {
1239 size_t i;
1240
1241 for (i = 0; i < bytes; i += sizeof(long)) {
1242 if (*(const long *)(buf + i) != 0)
1243 return (0);
1244 }
1245 return (1);
1246 }
1247
1248 /*
1249 * Helper
1250 *
1251 * Function to "write" a block that contains only zeros.
1252 */
1253 static
1254 void
1255 zero_write(char *data, hammer2_inode_t *ip,
1256 hammer2_chain_t **parentp,
1257 hammer2_key_t lbase, hammer2_tid_t mtid, int *errorp)
1258 {
1259 hammer2_chain_t *chain;
1260 hammer2_key_t key_dummy;
1261
1262 chain = hammer2_chain_lookup(parentp, &key_dummy,
1263 lbase, lbase,
1264 errorp,
1265 HAMMER2_LOOKUP_NODATA);
1266 if (chain) {
1267 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1268 hammer2_inode_data_t *wipdata;
1269
1270 if (*errorp == 0) {
1271 *errorp = hammer2_chain_modify_ip(ip, chain,
1272 mtid, 0);
1273 }
1274 if (*errorp == 0) {
1275 wipdata = &chain->data->ipdata;
1276 KKASSERT(wipdata->meta.op_flags &
1277 HAMMER2_OPFLAG_DIRECTDATA);
1278 bzero(wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1279 ++hammer2_iod_file_wembed;
1280 }
1281 } else {
1282 /* chain->error ok for deletion */
1283 hammer2_chain_delete(*parentp, chain,
1284 mtid, HAMMER2_DELETE_PERMANENT);
1285 ++hammer2_iod_file_wzero;
1286 }
1287 atomic_set_int(&ip->flags, HAMMER2_INODE_DIRTYDATA);
1288 hammer2_chain_unlock(chain);
1289 hammer2_chain_drop(chain);
1290 } else {
1291 ++hammer2_iod_file_wzero;
1292 }
1293 }
1294
1295 /*
1296 * Helper
1297 *
1298 * Function to write the data as it is, without performing any sort of
1299 * compression. This function is used in path without compression and
1300 * default zero-checking path.
1301 */
1302 static
1303 void
1304 hammer2_write_bp(hammer2_chain_t *chain, char *data, int ioflag,
1305 int pblksize,
1306 hammer2_tid_t mtid, int *errorp, int check_algo)
1307 {
1308 hammer2_inode_data_t *wipdata;
1309 hammer2_io_t *dio;
1310 char *bdata;
1311 int error;
1312
1313 error = 0; /* XXX TODO below */
1314
1315 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1316
1317 switch(chain->bref.type) {
1318 case HAMMER2_BREF_TYPE_INODE:
1319 wipdata = &chain->data->ipdata;
1320 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
1321 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1322 error = 0;
1323 ++hammer2_iod_file_wembed;
1324 break;
1325 case HAMMER2_BREF_TYPE_DATA:
1326 error = hammer2_io_newnz(chain->hmp,
1327 chain->bref.type,
1328 chain->bref.data_off,
1329 chain->bytes, &dio);
1330 if (error) {
1331 hammer2_io_bqrelse(&dio);
1332 kprintf("hammer2: WRITE PATH: "
1333 "dbp bread error\n");
1334 break;
1335 }
1336 bdata = hammer2_io_data(dio, chain->bref.data_off);
1337
1338 chain->bref.methods = HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1339 HAMMER2_ENC_CHECK(check_algo);
1340 bcopy(data, bdata, chain->bytes);
1341
1342 /*
1343 * The flush code doesn't calculate check codes for
1344 * file data (doing so can result in excessive I/O),
1345 * so we do it here.
1346 */
1347 hammer2_chain_setcheck(chain, bdata);
1348
1349 /*
1350 * Device buffer is now valid, chain is no longer in
1351 * the initial state.
1352 *
1353 * (No blockref table worries with file data)
1354 */
1355 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1356 hammer2_dedup_record(chain, dio, bdata);
1357
1358 if (ioflag & IO_SYNC) {
1359 /*
1360 * Synchronous I/O requested.
1361 */
1362 hammer2_io_bwrite(&dio);
1363 /*
1364 } else if ((ioflag & IO_DIRECT) &&
1365 loff + n == pblksize) {
1366 hammer2_io_bdwrite(&dio);
1367 */
1368 } else if (ioflag & IO_ASYNC) {
1369 hammer2_io_bawrite(&dio);
1370 } else {
1371 hammer2_io_bdwrite(&dio);
1372 }
1373 break;
1374 default:
1375 panic("hammer2_write_bp: bad chain type %d\n",
1376 chain->bref.type);
1377 /* NOT REACHED */
1378 error = 0;
1379 break;
1380 }
1381 *errorp = error;
1382 }
1383
1384 /*
1385 * LIVE DEDUP HEURISTICS
1386 *
1387 * Record media and crc information for possible dedup operation. Note
1388 * that the dedup mask bits must also be set in the related DIO for a dedup
1389 * to be fully validated (which is handled in the freemap allocation code).
1390 *
1391 * WARNING! This code is SMP safe but the heuristic allows SMP collisions.
1392 * All fields must be loaded into locals and validated.
1393 *
1394 * WARNING! Should only be used for file data and directory entries,
1395 * hammer2_chain_modify() only checks for the dedup case on data
1396 * chains. Also, dedup data can only be recorded for committed
1397 * chains (so NOT strategy writes which can undergo further
1398 * modification after the fact!).
1399 */
1400 void
1401 hammer2_dedup_record(hammer2_chain_t *chain, hammer2_io_t *dio, char *data)
1402 {
1403 hammer2_dev_t *hmp;
1404 hammer2_dedup_t *dedup;
1405 uint64_t crc;
1406 uint64_t mask;
1407 int best = 0;
1408 int i;
1409 int dticks;
1410
1411 /*
1412 * We can only record a dedup if we have media data to test against.
1413 * If dedup is not enabled, return early, which allows a chain to
1414 * remain marked MODIFIED (which might have benefits in special
1415 * situations, though typically it does not).
1416 */
1417 if (hammer2_dedup_enable == 0)
1418 return;
1419 if (dio == NULL) {
1420 dio = chain->dio;
1421 if (dio == NULL)
1422 return;
1423 }
1424
1425 hmp = chain->hmp;
1426
1427 switch(HAMMER2_DEC_CHECK(chain->bref.methods)) {
1428 case HAMMER2_CHECK_ISCSI32:
1429 /*
1430 * XXX use the built-in crc (the dedup lookup sequencing
1431 * needs to be fixed so the check code is already present
1432 * when dedup_lookup is called)
1433 */
1434 #if 0
1435 crc = (uint64_t)(uint32_t)chain->bref.check.iscsi32.value;
1436 #endif
1437 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1438 break;
1439 case HAMMER2_CHECK_XXHASH64:
1440 crc = chain->bref.check.xxhash64.value;
1441 break;
1442 case HAMMER2_CHECK_SHA192:
1443 /*
1444 * XXX use the built-in crc (the dedup lookup sequencing
1445 * needs to be fixed so the check code is already present
1446 * when dedup_lookup is called)
1447 */
1448 #if 0
1449 crc = ((uint64_t *)chain->bref.check.sha192.data)[0] ^
1450 ((uint64_t *)chain->bref.check.sha192.data)[1] ^
1451 ((uint64_t *)chain->bref.check.sha192.data)[2];
1452 #endif
1453 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1454 break;
1455 default:
1456 /*
1457 * Cannot dedup without a check code
1458 *
1459 * NOTE: In particular, CHECK_NONE allows a sector to be
1460 * overwritten without copy-on-write, recording
1461 * a dedup block for a CHECK_NONE object would be
1462 * a disaster!
1463 */
1464 return;
1465 }
1466
1467 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEDUPABLE);
1468
1469 dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1470 for (i = 0; i < 4; ++i) {
1471 if (dedup[i].data_crc == crc) {
1472 best = i;
1473 break;
1474 }
1475 dticks = (int)(dedup[i].ticks - dedup[best].ticks);
1476 if (dticks < 0 || dticks > hz * 60 * 30)
1477 best = i;
1478 }
1479 dedup += best;
1480 if (hammer2_debug & 0x40000) {
1481 kprintf("REC %04x %016jx %016jx\n",
1482 (int)(dedup - hmp->heur_dedup),
1483 crc,
1484 chain->bref.data_off);
1485 }
1486 dedup->ticks = ticks;
1487 dedup->data_off = chain->bref.data_off;
1488 dedup->data_crc = crc;
1489
1490 /*
1491 * Set the valid bits for the dedup only after we know the data
1492 * buffer has been updated. The alloc bits were set (and the valid
1493 * bits cleared) when the media was allocated.
1494 *
1495 * This is done in two stages becuase the bulkfree code can race
1496 * the gap between allocation and data population. Both masks must
1497 * be set before a bcmp/dedup operation is able to use the block.
1498 */
1499 mask = hammer2_dedup_mask(dio, chain->bref.data_off, chain->bytes);
1500 atomic_set_64(&dio->dedup_valid, mask);
1501
1502 #if 0
1503 /*
1504 * XXX removed. MODIFIED is an integral part of the flush code,
1505 * lets not just clear it
1506 */
1507 /*
1508 * Once we record the dedup the chain must be marked clean to
1509 * prevent reuse of the underlying block. Remember that this
1510 * write occurs when the buffer cache is flushed (i.e. on sync(),
1511 * fsync(), filesystem periodic sync, or when the kernel needs to
1512 * flush a buffer), and not whenever the user write()s.
1513 */
1514 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
1515 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
1516 atomic_add_long(&hammer2_count_modified_chains, -1);
1517 if (chain->pmp)
1518 hammer2_pfs_memory_wakeup(chain->pmp);
1519 }
1520 #endif
1521 }
1522
1523 static
1524 hammer2_off_t
1525 hammer2_dedup_lookup(hammer2_dev_t *hmp, char **datap, int pblksize)
1526 {
1527 hammer2_dedup_t *dedup;
1528 hammer2_io_t *dio;
1529 hammer2_off_t off;
1530 uint64_t crc;
1531 uint64_t mask;
1532 char *data;
1533 char *dtmp;
1534 int i;
1535
1536 if (hammer2_dedup_enable == 0)
1537 return 0;
1538 data = *datap;
1539 if (data == NULL)
1540 return 0;
1541
1542 /*
1543 * XXX use the built-in crc (the dedup lookup sequencing
1544 * needs to be fixed so the check code is already present
1545 * when dedup_lookup is called)
1546 */
1547 crc = XXH64(data, pblksize, XXH_HAMMER2_SEED);
1548 dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1549
1550 if (hammer2_debug & 0x40000) {
1551 kprintf("LOC %04x/4 %016jx\n",
1552 (int)(dedup - hmp->heur_dedup),
1553 crc);
1554 }
1555
1556 for (i = 0; i < 4; ++i) {
1557 off = dedup[i].data_off;
1558 cpu_ccfence();
1559 if (dedup[i].data_crc != crc)
1560 continue;
1561 if ((1 << (int)(off & HAMMER2_OFF_MASK_RADIX)) != pblksize)
1562 continue;
1563 dio = hammer2_io_getquick(hmp, off, pblksize);
1564 if (dio) {
1565 dtmp = hammer2_io_data(dio, off),
1566 mask = hammer2_dedup_mask(dio, off, pblksize);
1567 if ((dio->dedup_alloc & mask) == mask &&
1568 (dio->dedup_valid & mask) == mask &&
1569 bcmp(data, dtmp, pblksize) == 0) {
1570 if (hammer2_debug & 0x40000) {
1571 kprintf("DEDUP SUCCESS %016jx\n",
1572 (intmax_t)off);
1573 }
1574 hammer2_io_putblk(&dio);
1575 *datap = NULL;
1576 dedup[i].ticks = ticks; /* update use */
1577 atomic_add_long(&hammer2_iod_file_wdedup,
1578 pblksize);
1579
1580 return off; /* RETURN */
1581 }
1582 hammer2_io_putblk(&dio);
1583 }
1584 }
1585 return 0;
1586 }
1587
1588 /*
1589 * Poof. Races are ok, if someone gets in and reuses a dedup offset
1590 * before or while we are clearing it they will also recover the freemap
1591 * entry (set it to fully allocated), so a bulkfree race can only set it
1592 * to a possibly-free state.
1593 *
1594 * XXX ok, well, not really sure races are ok but going to run with it
1595 * for the moment.
1596 */
1597 void
1598 hammer2_dedup_clear(hammer2_dev_t *hmp)
1599 {
1600 int i;
1601
1602 for (i = 0; i < HAMMER2_DEDUP_HEUR_SIZE; ++i) {
1603 hmp->heur_dedup[i].data_off = 0;
1604 hmp->heur_dedup[i].ticks = ticks - 1;
1605 }
1606 }
DragonFly BSD