?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hammer2 - Fix focus vs modify race
[dragonfly.git] / sys / vfs / hammer2 / hammer2_strategy.c
blob2b70108d6f3a55f21cd0b9673d9c2e89f2afe17a
1 /*
2 * Copyright (c) 2011-2018 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
7 * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
18 * distribution.
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36 /*
37 * This module handles low level logical file I/O (strategy) which backs
38 * the logical buffer cache.
39 *
40 * [De]compression, zero-block, check codes, and buffer cache operations
41 * for file data is handled here.
42 *
43 * Live dedup makes its home here as well.
44 */
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/kernel.h>
49 #include <sys/fcntl.h>
50 #include <sys/buf.h>
51 #include <sys/proc.h>
52 #include <sys/namei.h>
53 #include <sys/mount.h>
54 #include <sys/vnode.h>
55 #include <sys/mountctl.h>
56 #include <sys/dirent.h>
57 #include <sys/uio.h>
58 #include <sys/objcache.h>
59 #include <sys/event.h>
60 #include <sys/file.h>
61 #include <vfs/fifofs/fifo.h>
62
63 #include "hammer2.h"
64 #include "hammer2_lz4.h"
65
66 #include "zlib/hammer2_zlib.h"
67
68 struct objcache *cache_buffer_read;
69 struct objcache *cache_buffer_write;
70
71 /*
72 * Strategy code (async logical file buffer I/O from system)
73 *
74 * Except for the transaction init (which should normally not block),
75 * we essentially run the strategy operation asynchronously via a XOP.
76 *
77 * WARNING! The XOP deals with buffer synchronization. It is not synchronized
78 * to the current cpu.
79 *
80 * XXX This isn't supposed to be able to deadlock against vfs_sync vfsync()
81 * calls but it has in the past when multiple flushes are queued.
82 *
83 * XXX We currently terminate the transaction once we get a quorum, otherwise
84 * the frontend can stall, but this can leave the remaining nodes with
85 * a potential flush conflict. We need to delay flushes on those nodes
86 * until running transactions complete separately from the normal
87 * transaction sequencing. FIXME TODO.
88 */
89 static void hammer2_strategy_xop_read(hammer2_thread_t *thr,
90 hammer2_xop_t *arg);
91 static void hammer2_strategy_xop_write(hammer2_thread_t *thr,
92 hammer2_xop_t *arg);
93 static int hammer2_strategy_read(struct vop_strategy_args *ap);
94 static int hammer2_strategy_write(struct vop_strategy_args *ap);
95 static void hammer2_strategy_read_completion(hammer2_chain_t *focus,
96 const char *data, struct bio *bio);
97
98 static hammer2_off_t hammer2_dedup_lookup(hammer2_dev_t *hmp,
99 char **datap, int pblksize);
100
101 int
102 hammer2_vop_strategy(struct vop_strategy_args *ap)
103 {
104 struct bio *biop;
105 struct buf *bp;
106 int error;
107
108 biop = ap->a_bio;
109 bp = biop->bio_buf;
110
111 switch(bp->b_cmd) {
112 case BUF_CMD_READ:
113 error = hammer2_strategy_read(ap);
114 ++hammer2_iod_file_read;
115 break;
116 case BUF_CMD_WRITE:
117 error = hammer2_strategy_write(ap);
118 ++hammer2_iod_file_write;
119 break;
120 default:
121 bp->b_error = error = EINVAL;
122 bp->b_flags |= B_ERROR;
123 biodone(biop);
124 break;
125 }
126 return (error);
127 }
128
129 /*
130 * Return the largest contiguous physical disk range for the logical
131 * request, in bytes.
132 *
133 * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
134 *
135 * Basically disabled, the logical buffer write thread has to deal with
136 * buffers one-at-a-time. Note that this should not prevent cluster_read()
137 * from reading-ahead, it simply prevents it from trying form a single
138 * cluster buffer for the logical request. H2 already uses 64KB buffers!
139 */
140 int
141 hammer2_vop_bmap(struct vop_bmap_args *ap)
142 {
143 *ap->a_doffsetp = NOOFFSET;
144 if (ap->a_runp)
145 *ap->a_runp = 0;
146 if (ap->a_runb)
147 *ap->a_runb = 0;
148 return (EOPNOTSUPP);
149 }
150
151 /****************************************************************************
152 * READ SUPPORT *
153 ****************************************************************************/
154 /*
155 * Callback used in read path in case that a block is compressed with LZ4.
156 */
157 static
158 void
159 hammer2_decompress_LZ4_callback(const char *data, u_int bytes, struct bio *bio)
160 {
161 struct buf *bp;
162 char *compressed_buffer;
163 int compressed_size;
164 int result;
165
166 bp = bio->bio_buf;
167
168 #if 0
169 if bio->bio_caller_info2.index &&
170 bio->bio_caller_info1.uvalue32 !=
171 crc32(bp->b_data, bp->b_bufsize) --- return error
172 #endif
173
174 KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
175 compressed_size = *(const int *)data;
176 KKASSERT((uint32_t)compressed_size <= bytes - sizeof(int));
177
178 compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
179 result = LZ4_decompress_safe(__DECONST(char *, &data[sizeof(int)]),
180 compressed_buffer,
181 compressed_size,
182 bp->b_bufsize);
183 if (result < 0) {
184 kprintf("READ PATH: Error during decompression."
185 "bio %016jx/%d\n",
186 (intmax_t)bio->bio_offset, bytes);
187 /* make sure it isn't random garbage */
188 bzero(compressed_buffer, bp->b_bufsize);
189 }
190 KKASSERT(result <= bp->b_bufsize);
191 bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
192 if (result < bp->b_bufsize)
193 bzero(bp->b_data + result, bp->b_bufsize - result);
194 objcache_put(cache_buffer_read, compressed_buffer);
195 bp->b_resid = 0;
196 bp->b_flags |= B_AGE;
197 }
198
199 /*
200 * Callback used in read path in case that a block is compressed with ZLIB.
201 * It is almost identical to LZ4 callback, so in theory they can be unified,
202 * but we didn't want to make changes in bio structure for that.
203 */
204 static
205 void
206 hammer2_decompress_ZLIB_callback(const char *data, u_int bytes, struct bio *bio)
207 {
208 struct buf *bp;
209 char *compressed_buffer;
210 z_stream strm_decompress;
211 int result;
212 int ret;
213
214 bp = bio->bio_buf;
215
216 KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
217 strm_decompress.avail_in = 0;
218 strm_decompress.next_in = Z_NULL;
219
220 ret = inflateInit(&strm_decompress);
221
222 if (ret != Z_OK)
223 kprintf("HAMMER2 ZLIB: Fatal error in inflateInit.\n");
224
225 compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
226 strm_decompress.next_in = __DECONST(char *, data);
227
228 /* XXX supply proper size, subset of device bp */
229 strm_decompress.avail_in = bytes;
230 strm_decompress.next_out = compressed_buffer;
231 strm_decompress.avail_out = bp->b_bufsize;
232
233 ret = inflate(&strm_decompress, Z_FINISH);
234 if (ret != Z_STREAM_END) {
235 kprintf("HAMMER2 ZLIB: Fatar error during decompression.\n");
236 bzero(compressed_buffer, bp->b_bufsize);
237 }
238 bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
239 result = bp->b_bufsize - strm_decompress.avail_out;
240 if (result < bp->b_bufsize)
241 bzero(bp->b_data + result, strm_decompress.avail_out);
242 objcache_put(cache_buffer_read, compressed_buffer);
243 ret = inflateEnd(&strm_decompress);
244
245 bp->b_resid = 0;
246 bp->b_flags |= B_AGE;
247 }
248
249 /*
250 * Logical buffer I/O, async read.
251 */
252 static
253 int
254 hammer2_strategy_read(struct vop_strategy_args *ap)
255 {
256 hammer2_xop_strategy_t *xop;
257 struct buf *bp;
258 struct bio *bio;
259 struct bio *nbio;
260 hammer2_inode_t *ip;
261 hammer2_key_t lbase;
262
263 bio = ap->a_bio;
264 bp = bio->bio_buf;
265 ip = VTOI(ap->a_vp);
266 nbio = push_bio(bio);
267
268 lbase = bio->bio_offset;
269 KKASSERT(((int)lbase & HAMMER2_PBUFMASK) == 0);
270
271 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_STRATEGY);
272 xop->finished = 0;
273 xop->bio = bio;
274 xop->lbase = lbase;
275 hammer2_mtx_init(&xop->lock, "h2bior");
276 hammer2_xop_start(&xop->head, hammer2_strategy_xop_read);
277 /* asynchronous completion */
278
279 return(0);
280 }
281
282 /*
283 * Per-node XOP (threaded), do a synchronous lookup of the chain and
284 * its data. The frontend is asynchronous, so we are also responsible
285 * for racing to terminate the frontend.
286 */
287 static
288 void
289 hammer2_strategy_xop_read(hammer2_thread_t *thr, hammer2_xop_t *arg)
290 {
291 hammer2_xop_strategy_t *xop = &arg->xop_strategy;
292 hammer2_chain_t *parent;
293 hammer2_chain_t *chain;
294 hammer2_chain_t *focus;
295 hammer2_key_t key_dummy;
296 hammer2_key_t lbase;
297 struct bio *bio;
298 struct buf *bp;
299 const char *data;
300 int error;
301
302 /*
303 * Note that we can race completion of the bio supplied by
304 * the front-end so we cannot access it until we determine
305 * that we are the ones finishing it up.
306 */
307 lbase = xop->lbase;
308
309 /*
310 * This is difficult to optimize. The logical buffer might be
311 * partially dirty (contain dummy zero-fill pages), which would
312 * mess up our crc calculation if we were to try a direct read.
313 * So for now we always double-buffer through the underlying
314 * storage.
315 *
316 * If not for the above problem we could conditionalize on
317 * (1) 64KB buffer, (2) one chain (not multi-master) and
318 * (3) !hammer2_double_buffer, and issue a direct read into the
319 * logical buffer.
320 */
321 parent = hammer2_inode_chain(xop->head.ip1, thr->clindex,
322 HAMMER2_RESOLVE_ALWAYS |
323 HAMMER2_RESOLVE_SHARED);
324 if (parent) {
325 chain = hammer2_chain_lookup(&parent, &key_dummy,
326 lbase, lbase,
327 &error,
328 HAMMER2_LOOKUP_ALWAYS |
329 HAMMER2_LOOKUP_SHARED);
330 if (chain)
331 error = chain->error;
332 } else {
333 error = HAMMER2_ERROR_EIO;
334 chain = NULL;
335 }
336 error = hammer2_xop_feed(&xop->head, chain, thr->clindex, error);
337 if (chain) {
338 hammer2_chain_unlock(chain);
339 hammer2_chain_drop(chain);
340 }
341 if (parent) {
342 hammer2_chain_unlock(parent);
343 hammer2_chain_drop(parent);
344 }
345 chain = NULL; /* safety */
346 parent = NULL; /* safety */
347
348 /*
349 * Race to finish the frontend. First-to-complete. bio is only
350 * valid if we are determined to be the ones able to complete
351 * the operation.
352 */
353 if (xop->finished)
354 return;
355 hammer2_mtx_ex(&xop->lock);
356 if (xop->finished) {
357 hammer2_mtx_unlock(&xop->lock);
358 return;
359 }
360 bio = xop->bio;
361 bp = bio->bio_buf;
362 bkvasync(bp);
363
364 /*
365 * Async operation has not completed and we now own the lock.
366 * Determine if we can complete the operation by issuing the
367 * frontend collection non-blocking.
368 *
369 * H2 double-buffers the data, setting B_NOTMETA on the logical
370 * buffer hints to the OS that the logical buffer should not be
371 * swapcached (since the device buffer can be).
372 *
373 * Also note that even for compressed data we would rather the
374 * kernel cache/swapcache device buffers more and (decompressed)
375 * logical buffers less, since that will significantly improve
376 * the amount of end-user data that can be cached.
377 *
378 * NOTE: The chain->data for xop->head.cluster.focus will be
379 * synchronized to the current cpu by xop_collect(),
380 * but other chains in the cluster might not be.
381 */
382 error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
383
384 switch(error) {
385 case 0:
386 xop->finished = 1;
387 hammer2_mtx_unlock(&xop->lock);
388 bp->b_flags |= B_NOTMETA;
389 focus = xop->head.cluster.focus;
390 data = hammer2_xop_gdata(&xop->head)->buf;
391 hammer2_strategy_read_completion(focus, data, xop->bio);
392 hammer2_xop_pdata(&xop->head);
393 biodone(bio);
394 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
395 break;
396 case HAMMER2_ERROR_ENOENT:
397 xop->finished = 1;
398 hammer2_mtx_unlock(&xop->lock);
399 bp->b_flags |= B_NOTMETA;
400 bp->b_resid = 0;
401 bp->b_error = 0;
402 bzero(bp->b_data, bp->b_bcount);
403 biodone(bio);
404 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
405 break;
406 case HAMMER2_ERROR_EINPROGRESS:
407 hammer2_mtx_unlock(&xop->lock);
408 break;
409 default:
410 kprintf("strategy_xop_read: error %08x loff=%016jx\n",
411 error, bp->b_loffset);
412 xop->finished = 1;
413 hammer2_mtx_unlock(&xop->lock);
414 bp->b_flags |= B_ERROR;
415 bp->b_error = EIO;
416 biodone(bio);
417 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
418 break;
419 }
420 }
421
422 static
423 void
424 hammer2_strategy_read_completion(hammer2_chain_t *focus, const char *data,
425 struct bio *bio)
426 {
427 struct buf *bp = bio->bio_buf;
428
429 if (focus->bref.type == HAMMER2_BREF_TYPE_INODE) {
430 /*
431 * Copy from in-memory inode structure.
432 */
433 bcopy(((const hammer2_inode_data_t *)data)->u.data,
434 bp->b_data, HAMMER2_EMBEDDED_BYTES);
435 bzero(bp->b_data + HAMMER2_EMBEDDED_BYTES,
436 bp->b_bcount - HAMMER2_EMBEDDED_BYTES);
437 bp->b_resid = 0;
438 bp->b_error = 0;
439 } else if (focus->bref.type == HAMMER2_BREF_TYPE_DATA) {
440 /*
441 * Data is on-media, record for live dedup. Release the
442 * chain (try to free it) when done. The data is still
443 * cached by both the buffer cache in front and the
444 * block device behind us. This leaves more room in the
445 * LRU chain cache for meta-data chains which we really
446 * want to retain.
447 *
448 * NOTE: Deduplication cannot be safely recorded for
449 * records without a check code.
450 */
451 hammer2_dedup_record(focus, NULL, data);
452 atomic_set_int(&focus->flags, HAMMER2_CHAIN_RELEASE);
453
454 /*
455 * Decompression and copy.
456 */
457 switch (HAMMER2_DEC_COMP(focus->bref.methods)) {
458 case HAMMER2_COMP_LZ4:
459 hammer2_decompress_LZ4_callback(data, focus->bytes,
460 bio);
461 /* b_resid set by call */
462 break;
463 case HAMMER2_COMP_ZLIB:
464 hammer2_decompress_ZLIB_callback(data, focus->bytes,
465 bio);
466 /* b_resid set by call */
467 break;
468 case HAMMER2_COMP_NONE:
469 KKASSERT(focus->bytes <= bp->b_bcount);
470 bcopy(data, bp->b_data, focus->bytes);
471 if (focus->bytes < bp->b_bcount) {
472 bzero(bp->b_data + focus->bytes,
473 bp->b_bcount - focus->bytes);
474 }
475 bp->b_resid = 0;
476 bp->b_error = 0;
477 break;
478 default:
479 panic("hammer2_strategy_read: "
480 "unknown compression type");
481 }
482 } else {
483 panic("hammer2_strategy_read: unknown bref type");
484 }
485 }
486
487 /****************************************************************************
488 * WRITE SUPPORT *
489 ****************************************************************************/
490
491 /*
492 * Functions for compression in threads,
493 * from hammer2_vnops.c
494 */
495 static void hammer2_write_file_core(char *data, hammer2_inode_t *ip,
496 hammer2_chain_t **parentp,
497 hammer2_key_t lbase, int ioflag, int pblksize,
498 hammer2_tid_t mtid, int *errorp);
499 static void hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
500 hammer2_chain_t **parentp,
501 hammer2_key_t lbase, int ioflag, int pblksize,
502 hammer2_tid_t mtid, int *errorp,
503 int comp_algo, int check_algo);
504 static void hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
505 hammer2_chain_t **parentp,
506 hammer2_key_t lbase, int ioflag, int pblksize,
507 hammer2_tid_t mtid, int *errorp,
508 int check_algo);
509 static int test_block_zeros(const char *buf, size_t bytes);
510 static void zero_write(char *data, hammer2_inode_t *ip,
511 hammer2_chain_t **parentp,
512 hammer2_key_t lbase,
513 hammer2_tid_t mtid, int *errorp);
514 static void hammer2_write_bp(hammer2_chain_t *chain, char *data,
515 int ioflag, int pblksize,
516 hammer2_tid_t mtid, int *errorp,
517 int check_algo);
518
519 static
520 int
521 hammer2_strategy_write(struct vop_strategy_args *ap)
522 {
523 hammer2_xop_strategy_t *xop;
524 hammer2_pfs_t *pmp;
525 struct bio *bio;
526 struct buf *bp;
527 hammer2_inode_t *ip;
528
529 bio = ap->a_bio;
530 bp = bio->bio_buf;
531 ip = VTOI(ap->a_vp);
532 pmp = ip->pmp;
533
534 atomic_set_int(&ip->flags, HAMMER2_INODE_DIRTYDATA);
535 hammer2_lwinprog_ref(pmp);
536 hammer2_trans_assert_strategy(pmp);
537 hammer2_trans_init(pmp, HAMMER2_TRANS_BUFCACHE);
538
539 xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
540 HAMMER2_XOP_STRATEGY);
541 xop->finished = 0;
542 xop->bio = bio;
543 xop->lbase = bio->bio_offset;
544 hammer2_mtx_init(&xop->lock, "h2biow");
545 hammer2_xop_start(&xop->head, hammer2_strategy_xop_write);
546 /* asynchronous completion */
547
548 hammer2_lwinprog_wait(pmp, hammer2_flush_pipe);
549
550 return(0);
551 }
552
553 /*
554 * Per-node XOP (threaded). Write the logical buffer to the media.
555 *
556 * This is a bit problematic because there may be multiple target and
557 * any of them may be able to release the bp. In addition, if our
558 * particulr target is offline we don't want to block the bp (and thus
559 * the frontend). To accomplish this we copy the data to the per-thr
560 * scratch buffer.
561 */
562 static
563 void
564 hammer2_strategy_xop_write(hammer2_thread_t *thr, hammer2_xop_t *arg)
565 {
566 hammer2_xop_strategy_t *xop = &arg->xop_strategy;
567 hammer2_chain_t *parent;
568 hammer2_key_t lbase;
569 hammer2_inode_t *ip;
570 struct bio *bio;
571 struct buf *bp;
572 int error;
573 int lblksize;
574 int pblksize;
575 hammer2_off_t bio_offset;
576 char *bio_data;
577
578 /*
579 * We can only access the bp/bio if the frontend has not yet
580 * completed.
581 */
582 if (xop->finished)
583 return;
584 hammer2_mtx_sh(&xop->lock);
585 if (xop->finished) {
586 hammer2_mtx_unlock(&xop->lock);
587 return;
588 }
589
590 lbase = xop->lbase;
591 bio = xop->bio; /* ephermal */
592 bp = bio->bio_buf; /* ephermal */
593 ip = xop->head.ip1; /* retained by ref */
594 bio_offset = bio->bio_offset;
595 bio_data = thr->scratch;
596
597 /* hammer2_trans_init(parent->hmp->spmp, HAMMER2_TRANS_BUFCACHE); */
598
599 lblksize = hammer2_calc_logical(ip, bio->bio_offset, &lbase, NULL);
600 pblksize = hammer2_calc_physical(ip, lbase);
601 bkvasync(bp);
602 KKASSERT(lblksize <= MAXPHYS);
603 bcopy(bp->b_data, bio_data, lblksize);
604
605 hammer2_mtx_unlock(&xop->lock);
606 bp = NULL; /* safety, illegal to access after unlock */
607 bio = NULL; /* safety, illegal to access after unlock */
608
609 /*
610 * Actual operation
611 */
612 parent = hammer2_inode_chain(ip, thr->clindex, HAMMER2_RESOLVE_ALWAYS);
613 hammer2_write_file_core(bio_data, ip, &parent,
614 lbase, IO_ASYNC, pblksize,
615 xop->head.mtid, &error);
616 if (parent) {
617 hammer2_chain_unlock(parent);
618 hammer2_chain_drop(parent);
619 parent = NULL; /* safety */
620 }
621 hammer2_xop_feed(&xop->head, NULL, thr->clindex, error);
622
623 /*
624 * Try to complete the operation on behalf of the front-end.
625 */
626 if (xop->finished)
627 return;
628 hammer2_mtx_ex(&xop->lock);
629 if (xop->finished) {
630 hammer2_mtx_unlock(&xop->lock);
631 return;
632 }
633
634 /*
635 * Async operation has not completed and we now own the lock.
636 * Determine if we can complete the operation by issuing the
637 * frontend collection non-blocking.
638 *
639 * H2 double-buffers the data, setting B_NOTMETA on the logical
640 * buffer hints to the OS that the logical buffer should not be
641 * swapcached (since the device buffer can be).
642 */
643 error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
644
645 if (error == HAMMER2_ERROR_EINPROGRESS) {
646 hammer2_mtx_unlock(&xop->lock);
647 return;
648 }
649
650 /*
651 * Async operation has completed.
652 */
653 xop->finished = 1;
654 hammer2_mtx_unlock(&xop->lock);
655
656 bio = xop->bio; /* now owned by us */
657 bp = bio->bio_buf; /* now owned by us */
658
659 if (error == HAMMER2_ERROR_ENOENT || error == 0) {
660 bp->b_flags |= B_NOTMETA;
661 bp->b_resid = 0;
662 bp->b_error = 0;
663 biodone(bio);
664 } else {
665 kprintf("strategy_xop_write: error %d loff=%016jx\n",
666 error, bp->b_loffset);
667 bp->b_flags |= B_ERROR;
668 bp->b_error = EIO;
669 biodone(bio);
670 }
671 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
672 hammer2_trans_assert_strategy(ip->pmp);
673 hammer2_lwinprog_drop(ip->pmp);
674 hammer2_trans_done(ip->pmp, 0);
675 }
676
677 /*
678 * Wait for pending I/O to complete
679 */
680 void
681 hammer2_bioq_sync(hammer2_pfs_t *pmp)
682 {
683 hammer2_lwinprog_wait(pmp, 0);
684 }
685
686 /*
687 * Assign physical storage at (cparent, lbase), returning a suitable chain
688 * and setting *errorp appropriately.
689 *
690 * If no error occurs, the returned chain will be in a modified state.
691 *
692 * If an error occurs, the returned chain may or may not be NULL. If
693 * not-null any chain->error (if not 0) will also be rolled up into *errorp.
694 * So the caller only needs to test *errorp.
695 *
696 * cparent can wind up being anything.
697 *
698 * If datap is not NULL, *datap points to the real data we intend to write.
699 * If we can dedup the storage location we set *datap to NULL to indicate
700 * to the caller that a dedup occurred.
701 *
702 * NOTE: Special case for data embedded in inode.
703 */
704 static
705 hammer2_chain_t *
706 hammer2_assign_physical(hammer2_inode_t *ip, hammer2_chain_t **parentp,
707 hammer2_key_t lbase, int pblksize,
708 hammer2_tid_t mtid, char **datap, int *errorp)
709 {
710 hammer2_chain_t *chain;
711 hammer2_key_t key_dummy;
712 hammer2_off_t dedup_off;
713 int pradix = hammer2_getradix(pblksize);
714
715 /*
716 * Locate the chain associated with lbase, return a locked chain.
717 * However, do not instantiate any data reference (which utilizes a
718 * device buffer) because we will be using direct IO via the
719 * logical buffer cache buffer.
720 */
721 KKASSERT(pblksize >= HAMMER2_ALLOC_MIN);
722
723 chain = hammer2_chain_lookup(parentp, &key_dummy,
724 lbase, lbase,
725 errorp,
726 HAMMER2_LOOKUP_NODATA);
727
728 /*
729 * The lookup code should not return a DELETED chain to us, unless
730 * its a short-file embedded in the inode. Then it is possible for
731 * the lookup to return a deleted inode.
732 */
733 if (chain && (chain->flags & HAMMER2_CHAIN_DELETED) &&
734 chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
735 kprintf("assign physical deleted chain @ "
736 "%016jx (%016jx.%02x) ip %016jx\n",
737 lbase, chain->bref.data_off, chain->bref.type,
738 ip->meta.inum);
739 Debugger("bleh");
740 }
741
742 if (chain == NULL) {
743 /*
744 * We found a hole, create a new chain entry.
745 *
746 * NOTE: DATA chains are created without device backing
747 * store (nor do we want any).
748 */
749 dedup_off = hammer2_dedup_lookup((*parentp)->hmp, datap,
750 pblksize);
751 *errorp |= hammer2_chain_create(parentp, &chain,
752 ip->pmp,
753 HAMMER2_ENC_CHECK(ip->meta.check_algo) |
754 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE),
755 lbase, HAMMER2_PBUFRADIX,
756 HAMMER2_BREF_TYPE_DATA,
757 pblksize, mtid,
758 dedup_off, 0);
759 if (chain == NULL)
760 goto failed;
761 /*ip->delta_dcount += pblksize;*/
762 } else if (chain->error == 0) {
763 switch (chain->bref.type) {
764 case HAMMER2_BREF_TYPE_INODE:
765 /*
766 * The data is embedded in the inode, which requires
767 * a bit more finess.
768 */
769 *errorp |= hammer2_chain_modify_ip(ip, chain, mtid, 0);
770 break;
771 case HAMMER2_BREF_TYPE_DATA:
772 dedup_off = hammer2_dedup_lookup(chain->hmp, datap,
773 pblksize);
774 if (chain->bytes != pblksize) {
775 *errorp |= hammer2_chain_resize(chain,
776 mtid, dedup_off,
777 pradix,
778 HAMMER2_MODIFY_OPTDATA);
779 if (*errorp)
780 break;
781 }
782
783 /*
784 * DATA buffers must be marked modified whether the
785 * data is in a logical buffer or not. We also have
786 * to make this call to fixup the chain data pointers
787 * after resizing in case this is an encrypted or
788 * compressed buffer.
789 */
790 *errorp |= hammer2_chain_modify(chain, mtid, dedup_off,
791 HAMMER2_MODIFY_OPTDATA);
792 break;
793 default:
794 panic("hammer2_assign_physical: bad type");
795 /* NOT REACHED */
796 break;
797 }
798 } else {
799 *errorp = chain->error;
800 }
801 atomic_set_int(&ip->flags, HAMMER2_INODE_DIRTYDATA);
802 failed:
803 return (chain);
804 }
805
806 /*
807 * hammer2_write_file_core() - hammer2_write_thread() helper
808 *
809 * The core write function which determines which path to take
810 * depending on compression settings. We also have to locate the
811 * related chains so we can calculate and set the check data for
812 * the blockref.
813 */
814 static
815 void
816 hammer2_write_file_core(char *data, hammer2_inode_t *ip,
817 hammer2_chain_t **parentp,
818 hammer2_key_t lbase, int ioflag, int pblksize,
819 hammer2_tid_t mtid, int *errorp)
820 {
821 hammer2_chain_t *chain;
822 char *bdata;
823
824 *errorp = 0;
825
826 switch(HAMMER2_DEC_ALGO(ip->meta.comp_algo)) {
827 case HAMMER2_COMP_NONE:
828 /*
829 * We have to assign physical storage to the buffer
830 * we intend to dirty or write now to avoid deadlocks
831 * in the strategy code later.
832 *
833 * This can return NOOFFSET for inode-embedded data.
834 * The strategy code will take care of it in that case.
835 */
836 bdata = data;
837 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
838 mtid, &bdata, errorp);
839 if (*errorp) {
840 /* skip modifications */
841 } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
842 hammer2_inode_data_t *wipdata;
843
844 wipdata = &chain->data->ipdata;
845 KKASSERT(wipdata->meta.op_flags &
846 HAMMER2_OPFLAG_DIRECTDATA);
847 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
848 ++hammer2_iod_file_wembed;
849 } else if (bdata == NULL) {
850 /*
851 * Copy of data already present on-media.
852 */
853 chain->bref.methods =
854 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
855 HAMMER2_ENC_CHECK(ip->meta.check_algo);
856 hammer2_chain_setcheck(chain, data);
857 } else {
858 hammer2_write_bp(chain, data, ioflag, pblksize,
859 mtid, errorp, ip->meta.check_algo);
860 }
861 if (chain) {
862 hammer2_chain_unlock(chain);
863 hammer2_chain_drop(chain);
864 }
865 break;
866 case HAMMER2_COMP_AUTOZERO:
867 /*
868 * Check for zero-fill only
869 */
870 hammer2_zero_check_and_write(data, ip, parentp,
871 lbase, ioflag, pblksize,
872 mtid, errorp,
873 ip->meta.check_algo);
874 break;
875 case HAMMER2_COMP_LZ4:
876 case HAMMER2_COMP_ZLIB:
877 default:
878 /*
879 * Check for zero-fill and attempt compression.
880 */
881 hammer2_compress_and_write(data, ip, parentp,
882 lbase, ioflag, pblksize,
883 mtid, errorp,
884 ip->meta.comp_algo,
885 ip->meta.check_algo);
886 break;
887 }
888 }
889
890 /*
891 * Helper
892 *
893 * Generic function that will perform the compression in compression
894 * write path. The compression algorithm is determined by the settings
895 * obtained from inode.
896 */
897 static
898 void
899 hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
900 hammer2_chain_t **parentp,
901 hammer2_key_t lbase, int ioflag, int pblksize,
902 hammer2_tid_t mtid, int *errorp, int comp_algo, int check_algo)
903 {
904 hammer2_chain_t *chain;
905 int comp_size;
906 int comp_block_size;
907 char *comp_buffer;
908 char *bdata;
909
910 /*
911 * An all-zeros write creates a hole unless the check code
912 * is disabled. When the check code is disabled all writes
913 * are done in-place, including any all-zeros writes.
914 *
915 * NOTE: A snapshot will still force a copy-on-write
916 * (see the HAMMER2_CHECK_NONE in hammer2_chain.c).
917 */
918 if (check_algo != HAMMER2_CHECK_NONE &&
919 test_block_zeros(data, pblksize)) {
920 zero_write(data, ip, parentp, lbase, mtid, errorp);
921 return;
922 }
923
924 /*
925 * Compression requested. Try to compress the block. We store
926 * the data normally if we cannot sufficiently compress it.
927 *
928 * We have a heuristic to detect files which are mostly
929 * uncompressable and avoid the compression attempt in that
930 * case. If the compression heuristic is turned off, we always
931 * try to compress.
932 */
933 comp_size = 0;
934 comp_buffer = NULL;
935
936 KKASSERT(pblksize / 2 <= 32768);
937
938 if (ip->comp_heuristic < 8 || (ip->comp_heuristic & 7) == 0 ||
939 hammer2_always_compress) {
940 z_stream strm_compress;
941 int comp_level;
942 int ret;
943
944 switch(HAMMER2_DEC_ALGO(comp_algo)) {
945 case HAMMER2_COMP_LZ4:
946 /*
947 * We need to prefix with the size, LZ4
948 * doesn't do it for us. Add the related
949 * overhead.
950 *
951 * NOTE: The LZ4 code seems to assume at least an
952 * 8-byte buffer size granularity and may
953 * overrun the buffer if given a 4-byte
954 * granularity.
955 */
956 comp_buffer = objcache_get(cache_buffer_write,
957 M_INTWAIT);
958 comp_size = LZ4_compress_limitedOutput(
959 data,
960 &comp_buffer[sizeof(int)],
961 pblksize,
962 pblksize / 2 - sizeof(int64_t));
963 *(int *)comp_buffer = comp_size;
964 if (comp_size)
965 comp_size += sizeof(int);
966 break;
967 case HAMMER2_COMP_ZLIB:
968 comp_level = HAMMER2_DEC_LEVEL(comp_algo);
969 if (comp_level == 0)
970 comp_level = 6; /* default zlib compression */
971 else if (comp_level < 6)
972 comp_level = 6;
973 else if (comp_level > 9)
974 comp_level = 9;
975 ret = deflateInit(&strm_compress, comp_level);
976 if (ret != Z_OK) {
977 kprintf("HAMMER2 ZLIB: fatal error "
978 "on deflateInit.\n");
979 }
980
981 comp_buffer = objcache_get(cache_buffer_write,
982 M_INTWAIT);
983 strm_compress.next_in = data;
984 strm_compress.avail_in = pblksize;
985 strm_compress.next_out = comp_buffer;
986 strm_compress.avail_out = pblksize / 2;
987 ret = deflate(&strm_compress, Z_FINISH);
988 if (ret == Z_STREAM_END) {
989 comp_size = pblksize / 2 -
990 strm_compress.avail_out;
991 } else {
992 comp_size = 0;
993 }
994 ret = deflateEnd(&strm_compress);
995 break;
996 default:
997 kprintf("Error: Unknown compression method.\n");
998 kprintf("Comp_method = %d.\n", comp_algo);
999 break;
1000 }
1001 }
1002
1003 if (comp_size == 0) {
1004 /*
1005 * compression failed or turned off
1006 */
1007 comp_block_size = pblksize; /* safety */
1008 if (++ip->comp_heuristic > 128)
1009 ip->comp_heuristic = 8;
1010 } else {
1011 /*
1012 * compression succeeded
1013 */
1014 ip->comp_heuristic = 0;
1015 if (comp_size <= 1024) {
1016 comp_block_size = 1024;
1017 } else if (comp_size <= 2048) {
1018 comp_block_size = 2048;
1019 } else if (comp_size <= 4096) {
1020 comp_block_size = 4096;
1021 } else if (comp_size <= 8192) {
1022 comp_block_size = 8192;
1023 } else if (comp_size <= 16384) {
1024 comp_block_size = 16384;
1025 } else if (comp_size <= 32768) {
1026 comp_block_size = 32768;
1027 } else {
1028 panic("hammer2: WRITE PATH: "
1029 "Weird comp_size value.");
1030 /* NOT REACHED */
1031 comp_block_size = pblksize;
1032 }
1033
1034 /*
1035 * Must zero the remainder or dedup (which operates on a
1036 * physical block basis) will not find matches.
1037 */
1038 if (comp_size < comp_block_size) {
1039 bzero(comp_buffer + comp_size,
1040 comp_block_size - comp_size);
1041 }
1042 }
1043
1044 /*
1045 * Assign physical storage, data will be set to NULL if a live-dedup
1046 * was successful.
1047 */
1048 bdata = comp_size ? comp_buffer : data;
1049 chain = hammer2_assign_physical(ip, parentp, lbase, comp_block_size,
1050 mtid, &bdata, errorp);
1051
1052 if (*errorp) {
1053 goto done;
1054 }
1055
1056 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1057 hammer2_inode_data_t *wipdata;
1058
1059 *errorp = hammer2_chain_modify_ip(ip, chain, mtid, 0);
1060 if (*errorp == 0) {
1061 wipdata = &chain->data->ipdata;
1062 KKASSERT(wipdata->meta.op_flags &
1063 HAMMER2_OPFLAG_DIRECTDATA);
1064 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1065 ++hammer2_iod_file_wembed;
1066 }
1067 } else if (bdata == NULL) {
1068 /*
1069 * Live deduplication, a copy of the data is already present
1070 * on the media.
1071 */
1072 if (comp_size) {
1073 chain->bref.methods =
1074 HAMMER2_ENC_COMP(comp_algo) +
1075 HAMMER2_ENC_CHECK(check_algo);
1076 } else {
1077 chain->bref.methods =
1078 HAMMER2_ENC_COMP(
1079 HAMMER2_COMP_NONE) +
1080 HAMMER2_ENC_CHECK(check_algo);
1081 }
1082 bdata = comp_size ? comp_buffer : data;
1083 hammer2_chain_setcheck(chain, bdata);
1084 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1085 } else {
1086 hammer2_io_t *dio;
1087
1088 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1089
1090 switch(chain->bref.type) {
1091 case HAMMER2_BREF_TYPE_INODE:
1092 panic("hammer2_write_bp: unexpected inode\n");
1093 break;
1094 case HAMMER2_BREF_TYPE_DATA:
1095 /*
1096 * Optimize out the read-before-write
1097 * if possible.
1098 */
1099 *errorp = hammer2_io_newnz(chain->hmp,
1100 chain->bref.type,
1101 chain->bref.data_off,
1102 chain->bytes,
1103 &dio);
1104 if (*errorp) {
1105 hammer2_io_brelse(&dio);
1106 kprintf("hammer2: WRITE PATH: "
1107 "dbp bread error\n");
1108 break;
1109 }
1110 bdata = hammer2_io_data(dio, chain->bref.data_off);
1111
1112 /*
1113 * When loading the block make sure we don't
1114 * leave garbage after the compressed data.
1115 */
1116 if (comp_size) {
1117 chain->bref.methods =
1118 HAMMER2_ENC_COMP(comp_algo) +
1119 HAMMER2_ENC_CHECK(check_algo);
1120 bcopy(comp_buffer, bdata, comp_size);
1121 } else {
1122 chain->bref.methods =
1123 HAMMER2_ENC_COMP(
1124 HAMMER2_COMP_NONE) +
1125 HAMMER2_ENC_CHECK(check_algo);
1126 bcopy(data, bdata, pblksize);
1127 }
1128
1129 /*
1130 * The flush code doesn't calculate check codes for
1131 * file data (doing so can result in excessive I/O),
1132 * so we do it here.
1133 */
1134 hammer2_chain_setcheck(chain, bdata);
1135
1136 /*
1137 * Device buffer is now valid, chain is no longer in
1138 * the initial state.
1139 *
1140 * (No blockref table worries with file data)
1141 */
1142 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1143 hammer2_dedup_record(chain, dio, bdata);
1144
1145 /* Now write the related bdp. */
1146 if (ioflag & IO_SYNC) {
1147 /*
1148 * Synchronous I/O requested.
1149 */
1150 hammer2_io_bwrite(&dio);
1151 /*
1152 } else if ((ioflag & IO_DIRECT) &&
1153 loff + n == pblksize) {
1154 hammer2_io_bdwrite(&dio);
1155 */
1156 } else if (ioflag & IO_ASYNC) {
1157 hammer2_io_bawrite(&dio);
1158 } else {
1159 hammer2_io_bdwrite(&dio);
1160 }
1161 break;
1162 default:
1163 panic("hammer2_write_bp: bad chain type %d\n",
1164 chain->bref.type);
1165 /* NOT REACHED */
1166 break;
1167 }
1168 }
1169 done:
1170 if (chain) {
1171 hammer2_chain_unlock(chain);
1172 hammer2_chain_drop(chain);
1173 }
1174 if (comp_buffer)
1175 objcache_put(cache_buffer_write, comp_buffer);
1176 }
1177
1178 /*
1179 * Helper
1180 *
1181 * Function that performs zero-checking and writing without compression,
1182 * it corresponds to default zero-checking path.
1183 */
1184 static
1185 void
1186 hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
1187 hammer2_chain_t **parentp,
1188 hammer2_key_t lbase, int ioflag, int pblksize,
1189 hammer2_tid_t mtid, int *errorp,
1190 int check_algo)
1191 {
1192 hammer2_chain_t *chain;
1193 char *bdata;
1194
1195 if (check_algo != HAMMER2_CHECK_NONE &&
1196 test_block_zeros(data, pblksize)) {
1197 /*
1198 * An all-zeros write creates a hole unless the check code
1199 * is disabled. When the check code is disabled all writes
1200 * are done in-place, including any all-zeros writes.
1201 *
1202 * NOTE: A snapshot will still force a copy-on-write
1203 * (see the HAMMER2_CHECK_NONE in hammer2_chain.c).
1204 */
1205 zero_write(data, ip, parentp, lbase, mtid, errorp);
1206 } else {
1207 /*
1208 * Normal write
1209 */
1210 bdata = data;
1211 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
1212 mtid, &bdata, errorp);
1213 if (*errorp) {
1214 /* do nothing */
1215 } else if (bdata) {
1216 hammer2_write_bp(chain, data, ioflag, pblksize,
1217 mtid, errorp, check_algo);
1218 } else {
1219 /* dedup occurred */
1220 chain->bref.methods =
1221 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1222 HAMMER2_ENC_CHECK(check_algo);
1223 hammer2_chain_setcheck(chain, data);
1224 }
1225 if (chain) {
1226 hammer2_chain_unlock(chain);
1227 hammer2_chain_drop(chain);
1228 }
1229 }
1230 }
1231
1232 /*
1233 * Helper
1234 *
1235 * A function to test whether a block of data contains only zeros,
1236 * returns TRUE (non-zero) if the block is all zeros.
1237 */
1238 static
1239 int
1240 test_block_zeros(const char *buf, size_t bytes)
1241 {
1242 size_t i;
1243
1244 for (i = 0; i < bytes; i += sizeof(long)) {
1245 if (*(const long *)(buf + i) != 0)
1246 return (0);
1247 }
1248 return (1);
1249 }
1250
1251 /*
1252 * Helper
1253 *
1254 * Function to "write" a block that contains only zeros.
1255 */
1256 static
1257 void
1258 zero_write(char *data, hammer2_inode_t *ip,
1259 hammer2_chain_t **parentp,
1260 hammer2_key_t lbase, hammer2_tid_t mtid, int *errorp)
1261 {
1262 hammer2_chain_t *chain;
1263 hammer2_key_t key_dummy;
1264
1265 chain = hammer2_chain_lookup(parentp, &key_dummy,
1266 lbase, lbase,
1267 errorp,
1268 HAMMER2_LOOKUP_NODATA);
1269 if (chain) {
1270 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1271 hammer2_inode_data_t *wipdata;
1272
1273 if (*errorp == 0) {
1274 *errorp = hammer2_chain_modify_ip(ip, chain,
1275 mtid, 0);
1276 }
1277 if (*errorp == 0) {
1278 wipdata = &chain->data->ipdata;
1279 KKASSERT(wipdata->meta.op_flags &
1280 HAMMER2_OPFLAG_DIRECTDATA);
1281 bzero(wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1282 ++hammer2_iod_file_wembed;
1283 }
1284 } else {
1285 /* chain->error ok for deletion */
1286 hammer2_chain_delete(*parentp, chain,
1287 mtid, HAMMER2_DELETE_PERMANENT);
1288 ++hammer2_iod_file_wzero;
1289 }
1290 atomic_set_int(&ip->flags, HAMMER2_INODE_DIRTYDATA);
1291 hammer2_chain_unlock(chain);
1292 hammer2_chain_drop(chain);
1293 } else {
1294 ++hammer2_iod_file_wzero;
1295 }
1296 }
1297
1298 /*
1299 * Helper
1300 *
1301 * Function to write the data as it is, without performing any sort of
1302 * compression. This function is used in path without compression and
1303 * default zero-checking path.
1304 */
1305 static
1306 void
1307 hammer2_write_bp(hammer2_chain_t *chain, char *data, int ioflag,
1308 int pblksize,
1309 hammer2_tid_t mtid, int *errorp, int check_algo)
1310 {
1311 hammer2_inode_data_t *wipdata;
1312 hammer2_io_t *dio;
1313 char *bdata;
1314 int error;
1315
1316 error = 0; /* XXX TODO below */
1317
1318 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1319
1320 switch(chain->bref.type) {
1321 case HAMMER2_BREF_TYPE_INODE:
1322 wipdata = &chain->data->ipdata;
1323 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
1324 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1325 error = 0;
1326 ++hammer2_iod_file_wembed;
1327 break;
1328 case HAMMER2_BREF_TYPE_DATA:
1329 error = hammer2_io_newnz(chain->hmp,
1330 chain->bref.type,
1331 chain->bref.data_off,
1332 chain->bytes, &dio);
1333 if (error) {
1334 hammer2_io_bqrelse(&dio);
1335 kprintf("hammer2: WRITE PATH: "
1336 "dbp bread error\n");
1337 break;
1338 }
1339 bdata = hammer2_io_data(dio, chain->bref.data_off);
1340
1341 chain->bref.methods = HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1342 HAMMER2_ENC_CHECK(check_algo);
1343 bcopy(data, bdata, chain->bytes);
1344
1345 /*
1346 * The flush code doesn't calculate check codes for
1347 * file data (doing so can result in excessive I/O),
1348 * so we do it here.
1349 */
1350 hammer2_chain_setcheck(chain, bdata);
1351
1352 /*
1353 * Device buffer is now valid, chain is no longer in
1354 * the initial state.
1355 *
1356 * (No blockref table worries with file data)
1357 */
1358 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1359 hammer2_dedup_record(chain, dio, bdata);
1360
1361 if (ioflag & IO_SYNC) {
1362 /*
1363 * Synchronous I/O requested.
1364 */
1365 hammer2_io_bwrite(&dio);
1366 /*
1367 } else if ((ioflag & IO_DIRECT) &&
1368 loff + n == pblksize) {
1369 hammer2_io_bdwrite(&dio);
1370 */
1371 } else if (ioflag & IO_ASYNC) {
1372 hammer2_io_bawrite(&dio);
1373 } else {
1374 hammer2_io_bdwrite(&dio);
1375 }
1376 break;
1377 default:
1378 panic("hammer2_write_bp: bad chain type %d\n",
1379 chain->bref.type);
1380 /* NOT REACHED */
1381 error = 0;
1382 break;
1383 }
1384 *errorp = error;
1385 }
1386
1387 /*
1388 * LIVE DEDUP HEURISTICS
1389 *
1390 * Record media and crc information for possible dedup operation. Note
1391 * that the dedup mask bits must also be set in the related DIO for a dedup
1392 * to be fully validated (which is handled in the freemap allocation code).
1393 *
1394 * WARNING! This code is SMP safe but the heuristic allows SMP collisions.
1395 * All fields must be loaded into locals and validated.
1396 *
1397 * WARNING! Should only be used for file data and directory entries,
1398 * hammer2_chain_modify() only checks for the dedup case on data
1399 * chains. Also, dedup data can only be recorded for committed
1400 * chains (so NOT strategy writes which can undergo further
1401 * modification after the fact!).
1402 */
1403 void
1404 hammer2_dedup_record(hammer2_chain_t *chain, hammer2_io_t *dio,
1405 const char *data)
1406 {
1407 hammer2_dev_t *hmp;
1408 hammer2_dedup_t *dedup;
1409 uint64_t crc;
1410 uint64_t mask;
1411 int best = 0;
1412 int i;
1413 int dticks;
1414
1415 /*
1416 * We can only record a dedup if we have media data to test against.
1417 * If dedup is not enabled, return early, which allows a chain to
1418 * remain marked MODIFIED (which might have benefits in special
1419 * situations, though typically it does not).
1420 */
1421 if (hammer2_dedup_enable == 0)
1422 return;
1423 if (dio == NULL) {
1424 dio = chain->dio;
1425 if (dio == NULL)
1426 return;
1427 }
1428
1429 hmp = chain->hmp;
1430
1431 switch(HAMMER2_DEC_CHECK(chain->bref.methods)) {
1432 case HAMMER2_CHECK_ISCSI32:
1433 /*
1434 * XXX use the built-in crc (the dedup lookup sequencing
1435 * needs to be fixed so the check code is already present
1436 * when dedup_lookup is called)
1437 */
1438 #if 0
1439 crc = (uint64_t)(uint32_t)chain->bref.check.iscsi32.value;
1440 #endif
1441 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1442 break;
1443 case HAMMER2_CHECK_XXHASH64:
1444 crc = chain->bref.check.xxhash64.value;
1445 break;
1446 case HAMMER2_CHECK_SHA192:
1447 /*
1448 * XXX use the built-in crc (the dedup lookup sequencing
1449 * needs to be fixed so the check code is already present
1450 * when dedup_lookup is called)
1451 */
1452 #if 0
1453 crc = ((uint64_t *)chain->bref.check.sha192.data)[0] ^
1454 ((uint64_t *)chain->bref.check.sha192.data)[1] ^
1455 ((uint64_t *)chain->bref.check.sha192.data)[2];
1456 #endif
1457 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1458 break;
1459 default:
1460 /*
1461 * Cannot dedup without a check code
1462 *
1463 * NOTE: In particular, CHECK_NONE allows a sector to be
1464 * overwritten without copy-on-write, recording
1465 * a dedup block for a CHECK_NONE object would be
1466 * a disaster!
1467 */
1468 return;
1469 }
1470
1471 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEDUPABLE);
1472
1473 dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1474 for (i = 0; i < 4; ++i) {
1475 if (dedup[i].data_crc == crc) {
1476 best = i;
1477 break;
1478 }
1479 dticks = (int)(dedup[i].ticks - dedup[best].ticks);
1480 if (dticks < 0 || dticks > hz * 60 * 30)
1481 best = i;
1482 }
1483 dedup += best;
1484 if (hammer2_debug & 0x40000) {
1485 kprintf("REC %04x %016jx %016jx\n",
1486 (int)(dedup - hmp->heur_dedup),
1487 crc,
1488 chain->bref.data_off);
1489 }
1490 dedup->ticks = ticks;
1491 dedup->data_off = chain->bref.data_off;
1492 dedup->data_crc = crc;
1493
1494 /*
1495 * Set the valid bits for the dedup only after we know the data
1496 * buffer has been updated. The alloc bits were set (and the valid
1497 * bits cleared) when the media was allocated.
1498 *
1499 * This is done in two stages becuase the bulkfree code can race
1500 * the gap between allocation and data population. Both masks must
1501 * be set before a bcmp/dedup operation is able to use the block.
1502 */
1503 mask = hammer2_dedup_mask(dio, chain->bref.data_off, chain->bytes);
1504 atomic_set_64(&dio->dedup_valid, mask);
1505
1506 #if 0
1507 /*
1508 * XXX removed. MODIFIED is an integral part of the flush code,
1509 * lets not just clear it
1510 */
1511 /*
1512 * Once we record the dedup the chain must be marked clean to
1513 * prevent reuse of the underlying block. Remember that this
1514 * write occurs when the buffer cache is flushed (i.e. on sync(),
1515 * fsync(), filesystem periodic sync, or when the kernel needs to
1516 * flush a buffer), and not whenever the user write()s.
1517 */
1518 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
1519 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
1520 atomic_add_long(&hammer2_count_modified_chains, -1);
1521 if (chain->pmp)
1522 hammer2_pfs_memory_wakeup(chain->pmp);
1523 }
1524 #endif
1525 }
1526
1527 static
1528 hammer2_off_t
1529 hammer2_dedup_lookup(hammer2_dev_t *hmp, char **datap, int pblksize)
1530 {
1531 hammer2_dedup_t *dedup;
1532 hammer2_io_t *dio;
1533 hammer2_off_t off;
1534 uint64_t crc;
1535 uint64_t mask;
1536 char *data;
1537 char *dtmp;
1538 int i;
1539
1540 if (hammer2_dedup_enable == 0)
1541 return 0;
1542 data = *datap;
1543 if (data == NULL)
1544 return 0;
1545
1546 /*
1547 * XXX use the built-in crc (the dedup lookup sequencing
1548 * needs to be fixed so the check code is already present
1549 * when dedup_lookup is called)
1550 */
1551 crc = XXH64(data, pblksize, XXH_HAMMER2_SEED);
1552 dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1553
1554 if (hammer2_debug & 0x40000) {
1555 kprintf("LOC %04x/4 %016jx\n",
1556 (int)(dedup - hmp->heur_dedup),
1557 crc);
1558 }
1559
1560 for (i = 0; i < 4; ++i) {
1561 off = dedup[i].data_off;
1562 cpu_ccfence();
1563 if (dedup[i].data_crc != crc)
1564 continue;
1565 if ((1 << (int)(off & HAMMER2_OFF_MASK_RADIX)) != pblksize)
1566 continue;
1567 dio = hammer2_io_getquick(hmp, off, pblksize);
1568 if (dio) {
1569 dtmp = hammer2_io_data(dio, off),
1570 mask = hammer2_dedup_mask(dio, off, pblksize);
1571 if ((dio->dedup_alloc & mask) == mask &&
1572 (dio->dedup_valid & mask) == mask &&
1573 bcmp(data, dtmp, pblksize) == 0) {
1574 if (hammer2_debug & 0x40000) {
1575 kprintf("DEDUP SUCCESS %016jx\n",
1576 (intmax_t)off);
1577 }
1578 hammer2_io_putblk(&dio);
1579 *datap = NULL;
1580 dedup[i].ticks = ticks; /* update use */
1581 atomic_add_long(&hammer2_iod_file_wdedup,
1582 pblksize);
1583
1584 return off; /* RETURN */
1585 }
1586 hammer2_io_putblk(&dio);
1587 }
1588 }
1589 return 0;
1590 }
1591
1592 /*
1593 * Poof. Races are ok, if someone gets in and reuses a dedup offset
1594 * before or while we are clearing it they will also recover the freemap
1595 * entry (set it to fully allocated), so a bulkfree race can only set it
1596 * to a possibly-free state.
1597 *
1598 * XXX ok, well, not really sure races are ok but going to run with it
1599 * for the moment.
1600 */
1601 void
1602 hammer2_dedup_clear(hammer2_dev_t *hmp)
1603 {
1604 int i;
1605
1606 for (i = 0; i < HAMMER2_DEDUP_HEUR_SIZE; ++i) {
1607 hmp->heur_dedup[i].data_off = 0;
1608 hmp->heur_dedup[i].ticks = ticks - 1;
1609 }
1610 }
DragonFly BSD