2 * Copyright (c) 2008 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41 #define SERIALBUF_SIZE (512 * 1024)
43 typedef struct histogram
{
48 const char *ScoreBoardFile
;
49 const char *RestrictTarget
;
51 static int read_mrecords(int fd
, char *buf
, u_int size
,
52 hammer_ioc_mrecord_head_t pickup
);
53 static int generate_histogram(int fd
, const char *filesystem
,
54 histogram_t
*histogram_ary
,
55 struct hammer_ioc_mirror_rw
*mirror_base
,
57 static hammer_ioc_mrecord_any_t
read_mrecord(int fdin
, int *errorp
,
58 hammer_ioc_mrecord_head_t pickup
);
59 static void write_mrecord(int fdout
, uint32_t type
,
60 hammer_ioc_mrecord_any_t mrec
, int bytes
);
61 static void generate_mrec_header(int fd
, int pfs_id
,
62 union hammer_ioc_mrecord_any
*mrec_tmp
);
63 static int validate_mrec_header(int fd
, int fdin
, int is_target
, int pfs_id
,
64 struct hammer_ioc_mrecord_head
*pickup
,
65 hammer_tid_t
*tid_begp
, hammer_tid_t
*tid_endp
);
66 static void update_pfs_snapshot(int fd
, hammer_tid_t snapshot_tid
, int pfs_id
);
67 static ssize_t
writebw(int fd
, const void *buf
, size_t nbytes
,
68 uint64_t *bwcount
, struct timeval
*tv1
);
69 static int getyntty(void);
70 static void score_printf(size_t i
, size_t w
, const char *ctl
, ...);
71 static void hammer_check_restrict(const char *filesystem
);
72 static void mirror_usage(int code
);
75 * Generate a mirroring data stream from the specific source over the
76 * entire key range, but restricted to the specified transaction range.
78 * The HAMMER VFS does most of the work, we add a few new mrecord
79 * types to negotiate the TID ranges and verify that the entire
80 * stream made it to the destination.
82 * streaming will be 0 for mirror-read, 1 for mirror-stream. The code will
83 * set up a fake value of -1 when running the histogram for mirror-read.
86 hammer_cmd_mirror_read(char **av
, int ac
, int streaming
)
88 struct hammer_ioc_mirror_rw mirror
;
89 struct hammer_ioc_pseudofs_rw pfs
;
90 union hammer_ioc_mrecord_any mrec_tmp
;
91 struct hammer_ioc_mrecord_head pickup
;
92 hammer_ioc_mrecord_any_t mrec
;
93 hammer_tid_t sync_tid
;
94 histogram_t histogram_ary
;
95 const char *filesystem
;
96 char *buf
= malloc(SERIALBUF_SIZE
);
108 time_t base_t
= time(NULL
);
113 if (ac
== 0 || ac
> 2)
116 hammer_check_restrict(filesystem
);
118 pickup
.signature
= 0;
123 histogram_ary
= NULL
;
127 bzero(&mirror
, sizeof(mirror
));
128 hammer_key_beg_init(&mirror
.key_beg
);
129 hammer_key_end_init(&mirror
.key_end
);
131 fd
= getpfs(&pfs
, filesystem
);
134 score_printf(LINE1
, "Running");
136 if (streaming
>= 0 && VerboseOpt
&& VerboseOpt
< 2) {
137 fprintf(stderr
, "%cRunning \b\b", (sameline
? '\r' : '\n'));
143 gettimeofday(&bwtv
, NULL
);
147 * Send initial header for the purpose of determining the
150 generate_mrec_header(fd
, pfs
.pfs_id
, &mrec_tmp
);
151 write_mrecord(1, HAMMER_MREC_TYPE_PFSD
,
152 &mrec_tmp
, sizeof(mrec_tmp
.pfs
));
155 * In 2-way mode the target will send us a PFS info packet
156 * first. Use the target's current snapshot TID as our default
161 n
= validate_mrec_header(fd
, 0, 0, pfs
.pfs_id
, &pickup
,
162 NULL
, &mirror
.tid_beg
);
163 if (n
< 0) { /* got TERM record */
170 } else if (streaming
&& histogram
) {
171 mirror
.tid_beg
= histogram_ary
[histindex
].tid
+ 1;
177 * Write out the PFS header, tid_beg will be updated if our PFS
178 * has a larger begin sync. tid_end is set to the latest source
179 * TID whos flush cycle has completed.
181 generate_mrec_header(fd
, pfs
.pfs_id
, &mrec_tmp
);
182 if (mirror
.tid_beg
< mrec_tmp
.pfs
.pfsd
.sync_beg_tid
)
183 mirror
.tid_beg
= mrec_tmp
.pfs
.pfsd
.sync_beg_tid
;
184 mirror
.tid_end
= mrec_tmp
.pfs
.pfsd
.sync_end_tid
;
186 mirror
.size
= SERIALBUF_SIZE
;
187 mirror
.pfs_id
= pfs
.pfs_id
;
188 mirror
.shared_uuid
= pfs
.ondisk
->shared_uuid
;
191 * XXX If the histogram is exhausted and the TID delta is large
192 * the stream might have been offline for a while and is
193 * now picking it up again. Do another histogram.
196 if (streaming
&& histogram
&& histindex
== histend
)
197 if (mirror
.tid_end
- mirror
.tid_beg
> BULK_MINIMUM
)
202 * Initial bulk startup control, try to do some incremental
203 * mirroring in order to allow the stream to be killed and
204 * restarted without having to start over.
206 if (histogram
== 0 && BulkOpt
== 0) {
207 if (VerboseOpt
&& repeat
== 0) {
208 fprintf(stderr
, "\n");
211 histmax
= generate_histogram(fd
, filesystem
,
212 &histogram_ary
, &mirror
,
218 * Just stream the histogram, then stop
224 if (streaming
&& histogram
) {
226 mirror
.tid_end
= histogram_ary
[histindex
].tid
;
227 estbytes
= histogram_ary
[histindex
-1].bytes
;
228 mrec_tmp
.pfs
.pfsd
.sync_end_tid
= mirror
.tid_end
;
233 write_mrecord(1, HAMMER_MREC_TYPE_PFSD
,
234 &mrec_tmp
, sizeof(mrec_tmp
.pfs
));
237 * A cycle file overrides the beginning TID only if we are
238 * not operating in two-way or histogram mode.
240 if (TwoWayPipeOpt
== 0 && histogram
== 0)
241 hammer_get_cycle(&mirror
.key_beg
, &mirror
.tid_beg
);
244 * An additional argument overrides the beginning TID regardless
245 * of what mode we are in. This is not recommending if operating
249 mirror
.tid_beg
= strtoull(av
[1], NULL
, 0);
251 if (streaming
== 0 || VerboseOpt
>= 2) {
253 "Mirror-read: Mirror %016jx to %016jx",
254 (uintmax_t)mirror
.tid_beg
, (uintmax_t)mirror
.tid_end
);
256 fprintf(stderr
, " (bulk= %ju)", (uintmax_t)estbytes
);
257 fprintf(stderr
, "\n");
260 if (mirror
.key_beg
.obj_id
!= (int64_t)HAMMER_MIN_OBJID
)
261 fprintf(stderr
, "Mirror-read: Resuming at object %016jx\n",
262 (uintmax_t)mirror
.key_beg
.obj_id
);
265 * Nothing to do if begin equals end.
267 if (mirror
.tid_beg
>= mirror
.tid_end
) {
268 if (streaming
== 0 || VerboseOpt
>= 2)
269 fprintf(stderr
, "Mirror-read: No work to do\n");
278 * Write out bulk records
281 mirror
.size
= SERIALBUF_SIZE
;
285 mirror
.pfs_id
= pfs
.pfs_id
;
286 mirror
.shared_uuid
= pfs
.ondisk
->shared_uuid
;
287 if (ioctl(fd
, HAMMERIOC_MIRROR_READ
, &mirror
) < 0) {
288 score_printf(LINE3
, "Mirror-read %s failed: %s",
289 filesystem
, strerror(errno
));
290 err(1, "Mirror-read %s failed", filesystem
);
292 if (mirror
.head
.flags
& HAMMER_IOC_HEAD_ERROR
) {
293 score_printf(LINE3
, "Mirror-read %s fatal error %d",
294 filesystem
, mirror
.head
.error
);
295 errx(1, "Mirror-read %s fatal error %d",
296 filesystem
, mirror
.head
.error
);
300 n
= writebw(1, mirror
.ubuf
, mirror
.count
,
303 n
= write(1, mirror
.ubuf
, mirror
.count
);
304 if (n
!= mirror
.count
) {
306 "Mirror-read %s failed: "
309 errx(1, "Mirror-read %s failed: short write",
313 total_bytes
+= mirror
.count
;
314 if (streaming
&& VerboseOpt
) {
316 "\rscan obj=%016jx tids=%016jx:%016jx %11jd",
317 (uintmax_t)mirror
.key_cur
.obj_id
,
318 (uintmax_t)mirror
.tid_beg
,
319 (uintmax_t)mirror
.tid_end
,
320 (intmax_t)total_bytes
);
323 } else if (streaming
) {
325 "obj=%016jx tids=%016jx:%016jx %11jd",
326 (uintmax_t)mirror
.key_cur
.obj_id
,
327 (uintmax_t)mirror
.tid_beg
,
328 (uintmax_t)mirror
.tid_end
,
329 (intmax_t)total_bytes
);
331 mirror
.key_beg
= mirror
.key_cur
;
334 * Deal with time limit option
337 (unsigned)(time(NULL
) - base_t
) > (unsigned)TimeoutOpt
) {
339 "Mirror-read %s interrupted by timer at"
342 (uintmax_t)mirror
.key_cur
.obj_id
);
344 "Mirror-read %s interrupted by timer at"
347 (uintmax_t)mirror
.key_cur
.obj_id
);
351 } while (mirror
.count
!= 0);
354 if (streaming
&& VerboseOpt
&& sameline
== 0) {
355 fprintf(stderr
, "\n");
361 * Write out the termination sync record - only if not interrupted
363 if (interrupted
== 0) {
365 write_mrecord(1, HAMMER_MREC_TYPE_SYNC
,
366 &mrec_tmp
, sizeof(mrec_tmp
.sync
));
368 write_mrecord(1, HAMMER_MREC_TYPE_IDLE
,
369 &mrec_tmp
, sizeof(mrec_tmp
.sync
));
373 * If the -2 option was given (automatic when doing mirror-copy),
374 * a two-way pipe is assumed and we expect a response mrec from
378 mrec
= read_mrecord(0, &error
, &pickup
);
380 mrec
->head
.type
!= HAMMER_MREC_TYPE_UPDATE
||
381 mrec
->head
.rec_size
!= sizeof(mrec
->update
))
382 errx(1, "mirror_read: Did not get final "
383 "acknowledgement packet from target");
386 hammer_set_cycle(&mirror
.key_cur
,
388 fprintf(stderr
, "Cyclefile %s updated for "
389 "continuation\n", CyclePath
);
392 sync_tid
= mrec
->update
.tid
;
394 hammer_key_beg_init(&mirror
.key_beg
);
395 hammer_set_cycle(&mirror
.key_beg
, sync_tid
);
397 "Cyclefile %s updated to 0x%016jx\n",
398 CyclePath
, (uintmax_t)sync_tid
);
402 } else if (CyclePath
) {
403 /* NOTE! mirror.tid_beg cannot be updated */
404 fprintf(stderr
, "Warning: cycle file (-c option) cannot be "
405 "fully updated unless you use mirror-copy\n");
406 hammer_set_cycle(&mirror
.key_beg
, mirror
.tid_beg
);
408 if (streaming
&& interrupted
== 0) {
409 time_t t1
= time(NULL
);
413 * Try to break down large bulk transfers into smaller ones
414 * so it can sync the transaction id on the slave. This
415 * way if we get interrupted a restart doesn't have to
416 * start from scratch.
418 if (streaming
&& histogram
)
419 if (histindex
!= histmax
) {
420 if (VerboseOpt
&& VerboseOpt
< 2 &&
422 fprintf(stderr
, " (bulk incremental)");
427 if (VerboseOpt
&& streaming
>= 0) {
428 fprintf(stderr
, " W");
430 } else if (streaming
>= 0) {
431 score_printf(LINE1
, "Waiting");
433 pfs
.ondisk
->sync_end_tid
= mirror
.tid_end
;
436 * Fake streaming mode when using a histogram to
437 * break up a mirror-read, do not wait on source.
440 } else if (ioctl(fd
, HAMMERIOC_WAI_PSEUDOFS
, &pfs
) < 0) {
442 "Mirror-read %s: cannot stream: %s\n",
443 filesystem
, strerror(errno
));
445 "Mirror-read %s: cannot stream: %s\n",
446 filesystem
, strerror(errno
));
448 t2
= time(NULL
) - t1
;
449 if (t2
>= 0 && t2
< DelayOpt
) {
451 fprintf(stderr
, "\bD");
454 sleep(DelayOpt
- t2
);
457 fprintf(stderr
, "\b ");
464 write_mrecord(1, HAMMER_MREC_TYPE_TERM
,
465 &mrec_tmp
, sizeof(mrec_tmp
.sync
));
469 fprintf(stderr
, "Mirror-read %s succeeded\n", filesystem
);
473 * What we are trying to do here is figure out how much data is
474 * going to be sent for the TID range and to break the TID range
475 * down into reasonably-sized slices (from the point of view of
476 * data sent) so a lost connection can restart at a reasonable
477 * place and not all the way back at the beginning.
479 * An entry's TID serves as the end_tid for the prior entry
480 * So we have to offset the calculation by 1 so that TID falls into
481 * the previous entry when populating entries.
483 * Because the transaction id space is bursty we need a relatively
484 * large number of buckets (like a million) to do a reasonable job
485 * for things like an initial bulk mirrors on a very large filesystem.
487 #define HIST_COUNT (1024 * 1024)
490 generate_histogram(int fd
, const char *filesystem
,
491 histogram_t
*histogram_ary
,
492 struct hammer_ioc_mirror_rw
*mirror_base
,
495 struct hammer_ioc_mirror_rw mirror
;
496 union hammer_ioc_mrecord_any
*mrec
;
497 hammer_tid_t tid_beg
;
498 hammer_tid_t tid_end
;
510 mirror
= *mirror_base
;
511 tid_beg
= mirror
.tid_beg
;
512 tid_end
= mirror
.tid_end
;
513 mirror
.head
.flags
|= HAMMER_IOC_MIRROR_NODATA
;
515 if (*histogram_ary
== NULL
)
516 *histogram_ary
= malloc(sizeof(struct histogram
) *
518 if (tid_beg
>= tid_end
)
522 tid_bytes
= malloc(sizeof(*tid_bytes
) * (HIST_COUNT
+ 2));
523 bzero(tid_bytes
, sizeof(*tid_bytes
) * (HIST_COUNT
+ 2));
526 fprintf(stderr
, "Prescan to break up bulk transfer");
528 fprintf(stderr
, " (%juMB chunks)",
529 (uintmax_t)(SplitupOpt
/ (1024 * 1024)));
530 fprintf(stderr
, "\n");
534 * Note: (tid_beg,tid_end), range is inclusive of both beg & end.
536 * Note: Estimates can be off when the mirror is way behind due
544 if (ioctl(fd
, HAMMERIOC_MIRROR_READ
, &mirror
) < 0)
545 err(1, "Mirror-read %s failed", filesystem
);
546 if (mirror
.head
.flags
& HAMMER_IOC_HEAD_ERROR
)
547 errx(1, "Mirror-read %s fatal error %d",
548 filesystem
, mirror
.head
.error
);
551 off
+= HAMMER_HEAD_DOALIGN(mrec
->head
.rec_size
)) {
552 mrec
= (void *)((char *)mirror
.ubuf
+ off
);
555 * We only care about general RECs and PASS
556 * records. We ignore SKIPs.
558 switch (mrec
->head
.type
& HAMMER_MRECF_TYPE_LOMASK
) {
559 case HAMMER_MREC_TYPE_REC
:
560 case HAMMER_MREC_TYPE_PASS
:
567 * Calculate for two indices, create_tid and
568 * delete_tid. Record data only applies to
571 * When tid is exactly on the boundary it really
572 * belongs to the previous entry because scans
573 * are inclusive of the ending entry.
575 tid
= mrec
->rec
.leaf
.base
.delete_tid
;
576 if (tid
&& tid
>= tid_beg
&& tid
<= tid_end
) {
577 len
= HAMMER_HEAD_DOALIGN(mrec
->head
.rec_size
);
578 if (mrec
->head
.type
==
579 HAMMER_MREC_TYPE_REC
) {
580 len
-= HAMMER_HEAD_DOALIGN(
581 mrec
->rec
.leaf
.data_len
);
584 i
= (tid
- tid_beg
) * HIST_COUNT
/
586 tidx
= tid_beg
+ i
* (tid_end
- tid_beg
) /
588 if (tid
== tidx
&& i
)
590 assert(i
>= 0 && i
< HIST_COUNT
);
596 tid
= mrec
->rec
.leaf
.base
.create_tid
;
597 if (tid
&& tid
>= tid_beg
&& tid
<= tid_end
) {
598 len
= HAMMER_HEAD_DOALIGN(mrec
->head
.rec_size
);
599 if (mrec
->head
.type
==
600 HAMMER_MREC_TYPE_REC_NODATA
)
601 len
+= HAMMER_HEAD_DOALIGN(
602 mrec
->rec
.leaf
.data_len
);
603 i
= (tid
- tid_beg
) * HIST_COUNT
/
605 tidx
= tid_beg
+ i
* (tid_end
- tid_beg
) /
607 if (tid
== tidx
&& i
)
609 assert(i
>= 0 && i
< HIST_COUNT
);
615 if (*repeatp
== 0 && accum
> SplitupOpt
) {
616 if (VerboseOpt
> 1) {
617 fprintf(stderr
, ".");
621 score_printf(LINE2
, "Prescan chunk %d", chunkno
);
624 if (mirror
.count
== 0)
626 mirror
.key_beg
= mirror
.key_cur
;
630 * Reduce to SplitupOpt (default 4GB) chunks. This code may
631 * use up to two additional elements. Do the array in-place.
633 * Inefficient degenerate cases can occur if we do not accumulate
634 * at least the requested split amount, so error on the side of
638 (*histogram_ary
)[res
].tid
= tid_beg
;
639 (*histogram_ary
)[res
].bytes
= tid_bytes
[0];
640 for (i
= 1; i
< HIST_COUNT
; ++i
) {
641 if ((*histogram_ary
)[res
].bytes
>= SplitupOpt
) {
643 (*histogram_ary
)[res
].tid
= tid_beg
+
644 i
* (tid_end
- tid_beg
) /
646 (*histogram_ary
)[res
].bytes
= 0;
649 (*histogram_ary
)[res
].bytes
+= tid_bytes
[i
];
652 (*histogram_ary
)[res
].tid
= tid_end
;
653 (*histogram_ary
)[res
].bytes
= -1;
657 fprintf(stderr
, "\n"); /* newline after ... */
658 score_printf(LINE3
, "Prescan %d chunks, total %ju MBytes",
659 res
, (uintmax_t)total
/ (1024 * 1024));
660 fprintf(stderr
, "Prescan %d chunks, total %ju MBytes (",
661 res
, (uintmax_t)total
/ (1024 * 1024));
662 for (i
= 0; i
< res
&& i
< 3; ++i
) {
664 fprintf(stderr
, ", ");
665 fprintf(stderr
, "%ju",
666 (uintmax_t)(*histogram_ary
)[i
].bytes
);
669 fprintf(stderr
, ", ...");
670 fprintf(stderr
, ")\n");
672 assert(res
<= HIST_COUNT
);
680 create_pfs(const char *filesystem
, uuid_t
*s_uuid
)
682 if (ForceYesOpt
== 1) {
683 fprintf(stderr
, "PFS slave %s does not exist. "
684 "Auto create new slave PFS!\n", filesystem
);
687 fprintf(stderr
, "PFS slave %s does not exist.\n"
688 "Do you want to create a new slave PFS? [y/n] ",
692 errx(1, "Aborting operation");
696 char *shared_uuid
= NULL
;
697 uuid_to_string(s_uuid
, &shared_uuid
, &status
);
700 asprintf(&cmd
, "/sbin/hammer pfs-slave '%s' shared-uuid=%s 1>&2",
701 filesystem
, shared_uuid
);
705 errx(1, "Failed to alloc memory");
706 if (system(cmd
) != 0)
707 fprintf(stderr
, "Failed to create PFS\n");
712 * Pipe the mirroring data stream on stdin to the HAMMER VFS, adding
713 * some additional packet types to negotiate TID ranges and to verify
714 * completion. The HAMMER VFS does most of the work.
716 * It is important to note that the mirror.key_{beg,end} range must
717 * match the ranged used by the original. For now both sides use
718 * range the entire key space.
720 * It is even more important that the records in the stream conform
721 * to the TID range also supplied in the stream. The HAMMER VFS will
722 * use the REC, PASS, and SKIP record types to track the portions of
723 * the B-Tree being scanned in order to be able to proactively delete
724 * records on the target within those active areas that are not mentioned
727 * The mirror.key_cur field is used by the VFS to do this tracking. It
728 * must be initialized to key_beg but then is persistently updated by
729 * the HAMMER VFS on each successive ioctl() call. If you blow up this
730 * field you will blow up the mirror target, possibly to the point of
731 * deleting everything. As a safety measure the HAMMER VFS simply marks
732 * the records that the source has destroyed as deleted on the target,
733 * and normal pruning operations will deal with their final disposition
734 * at some later time.
737 hammer_cmd_mirror_write(char **av
, int ac
)
739 struct hammer_ioc_mirror_rw mirror
;
740 const char *filesystem
;
741 char *buf
= malloc(SERIALBUF_SIZE
);
742 struct hammer_ioc_pseudofs_rw pfs
;
743 struct hammer_ioc_mrecord_head pickup
;
744 struct hammer_ioc_synctid synctid
;
745 union hammer_ioc_mrecord_any mrec_tmp
;
746 hammer_ioc_mrecord_any_t mrec
;
755 hammer_check_restrict(filesystem
);
757 pickup
.signature
= 0;
761 bzero(&mirror
, sizeof(mirror
));
762 hammer_key_beg_init(&mirror
.key_beg
);
763 hammer_key_end_init(&mirror
.key_end
);
764 mirror
.key_end
= mirror
.key_beg
;
767 * Read initial packet
769 mrec
= read_mrecord(0, &error
, &pickup
);
772 errx(1, "validate_mrec_header: short read");
778 if (mrec
->head
.type
== HAMMER_MREC_TYPE_TERM
) {
782 if (mrec
->head
.type
!= HAMMER_MREC_TYPE_PFSD
)
783 errx(1, "validate_mrec_header: did not get expected "
785 if (mrec
->head
.rec_size
!= sizeof(mrec
->pfs
))
786 errx(1, "validate_mrec_header: unexpected payload size");
789 * Create slave PFS if it doesn't yet exist
791 if (lstat(filesystem
, &st
) != 0)
792 create_pfs(filesystem
, &mrec
->pfs
.pfsd
.shared_uuid
);
796 fd
= getpfs(&pfs
, filesystem
);
799 * In two-way mode the target writes out a PFS packet first.
800 * The source uses our tid_end as its tid_beg by default,
801 * picking up where it left off.
805 generate_mrec_header(fd
, pfs
.pfs_id
, &mrec_tmp
);
806 if (mirror
.tid_beg
< mrec_tmp
.pfs
.pfsd
.sync_beg_tid
)
807 mirror
.tid_beg
= mrec_tmp
.pfs
.pfsd
.sync_beg_tid
;
808 mirror
.tid_end
= mrec_tmp
.pfs
.pfsd
.sync_end_tid
;
809 write_mrecord(1, HAMMER_MREC_TYPE_PFSD
,
810 &mrec_tmp
, sizeof(mrec_tmp
.pfs
));
814 * Read and process the PFS header. The source informs us of
815 * the TID range the stream represents.
817 n
= validate_mrec_header(fd
, 0, 1, pfs
.pfs_id
, &pickup
,
818 &mirror
.tid_beg
, &mirror
.tid_end
);
819 if (n
< 0) { /* got TERM record */
826 mirror
.size
= SERIALBUF_SIZE
;
829 * Read and process bulk records (REC, PASS, and SKIP types).
831 * On your life, do NOT mess with mirror.key_cur or your mirror
832 * target may become history.
836 mirror
.pfs_id
= pfs
.pfs_id
;
837 mirror
.shared_uuid
= pfs
.ondisk
->shared_uuid
;
838 mirror
.size
= read_mrecords(0, buf
, SERIALBUF_SIZE
, &pickup
);
839 if (mirror
.size
<= 0)
841 if (ioctl(fd
, HAMMERIOC_MIRROR_WRITE
, &mirror
) < 0)
842 err(1, "Mirror-write %s failed", filesystem
);
843 if (mirror
.head
.flags
& HAMMER_IOC_HEAD_ERROR
)
844 errx(1, "Mirror-write %s fatal error %d",
845 filesystem
, mirror
.head
.error
);
847 if (mirror
.head
.flags
& HAMMER_IOC_HEAD_INTR
)
848 errx(1, "Mirror-write %s interrupted by timer at"
851 mirror
.key_cur
.obj_id
);
856 * Read and process the termination sync record.
858 mrec
= read_mrecord(0, &error
, &pickup
);
860 if (mrec
&& mrec
->head
.type
== HAMMER_MREC_TYPE_TERM
) {
861 fprintf(stderr
, "Mirror-write: received termination request\n");
869 (mrec
->head
.type
!= HAMMER_MREC_TYPE_SYNC
&&
870 mrec
->head
.type
!= HAMMER_MREC_TYPE_IDLE
) ||
871 mrec
->head
.rec_size
!= sizeof(mrec
->sync
))
872 errx(1, "Mirror-write %s: Did not get termination "
873 "sync record, or rec_size is wrong rt=%d",
874 filesystem
, (mrec
? (int)mrec
->head
.type
: -1));
877 * Update the PFS info on the target so the user has visibility
878 * into the new snapshot, and sync the target filesystem.
880 if (mrec
->head
.type
== HAMMER_MREC_TYPE_SYNC
) {
881 update_pfs_snapshot(fd
, mirror
.tid_end
, pfs
.pfs_id
);
883 bzero(&synctid
, sizeof(synctid
));
884 synctid
.op
= HAMMER_SYNCTID_SYNC2
;
885 ioctl(fd
, HAMMERIOC_SYNCTID
, &synctid
);
888 fprintf(stderr
, "Mirror-write %s: succeeded\n",
896 * Report back to the originator.
899 mrec_tmp
.update
.tid
= mirror
.tid_end
;
900 write_mrecord(1, HAMMER_MREC_TYPE_UPDATE
,
901 &mrec_tmp
, sizeof(mrec_tmp
.update
));
903 printf("Source can update synctid to 0x%016jx\n",
904 (uintmax_t)mirror
.tid_end
);
911 hammer_cmd_mirror_dump(char **av
, int ac
)
913 char *buf
= malloc(SERIALBUF_SIZE
);
914 struct hammer_ioc_mrecord_head pickup
;
915 hammer_ioc_mrecord_any_t mrec
;
922 if (ac
== 1 && strcmp(*av
, "header") == 0)
928 * Read and process the PFS header
930 pickup
.signature
= 0;
933 mrec
= read_mrecord(0, &error
, &pickup
);
936 * Dump the PFS header. mirror-dump takes its input from the output
937 * of a mirror-read so getpfs() can't be used to get a fd to be passed
940 if (header_only
&& mrec
!= NULL
) {
941 dump_pfsd(&mrec
->pfs
.pfsd
, -1);
950 * Read and process bulk records
953 size
= read_mrecords(0, buf
, SERIALBUF_SIZE
, &pickup
);
957 while (offset
< size
) {
958 mrec
= (void *)((char *)buf
+ offset
);
959 bytes
= HAMMER_HEAD_DOALIGN(mrec
->head
.rec_size
);
960 if (offset
+ bytes
> size
)
961 errx(1, "Misaligned record");
963 switch(mrec
->head
.type
& HAMMER_MRECF_TYPE_MASK
) {
964 case HAMMER_MREC_TYPE_REC_BADCRC
:
965 case HAMMER_MREC_TYPE_REC
:
966 printf("Record lo=%08x obj=%016jx key=%016jx "
968 mrec
->rec
.leaf
.base
.localization
,
969 (uintmax_t)mrec
->rec
.leaf
.base
.obj_id
,
970 (uintmax_t)mrec
->rec
.leaf
.base
.key
,
971 mrec
->rec
.leaf
.base
.rec_type
,
972 mrec
->rec
.leaf
.base
.obj_type
);
973 if (mrec
->head
.type
==
974 HAMMER_MREC_TYPE_REC_BADCRC
)
975 printf(" (BAD CRC)");
977 printf(" tids %016jx:%016jx data=%d\n",
978 (uintmax_t)mrec
->rec
.leaf
.base
.create_tid
,
979 (uintmax_t)mrec
->rec
.leaf
.base
.delete_tid
,
980 mrec
->rec
.leaf
.data_len
);
982 case HAMMER_MREC_TYPE_PASS
:
983 printf("Pass lo=%08x obj=%016jx key=%016jx "
985 mrec
->rec
.leaf
.base
.localization
,
986 (uintmax_t)mrec
->rec
.leaf
.base
.obj_id
,
987 (uintmax_t)mrec
->rec
.leaf
.base
.key
,
988 mrec
->rec
.leaf
.base
.rec_type
,
989 mrec
->rec
.leaf
.base
.obj_type
);
990 printf(" tids %016jx:%016jx data=%d\n",
991 (uintmax_t)mrec
->rec
.leaf
.base
.create_tid
,
992 (uintmax_t)mrec
->rec
.leaf
.base
.delete_tid
,
993 mrec
->rec
.leaf
.data_len
);
995 case HAMMER_MREC_TYPE_SKIP
:
996 printf("Skip lo=%08x obj=%016jx key=%016jx rt=%02x to\n"
997 " lo=%08x obj=%016jx key=%016jx rt=%02x\n",
998 mrec
->skip
.skip_beg
.localization
,
999 (uintmax_t)mrec
->skip
.skip_beg
.obj_id
,
1000 (uintmax_t)mrec
->skip
.skip_beg
.key
,
1001 mrec
->skip
.skip_beg
.rec_type
,
1002 mrec
->skip
.skip_end
.localization
,
1003 (uintmax_t)mrec
->skip
.skip_end
.obj_id
,
1004 (uintmax_t)mrec
->skip
.skip_end
.key
,
1005 mrec
->skip
.skip_end
.rec_type
);
1014 * Read and process the termination sync record.
1016 mrec
= read_mrecord(0, &error
, &pickup
);
1018 (mrec
->head
.type
!= HAMMER_MREC_TYPE_SYNC
&&
1019 mrec
->head
.type
!= HAMMER_MREC_TYPE_IDLE
))
1020 fprintf(stderr
, "Mirror-dump: Did not get termination "
1025 * Continue with more batches until EOF.
1027 mrec
= read_mrecord(0, &error
, &pickup
);
1036 hammer_cmd_mirror_copy(char **av
, int ac
, int streaming
)
1041 const char *xav
[32];
1043 char *sh
, *user
, *host
, *rfs
;
1050 signal(SIGPIPE
, SIG_IGN
);
1059 if ((pid1
= fork()) == 0) {
1060 signal(SIGPIPE
, SIG_DFL
);
1065 if ((rfs
= strchr(av
[0], ':')) != NULL
) {
1068 if((sh
= getenv("HAMMER_RSH")) == NULL
)
1076 if ((host
= strchr(av
[0], '@')) != NULL
) {
1077 user
= strndup( av
[0], (host
++ - av
[0]));
1078 host
= strndup( host
, (rfs
++ - host
));
1083 host
= strndup( av
[0], (rfs
++ - av
[0]));
1091 xav
[xac
++] = SshPort
;
1094 xav
[xac
++] = "hammer";
1096 switch(VerboseOpt
) {
1106 xav
[xac
++] = "-vvv";
1113 snprintf(tbuf
, sizeof(tbuf
), "%d", TimeoutOpt
);
1117 if (SplitupOptStr
) {
1119 xav
[xac
++] = SplitupOptStr
;
1122 xav
[xac
++] = "mirror-read-stream";
1124 xav
[xac
++] = "mirror-read";
1127 execvp(*xav
, (void *)xav
);
1129 hammer_cmd_mirror_read(av
, 1, streaming
);
1139 if ((pid2
= fork()) == 0) {
1140 signal(SIGPIPE
, SIG_DFL
);
1145 if ((rfs
= strchr(av
[1], ':')) != NULL
) {
1148 if((sh
= getenv("HAMMER_RSH")) == NULL
)
1156 if ((host
= strchr(av
[1], '@')) != NULL
) {
1157 user
= strndup( av
[1], (host
++ - av
[1]));
1158 host
= strndup( host
, (rfs
++ - host
));
1163 host
= strndup( av
[1], (rfs
++ - av
[1]));
1170 xav
[xac
++] = SshPort
;
1173 xav
[xac
++] = "hammer";
1175 switch(VerboseOpt
) {
1185 xav
[xac
++] = "-vvv";
1191 xav
[xac
++] = "mirror-write";
1194 execvp(*xav
, (void *)xav
);
1196 hammer_cmd_mirror_write(av
+ 1, 1);
1205 while (waitpid(pid1
, NULL
, 0) <= 0)
1207 while (waitpid(pid2
, NULL
, 0) <= 0)
1211 * If the link is lost restart
1215 fprintf(stderr
, "\nLost Link\n");
1218 sleep(15 + DelayOpt
);
1225 * Read and return multiple mrecords
1228 read_mrecords(int fd
, char *buf
, u_int size
, hammer_ioc_mrecord_head_t pickup
)
1230 hammer_ioc_mrecord_any_t mrec
;
1238 while (size
- count
>= HAMMER_MREC_HEADSIZE
) {
1240 * Cached the record header in case we run out of buffer
1244 if (pickup
->signature
== 0) {
1245 for (n
= 0; n
< HAMMER_MREC_HEADSIZE
; n
+= i
) {
1246 i
= read(fd
, (char *)pickup
+ n
,
1247 HAMMER_MREC_HEADSIZE
- n
);
1253 if (n
!= HAMMER_MREC_HEADSIZE
)
1254 errx(1, "read_mrecords: short read on pipe");
1255 if (pickup
->signature
!= HAMMER_IOC_MIRROR_SIGNATURE
)
1256 errx(1, "read_mrecords: malformed record on pipe, "
1259 if (pickup
->rec_size
< HAMMER_MREC_HEADSIZE
||
1260 pickup
->rec_size
> sizeof(*mrec
) + HAMMER_XBUFSIZE
)
1261 errx(1, "read_mrecords: malformed record on pipe, "
1262 "illegal rec_size");
1265 * Stop if we have insufficient space for the record and data.
1267 bytes
= HAMMER_HEAD_DOALIGN(pickup
->rec_size
);
1268 if (size
- count
< bytes
)
1272 * Stop if the record type is not a REC, SKIP, or PASS,
1273 * which are the only types the ioctl supports. Other types
1274 * are used only by the userland protocol.
1278 type
= pickup
->type
& HAMMER_MRECF_TYPE_LOMASK
;
1279 if (type
!= HAMMER_MREC_TYPE_PFSD
&&
1280 type
!= HAMMER_MREC_TYPE_REC
&&
1281 type
!= HAMMER_MREC_TYPE_SKIP
&&
1282 type
!= HAMMER_MREC_TYPE_PASS
)
1286 * Read the remainder and clear the pickup signature.
1288 for (n
= HAMMER_MREC_HEADSIZE
; n
< bytes
; n
+= i
) {
1289 i
= read(fd
, buf
+ count
+ n
, bytes
- n
);
1294 errx(1, "read_mrecords: short read on pipe");
1296 bcopy(pickup
, buf
+ count
, HAMMER_MREC_HEADSIZE
);
1297 pickup
->signature
= 0;
1299 mrec
= (void *)(buf
+ count
);
1302 * Validate the completed record
1304 if (!hammer_crc_test_mrec_head(&mrec
->head
, mrec
->head
.rec_size
))
1305 errx(1, "read_mrecords: malformed record on pipe, bad crc");
1308 * If its a B-Tree record validate the data crc.
1310 * NOTE: If the VFS passes us an explicitly errorde mrec
1311 * we just pass it through.
1313 type
= mrec
->head
.type
& HAMMER_MRECF_TYPE_MASK
;
1315 if (type
== HAMMER_MREC_TYPE_REC
) {
1316 if (mrec
->head
.rec_size
<
1317 sizeof(mrec
->rec
) + mrec
->rec
.leaf
.data_len
)
1318 errx(1, "read_mrecords: malformed record on "
1319 "pipe, illegal element data_len");
1320 if (mrec
->rec
.leaf
.data_len
&&
1321 mrec
->rec
.leaf
.data_offset
&&
1322 hammer_crc_test_leaf(HammerVersion
, &mrec
->rec
+ 1, &mrec
->rec
.leaf
) == 0) {
1324 "read_mrecords: data_crc did not "
1325 "match data! obj=%016jx key=%016jx\n",
1326 (uintmax_t)mrec
->rec
.leaf
.base
.obj_id
,
1327 (uintmax_t)mrec
->rec
.leaf
.base
.key
);
1329 "continuing, but there are problems\n");
1338 * Read and return a single mrecord.
1341 hammer_ioc_mrecord_any_t
1342 read_mrecord(int fdin
, int *errorp
, hammer_ioc_mrecord_head_t pickup
)
1344 hammer_ioc_mrecord_any_t mrec
;
1345 struct hammer_ioc_mrecord_head mrechd
;
1350 if (pickup
&& pickup
->type
!= 0) {
1352 pickup
->signature
= 0;
1354 n
= HAMMER_MREC_HEADSIZE
;
1357 * Read in the PFSD header from the sender.
1359 for (n
= 0; n
< HAMMER_MREC_HEADSIZE
; n
+= i
) {
1360 i
= read(fdin
, (char *)&mrechd
+ n
, HAMMER_MREC_HEADSIZE
- n
);
1365 *errorp
= 0; /* EOF */
1368 if (n
!= HAMMER_MREC_HEADSIZE
) {
1369 fprintf(stderr
, "short read of mrecord header\n");
1374 if (mrechd
.signature
!= HAMMER_IOC_MIRROR_SIGNATURE
) {
1375 fprintf(stderr
, "read_mrecord: bad signature\n");
1379 bytes
= HAMMER_HEAD_DOALIGN(mrechd
.rec_size
);
1380 assert(bytes
>= sizeof(mrechd
));
1381 mrec
= malloc(bytes
);
1382 mrec
->head
= mrechd
;
1385 i
= read(fdin
, (char *)mrec
+ n
, bytes
- n
);
1391 fprintf(stderr
, "read_mrecord: short read on payload\n");
1395 if (!hammer_crc_test_mrec_head(&mrec
->head
, mrec
->head
.rec_size
)) {
1396 fprintf(stderr
, "read_mrecord: bad CRC\n");
1406 write_mrecord(int fdout
, uint32_t type
, hammer_ioc_mrecord_any_t mrec
,
1409 char zbuf
[HAMMER_HEAD_ALIGN
];
1412 pad
= HAMMER_HEAD_DOALIGN(bytes
) - bytes
;
1414 assert(bytes
>= (int)sizeof(mrec
->head
));
1415 bzero(&mrec
->head
, sizeof(mrec
->head
));
1416 mrec
->head
.signature
= HAMMER_IOC_MIRROR_SIGNATURE
;
1417 mrec
->head
.type
= type
;
1418 mrec
->head
.rec_size
= bytes
;
1419 hammer_crc_set_mrec_head(&mrec
->head
, bytes
);
1420 if (write(fdout
, mrec
, bytes
) != bytes
)
1421 err(1, "write_mrecord");
1424 if (write(fdout
, zbuf
, pad
) != pad
)
1425 err(1, "write_mrecord");
1430 * Generate a mirroring header with the pfs information of the
1431 * originating filesytem.
1434 generate_mrec_header(int fd
, int pfs_id
,
1435 union hammer_ioc_mrecord_any
*mrec_tmp
)
1437 struct hammer_ioc_pseudofs_rw pfs
;
1439 bzero(mrec_tmp
, sizeof(*mrec_tmp
));
1440 clrpfs(&pfs
, &mrec_tmp
->pfs
.pfsd
, pfs_id
);
1442 if (ioctl(fd
, HAMMERIOC_GET_PSEUDOFS
, &pfs
) != 0)
1443 err(1, "Mirror-read: not a HAMMER fs/pseudofs!");
1444 if (pfs
.version
!= HAMMER_IOC_PSEUDOFS_VERSION
)
1445 errx(1, "Mirror-read: HAMMER PFS version mismatch!");
1446 mrec_tmp
->pfs
.version
= pfs
.version
;
1450 * Validate the pfs information from the originating filesystem
1451 * against the target filesystem. shared_uuid must match.
1453 * return -1 if we got a TERM record
1456 validate_mrec_header(int fd
, int fdin
, int is_target
, int pfs_id
,
1457 struct hammer_ioc_mrecord_head
*pickup
,
1458 hammer_tid_t
*tid_begp
, hammer_tid_t
*tid_endp
)
1460 struct hammer_ioc_pseudofs_rw pfs
;
1461 struct hammer_pseudofs_data pfsd
;
1462 hammer_ioc_mrecord_any_t mrec
;
1466 * Get the PFSD info from the target filesystem.
1468 clrpfs(&pfs
, &pfsd
, pfs_id
);
1469 if (ioctl(fd
, HAMMERIOC_GET_PSEUDOFS
, &pfs
) != 0)
1470 err(1, "mirror-write: not a HAMMER fs/pseudofs!");
1471 if (pfs
.version
!= HAMMER_IOC_PSEUDOFS_VERSION
)
1472 errx(1, "mirror-write: HAMMER PFS version mismatch!");
1474 mrec
= read_mrecord(fdin
, &error
, pickup
);
1477 errx(1, "validate_mrec_header: short read");
1480 if (mrec
->head
.type
== HAMMER_MREC_TYPE_TERM
) {
1485 if (mrec
->head
.type
!= HAMMER_MREC_TYPE_PFSD
)
1486 errx(1, "validate_mrec_header: did not get expected "
1487 "PFSD record type");
1488 if (mrec
->head
.rec_size
!= sizeof(mrec
->pfs
))
1489 errx(1, "validate_mrec_header: unexpected payload size");
1490 if (mrec
->pfs
.version
!= pfs
.version
)
1491 errx(1, "validate_mrec_header: Version mismatch");
1494 * Whew. Ok, is the read PFS info compatible with the target?
1496 if (bcmp(&mrec
->pfs
.pfsd
.shared_uuid
, &pfsd
.shared_uuid
,
1497 sizeof(pfsd
.shared_uuid
)) != 0)
1498 errx(1, "mirror-write: source and target have "
1499 "different shared-uuid's!");
1500 if (is_target
&& hammer_is_pfs_master(&pfsd
))
1501 errx(1, "mirror-write: target must be in slave mode");
1503 *tid_begp
= mrec
->pfs
.pfsd
.sync_beg_tid
;
1505 *tid_endp
= mrec
->pfs
.pfsd
.sync_end_tid
;
1511 update_pfs_snapshot(int fd
, hammer_tid_t snapshot_tid
, int pfs_id
)
1513 struct hammer_ioc_pseudofs_rw pfs
;
1514 struct hammer_pseudofs_data pfsd
;
1516 clrpfs(&pfs
, &pfsd
, pfs_id
);
1517 if (ioctl(fd
, HAMMERIOC_GET_PSEUDOFS
, &pfs
) != 0)
1518 err(1, "update_pfs_snapshot (read)");
1520 if (pfsd
.sync_end_tid
!= snapshot_tid
) {
1521 pfsd
.sync_end_tid
= snapshot_tid
;
1522 if (ioctl(fd
, HAMMERIOC_SET_PSEUDOFS
, &pfs
) != 0)
1523 err(1, "update_pfs_snapshot (rewrite)");
1524 if (VerboseOpt
>= 2) {
1526 "Mirror-write: Completed, updated snapshot "
1528 (uintmax_t)snapshot_tid
);
1535 * Bandwidth-limited write in chunks
1539 writebw(int fd
, const void *buf
, size_t nbytes
,
1540 uint64_t *bwcount
, struct timeval
*tv1
)
1551 if (*bwcount
+ nbytes
> BandwidthOpt
)
1552 n
= BandwidthOpt
- *bwcount
;
1556 r
= write(fd
, buf
, n
);
1560 buf
= (const char *)buf
+ r
;
1565 if (*bwcount
>= BandwidthOpt
) {
1566 gettimeofday(&tv2
, NULL
);
1567 usec
= (int)(tv2
.tv_sec
- tv1
->tv_sec
) * 1000000 +
1568 (int)(tv2
.tv_usec
- tv1
->tv_usec
);
1569 if (usec
>= 0 && usec
< 1000000)
1570 usleep(1000000 - usec
);
1571 gettimeofday(tv1
, NULL
);
1572 *bwcount
-= BandwidthOpt
;
1579 * Get a yes or no answer from the terminal. The program may be run as
1580 * part of a two-way pipe so we cannot use stdin for this operation.
1589 fp
= fopen("/dev/tty", "r");
1591 fprintf(stderr
, "No terminal for response\n");
1595 while (fgets(buf
, sizeof(buf
), fp
) != NULL
) {
1596 if (buf
[0] == 'y' || buf
[0] == 'Y') {
1600 if (buf
[0] == 'n' || buf
[0] == 'N') {
1604 fprintf(stderr
, "Response not understood\n");
1612 score_printf(size_t i
, size_t w
, const char *ctl
, ...)
1616 static size_t SSize
;
1617 static int SFd
= -1;
1618 static char ScoreBuf
[1024];
1620 if (ScoreBoardFile
== NULL
)
1622 assert(i
+ w
< sizeof(ScoreBuf
));
1624 SFd
= open(ScoreBoardFile
, O_RDWR
|O_CREAT
|O_TRUNC
, 0644);
1629 for (n
= 0; n
< i
; ++n
) {
1630 if (ScoreBuf
[n
] == 0)
1634 vsnprintf(ScoreBuf
+ i
, w
- 1, ctl
, va
);
1636 n
= strlen(ScoreBuf
+ i
);
1638 ScoreBuf
[i
+ n
] = ' ';
1641 ScoreBuf
[i
+ n
] = '\n';
1644 pwrite(SFd
, ScoreBuf
, SSize
, 0);
1648 hammer_check_restrict(const char *filesystem
)
1653 if (RestrictTarget
== NULL
)
1655 rlen
= strlen(RestrictTarget
);
1656 if (strncmp(filesystem
, RestrictTarget
, rlen
) != 0)
1657 errx(1, "hammer-remote: restricted target");
1660 while (filesystem
[rlen
]) {
1662 filesystem
[rlen
] == '.' &&
1663 filesystem
[rlen
+1] == '.')
1664 errx(1, "hammer-remote: '..' not allowed");
1665 if (filesystem
[rlen
] == '/')
1674 mirror_usage(int code
)
1677 "hammer mirror-read <filesystem> [begin-tid]\n"
1678 "hammer mirror-read-stream <filesystem> [begin-tid]\n"
1679 "hammer mirror-write <filesystem>\n"
1680 "hammer mirror-dump [header]\n"
1681 "hammer mirror-copy [[user@]host:]<filesystem>"
1682 " [[user@]host:]<filesystem>\n"
1683 "hammer mirror-stream [[user@]host:]<filesystem>"
1684 " [[user@]host:]<filesystem>\n"