cmds-send.c

   1 /*
   2  * Copyright (C) 2012 Alexander Block.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #define _GNU_SOURCE
  20
  21 #include "kerncompat.h"
  22
  23 #include <unistd.h>
  24 #include <stdint.h>
  25 #include <dirent.h>
  26 #include <fcntl.h>
  27 #include <pthread.h>
  28 #include <math.h>
  29 #include <sys/stat.h>
  30 #include <sys/types.h>
  31 #include <sys/ioctl.h>
  32 #include <libgen.h>
  33 #include <mntent.h>
  34 #include <assert.h>
  35
  36 #include <uuid/uuid.h>
  37
  38 #include "ctree.h"
  39 #include "ioctl.h"
  40 #include "commands.h"
  41 #include "list.h"
  42
  43 #include "send.h"
  44 #include "send-utils.h"
  45
  46 static int g_verbose = 0;
  47
  48 struct btrfs_send {
  49         int send_fd;
  50         int dump_fd;
  51         int mnt_fd;
  52
  53         u64 *clone_sources;
  54         u64 clone_sources_count;
  55
  56         char *root_path;
  57         struct subvol_uuid_search sus;
  58 };
  59
  60 int find_mount_root(const char *path, char **mount_root)
  61 {
  62         FILE *mnttab;
  63         int fd;
  64         struct mntent *ent;
  65         int len;
  66         int ret;
  67         int longest_matchlen = 0;
  68         char *longest_match = NULL;
  69
  70         fd = open(path, O_RDONLY | O_NOATIME);
  71         if (fd < 0)
  72                 return -errno;
  73         close(fd);
  74
  75         mnttab = setmntent("/proc/self/mounts", "r");
  76         if (!mnttab)
  77                 return -errno;
  78
  79         while ((ent = getmntent(mnttab))) {
  80                 len = strlen(ent->mnt_dir);
  81                 if (strncmp(ent->mnt_dir, path, len) == 0) {
  82                         /* match found */
  83                         if (longest_matchlen < len) {
  84                                 free(longest_match);
  85                                 longest_matchlen = len;
  86                                 longest_match = strdup(ent->mnt_dir);
  87                         }
  88                 }
  89         }
  90         endmntent(mnttab);
  91
  92         if (!longest_match) {
  93                 fprintf(stderr,
  94                         "ERROR: Failed to find mount root for path %s.\n",
  95                         path);
  96                 return -ENOENT;
  97         }
  98
  99         ret = 0;
 100         *mount_root = realpath(longest_match, NULL);
 101         if (!*mount_root)
 102                 ret = -errno;
 103
 104         free(longest_match);
 105         return ret;
 106 }
 107
 108 static int get_root_id(struct btrfs_send *s, const char *path, u64 *root_id)
 109 {
 110         struct subvol_info *si;
 111
 112         si = subvol_uuid_search(&s->sus, 0, NULL, 0, path,
 113                         subvol_search_by_path);
 114         if (!si)
 115                 return -ENOENT;
 116         *root_id = si->root_id;
 117         free(si->path);
 118         free(si);
 119         return 0;
 120 }
 121
 122 static struct subvol_info *get_parent(struct btrfs_send *s, u64 root_id)
 123 {
 124         struct subvol_info *si_tmp;
 125         struct subvol_info *si;
 126
 127         si_tmp = subvol_uuid_search(&s->sus, root_id, NULL, 0, NULL,
 128                         subvol_search_by_root_id);
 129         if (!si_tmp)
 130                 return NULL;
 131
 132         si = subvol_uuid_search(&s->sus, 0, si_tmp->parent_uuid, 0, NULL,
 133                         subvol_search_by_uuid);
 134         free(si_tmp->path);
 135         free(si_tmp);
 136         return si;
 137 }
 138
 139 static int find_good_parent(struct btrfs_send *s, u64 root_id, u64 *found)
 140 {
 141         int ret;
 142         struct subvol_info *parent = NULL;
 143         struct subvol_info *parent2 = NULL;
 144         struct subvol_info *best_parent = NULL;
 145         __s64 tmp;
 146         u64 best_diff = (u64)-1;
 147         int i;
 148
 149         parent = get_parent(s, root_id);
 150         if (!parent) {
 151                 ret = -ENOENT;
 152                 goto out;
 153         }
 154
 155         for (i = 0; i < s->clone_sources_count; i++) {
 156                 if (s->clone_sources[i] == parent->root_id) {
 157                         best_parent = parent;
 158                         parent = NULL;
 159                         goto out_found;
 160                 }
 161         }
 162
 163         for (i = 0; i < s->clone_sources_count; i++) {
 164                 parent2 = get_parent(s, s->clone_sources[i]);
 165                 if (!parent2)
 166                         continue;
 167                 if (parent2->root_id != parent->root_id) {
 168                         free(parent2->path);
 169                         free(parent2);
 170                         parent2 = NULL;
 171                         continue;
 172                 }
 173
 174                 free(parent2->path);
 175                 free(parent2);
 176                 parent2 = subvol_uuid_search(&s->sus, s->clone_sources[i], NULL,
 177                                 0, NULL, subvol_search_by_root_id);
 178
 179                 assert(parent2);
 180                 tmp = parent2->ctransid - parent->ctransid;
 181                 if (tmp < 0)
 182                         tmp *= -1;
 183                 if (tmp < best_diff) {
 184                         if (best_parent) {
 185                                 free(best_parent->path);
 186                                 free(best_parent);
 187                         }
 188                         best_parent = parent2;
 189                         parent2 = NULL;
 190                         best_diff = tmp;
 191                 } else {
 192                         free(parent2->path);
 193                         free(parent2);
 194                         parent2 = NULL;
 195                 }
 196         }
 197
 198         if (!best_parent) {
 199                 ret = -ENOENT;
 200                 goto out;
 201         }
 202
 203 out_found:
 204         *found = best_parent->root_id;
 205         ret = 0;
 206
 207 out:
 208         if (parent) {
 209                 free(parent->path);
 210                 free(parent);
 211         }
 212         if (best_parent) {
 213                 free(best_parent->path);
 214                 free(best_parent);
 215         }
 216         return ret;
 217 }
 218
 219 static void add_clone_source(struct btrfs_send *s, u64 root_id)
 220 {
 221         s->clone_sources = realloc(s->clone_sources,
 222                 sizeof(*s->clone_sources) * (s->clone_sources_count + 1));
 223         s->clone_sources[s->clone_sources_count++] = root_id;
 224 }
 225
 226 static int write_buf(int fd, const void *buf, int size)
 227 {
 228         int ret;
 229         int pos = 0;
 230
 231         while (pos < size) {
 232                 ret = write(fd, (char*)buf + pos, size - pos);
 233                 if (ret < 0) {
 234                         ret = -errno;
 235                         fprintf(stderr, "ERROR: failed to dump stream. %s",
 236                                         strerror(-ret));
 237                         goto out;
 238                 }
 239                 if (!ret) {
 240                         ret = -EIO;
 241                         fprintf(stderr, "ERROR: failed to dump stream. %s",
 242                                         strerror(-ret));
 243                         goto out;
 244                 }
 245                 pos += ret;
 246         }
 247         ret = 0;
 248
 249 out:
 250         return ret;
 251 }
 252
 253 static void *dump_thread(void *arg_)
 254 {
 255         int ret;
 256         struct btrfs_send *s = (struct btrfs_send*)arg_;
 257         char buf[4096];
 258         int readed;
 259
 260         while (1) {
 261                 readed = read(s->send_fd, buf, sizeof(buf));
 262                 if (readed < 0) {
 263                         ret = -errno;
 264                         fprintf(stderr, "ERROR: failed to read stream from "
 265                                         "kernel. %s\n", strerror(-ret));
 266                         goto out;
 267                 }
 268                 if (!readed) {
 269                         ret = 0;
 270                         goto out;
 271                 }
 272                 ret = write_buf(s->dump_fd, buf, readed);
 273                 if (ret < 0)
 274                         goto out;
 275         }
 276
 277 out:
 278         if (ret < 0) {
 279                 exit(-ret);
 280         }
 281
 282         return ERR_PTR(ret);
 283 }
 284
 285 static int do_send(struct btrfs_send *send, u64 parent_root_id,
 286                    int is_first_subvol, int is_last_subvol, char *subvol)
 287 {
 288         int ret;
 289         pthread_t t_read;
 290         pthread_attr_t t_attr;
 291         struct btrfs_ioctl_send_args io_send;
 292         void *t_err = NULL;
 293         int subvol_fd = -1;
 294         int pipefd[2] = {-1, -1};
 295
 296         subvol_fd = openat(send->mnt_fd, subvol, O_RDONLY | O_NOATIME);
 297         if (subvol_fd < 0) {
 298                 ret = -errno;
 299                 fprintf(stderr, "ERROR: open %s failed. %s\n", subvol,
 300                                 strerror(-ret));
 301                 goto out;
 302         }
 303
 304         ret = pthread_attr_init(&t_attr);
 305
 306         ret = pipe(pipefd);
 307         if (ret < 0) {
 308                 ret = -errno;
 309                 fprintf(stderr, "ERROR: pipe failed. %s\n", strerror(-ret));
 310                 goto out;
 311         }
 312
 313         memset(&io_send, 0, sizeof(io_send));
 314         io_send.send_fd = pipefd[1];
 315         send->send_fd = pipefd[0];
 316
 317         if (!ret)
 318                 ret = pthread_create(&t_read, &t_attr, dump_thread,
 319                                         send);
 320         if (ret) {
 321                 ret = -ret;
 322                 fprintf(stderr, "ERROR: thread setup failed: %s\n",
 323                         strerror(-ret));
 324                 goto out;
 325         }
 326
 327         io_send.clone_sources = (__u64*)send->clone_sources;
 328         io_send.clone_sources_count = send->clone_sources_count;
 329         io_send.parent_root = parent_root_id;
 330         if (!is_first_subvol)
 331                 io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
 332         if (!is_last_subvol)
 333                 io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
 334         ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 335         if (ret) {
 336                 ret = -errno;
 337                 fprintf(stderr, "ERROR: send ioctl failed with %d: %s\n", ret,
 338                         strerror(-ret));
 339                 if (ret == -EINVAL && (!is_first_subvol || !is_last_subvol))
 340                         fprintf(stderr,
 341                                 "Try upgrading your kernel or don't use -e.\n");
 342                 goto out;
 343         }
 344         if (g_verbose > 0)
 345                 fprintf(stderr, "BTRFS_IOC_SEND returned %d\n", ret);
 346
 347         if (g_verbose > 0)
 348                 fprintf(stderr, "joining genl thread\n");
 349
 350         close(pipefd[1]);
 351         pipefd[1] = -1;
 352
 353         ret = pthread_join(t_read, &t_err);
 354         if (ret) {
 355                 ret = -ret;
 356                 fprintf(stderr, "ERROR: pthread_join failed: %s\n",
 357                         strerror(-ret));
 358                 goto out;
 359         }
 360         if (t_err) {
 361                 ret = (long int)t_err;
 362                 fprintf(stderr, "ERROR: failed to process send stream, ret=%ld "
 363                         "(%s)\n", (long int)t_err, strerror(-ret));
 364                 goto out;
 365         }
 366
 367         pthread_attr_destroy(&t_attr);
 368
 369         ret = 0;
 370
 371 out:
 372         if (subvol_fd != -1)
 373                 close(subvol_fd);
 374         if (pipefd[0] != -1)
 375                 close(pipefd[0]);
 376         if (pipefd[1] != -1)
 377                 close(pipefd[1]);
 378         return ret;
 379 }
 380
 381 char *get_subvol_name(char *mnt, char *full_path)
 382 {
 383         int len = strlen(mnt);
 384         if (!len)
 385                 return full_path;
 386         if (mnt[len - 1] != '/')
 387                 len += 1;
 388
 389         return full_path + len;
 390 }
 391
 392 static int init_root_path(struct btrfs_send *s, const char *subvol)
 393 {
 394         int ret = 0;
 395
 396         if (s->root_path)
 397                 goto out;
 398
 399         ret = find_mount_root(subvol, &s->root_path);
 400         if (ret < 0) {
 401                 ret = -EINVAL;
 402                 fprintf(stderr, "ERROR: failed to determine mount point "
 403                                 "for %s\n", subvol);
 404                 goto out;
 405         }
 406
 407         s->mnt_fd = open(s->root_path, O_RDONLY | O_NOATIME);
 408         if (s->mnt_fd < 0) {
 409                 ret = -errno;
 410                 fprintf(stderr, "ERROR: can't open '%s': %s\n", s->root_path,
 411                         strerror(-ret));
 412                 goto out;
 413         }
 414
 415         ret = subvol_uuid_search_init(s->mnt_fd, &s->sus);
 416         if (ret < 0) {
 417                 fprintf(stderr, "ERROR: failed to initialize subvol search. "
 418                                 "%s\n", strerror(-ret));
 419                 goto out;
 420         }
 421
 422 out:
 423         return ret;
 424
 425 }
 426
 427 static int is_subvol_ro(struct btrfs_send *s, char *subvol)
 428 {
 429         int ret;
 430         u64 flags;
 431         int fd = -1;
 432
 433         fd = openat(s->mnt_fd, subvol, O_RDONLY | O_NOATIME);
 434         if (fd < 0) {
 435                 ret = -errno;
 436                 fprintf(stderr, "ERROR: failed to open %s. %s\n",
 437                                 subvol, strerror(-ret));
 438                 goto out;
 439         }
 440
 441         ret = ioctl(fd, BTRFS_IOC_SUBVOL_GETFLAGS, &flags);
 442         if (ret < 0) {
 443                 ret = -errno;
 444                 fprintf(stderr, "ERROR: failed to get flags for subvolume. "
 445                                 "%s\n", strerror(-ret));
 446                 goto out;
 447         }
 448
 449         if (flags & BTRFS_SUBVOL_RDONLY)
 450                 ret = 1;
 451         else
 452                 ret = 0;
 453
 454 out:
 455         if (fd != -1)
 456                 close(fd);
 457
 458         return ret;
 459 }
 460
 461 int cmd_send(int argc, char **argv)
 462 {
 463         char *subvol = NULL;
 464         int c;
 465         int ret;
 466         char *outname = NULL;
 467         struct btrfs_send send;
 468         u32 i;
 469         char *mount_root = NULL;
 470         char *snapshot_parent = NULL;
 471         u64 root_id;
 472         u64 parent_root_id = 0;
 473         int full_send = 1;
 474         int new_end_cmd_semantic = 0;
 475
 476         memset(&send, 0, sizeof(send));
 477         send.dump_fd = fileno(stdout);
 478
 479         while ((c = getopt(argc, argv, "vec:f:i:p:")) != -1) {
 480                 switch (c) {
 481                 case 'v':
 482                         g_verbose++;
 483                         break;
 484                 case 'e':
 485                         new_end_cmd_semantic = 1;
 486                         break;
 487                 case 'c':
 488                         subvol = realpath(optarg, NULL);
 489                         if (!subvol) {
 490                                 ret = -errno;
 491                                 fprintf(stderr, "ERROR: realpath %s failed. "
 492                                                 "%s\n", optarg, strerror(-ret));
 493                                 goto out;
 494                         }
 495
 496                         ret = init_root_path(&send, subvol);
 497                         if (ret < 0)
 498                                 goto out;
 499
 500                         ret = get_root_id(&send, get_subvol_name(send.root_path, subvol),
 501                                         &root_id);
 502                         if (ret < 0) {
 503                                 fprintf(stderr, "ERROR: could not resolve "
 504                                                 "root_id for %s\n", subvol);
 505                                 goto out;
 506                         }
 507
 508                         ret = is_subvol_ro(&send, subvol);
 509                         if (ret < 0)
 510                                 goto out;
 511                         if (!ret) {
 512                                 ret = -EINVAL;
 513                                 fprintf(stderr,
 514                                 "ERROR: cloned subvol %s is not read-only.\n",
 515                                         subvol);
 516                                 goto out;
 517                         }
 518
 519                         add_clone_source(&send, root_id);
 520                         subvol_uuid_search_finit(&send.sus);
 521                         free(subvol);
 522                         subvol = NULL;
 523                         if (send.mnt_fd >= 0) {
 524                                 close(send.mnt_fd);
 525                                 send.mnt_fd = -1;
 526                         }
 527                         free(send.root_path);
 528                         send.root_path = NULL;
 529                         full_send = 0;
 530                         break;
 531                 case 'f':
 532                         outname = optarg;
 533                         break;
 534                 case 'p':
 535                         if (snapshot_parent) {
 536                                 fprintf(stderr, "ERROR: you cannot have more than one parent (-p)\n");
 537                                 ret = 1;
 538                                 goto out;
 539                         }
 540                         snapshot_parent = realpath(optarg, NULL);
 541                         if (!snapshot_parent) {
 542                                 ret = -errno;
 543                                 fprintf(stderr, "ERROR: realpath %s failed. "
 544                                                 "%s\n", optarg, strerror(-ret));
 545                                 goto out;
 546                         }
 547
 548                         ret = is_subvol_ro(&send, snapshot_parent);
 549                         if (ret < 0)
 550                                 goto out;
 551                         if (!ret) {
 552                                 ret = -EINVAL;
 553                                 fprintf(stderr,
 554                                         "ERROR: parent %s is not read-only.\n",
 555                                         snapshot_parent);
 556                                 goto out;
 557                         }
 558
 559                         full_send = 0;
 560                         break;
 561                 case 'i':
 562                         fprintf(stderr,
 563                                 "ERROR: -i was removed, use -c instead\n");
 564                         ret = 1;
 565                         goto out;
 566                 case '?':
 567                 default:
 568                         fprintf(stderr, "ERROR: send args invalid.\n");
 569                         ret = 1;
 570                         goto out;
 571                 }
 572         }
 573
 574         if (optind == argc) {
 575                 fprintf(stderr, "ERROR: send needs path to snapshot\n");
 576                 ret = 1;
 577                 goto out;
 578         }
 579
 580         if (outname != NULL) {
 581                 send.dump_fd = creat(outname, 0600);
 582                 if (send.dump_fd == -1) {
 583                         ret = -errno;
 584                         fprintf(stderr, "ERROR: can't create '%s': %s\n",
 585                                         outname, strerror(-ret));
 586                         goto out;
 587                 }
 588         }
 589
 590         if (isatty(send.dump_fd)) {
 591                 fprintf(stderr,
 592                         "ERROR: not dumping send stream into a terminal, "
 593                         "redirect it into a file\n");
 594                 ret = 1;
 595                 goto out;
 596         }
 597
 598         /* use first send subvol to determine mount_root */
 599         subvol = argv[optind];
 600
 601         subvol = realpath(argv[optind], NULL);
 602         if (!subvol) {
 603                 ret = -errno;
 604                 fprintf(stderr, "ERROR: unable to resolve %s\n", argv[optind]);
 605                 goto out;
 606         }
 607
 608         ret = init_root_path(&send, subvol);
 609         if (ret < 0)
 610                 goto out;
 611
 612         if (snapshot_parent != NULL) {
 613                 ret = get_root_id(&send,
 614                                 get_subvol_name(send.root_path, snapshot_parent),
 615                                 &parent_root_id);
 616                 if (ret < 0) {
 617                         fprintf(stderr, "ERROR: could not resolve root_id "
 618                                         "for %s\n", snapshot_parent);
 619                         goto out;
 620                 }
 621
 622                 add_clone_source(&send, parent_root_id);
 623         }
 624
 625         for (i = optind; i < argc; i++) {
 626                 free(subvol);
 627                 subvol = realpath(argv[i], NULL);
 628                 if (!subvol) {
 629                         ret = -errno;
 630                         fprintf(stderr, "ERROR: unable to resolve %s\n", argv[i]);
 631                         goto out;
 632                 }
 633
 634                 ret = find_mount_root(subvol, &mount_root);
 635                 if (ret < 0) {
 636                         fprintf(stderr, "ERROR: find_mount_root failed on %s: "
 637                                         "%s\n", subvol,
 638                                 strerror(-ret));
 639                         goto out;
 640                 }
 641                 if (strcmp(send.root_path, mount_root) != 0) {
 642                         ret = -EINVAL;
 643                         fprintf(stderr, "ERROR: all subvols must be from the "
 644                                         "same fs.\n");
 645                         goto out;
 646                 }
 647                 free(mount_root);
 648
 649                 ret = is_subvol_ro(&send, subvol);
 650                 if (ret < 0)
 651                         goto out;
 652                 if (!ret) {
 653                         ret = -EINVAL;
 654                         fprintf(stderr, "ERROR: %s is not read-only.\n",
 655                                         subvol);
 656                         goto out;
 657                 }
 658         }
 659
 660         for (i = optind; i < argc; i++) {
 661                 int is_first_subvol;
 662                 int is_last_subvol;
 663
 664                 free(subvol);
 665                 subvol = argv[i];
 666
 667                 fprintf(stderr, "At subvol %s\n", subvol);
 668
 669                 subvol = realpath(subvol, NULL);
 670                 if (!subvol) {
 671                         ret = -errno;
 672                         fprintf(stderr, "ERROR: realpath %s failed. "
 673                                         "%s\n", argv[i], strerror(-ret));
 674                         goto out;
 675                 }
 676
 677                 if (!full_send && !parent_root_id) {
 678                         ret = find_good_parent(&send, root_id, &parent_root_id);
 679                         if (ret < 0) {
 680                                 fprintf(stderr, "ERROR: parent determination failed for %lld\n",
 681                                         root_id);
 682                                 goto out;
 683                         }
 684                 }
 685
 686                 ret = is_subvol_ro(&send, subvol);
 687                 if (ret < 0)
 688                         goto out;
 689                 if (!ret) {
 690                         ret = -EINVAL;
 691                         fprintf(stderr, "ERROR: %s is not read-only.\n",
 692                                         subvol);
 693                         goto out;
 694                 }
 695
 696                 if (new_end_cmd_semantic) {
 697                         /* require new kernel */
 698                         is_first_subvol = (i == optind);
 699                         is_last_subvol = (i == argc - 1);
 700                 } else {
 701                         /* be compatible to old and new kernel */
 702                         is_first_subvol = 1;
 703                         is_last_subvol = 1;
 704                 }
 705                 ret = do_send(&send, parent_root_id, is_first_subvol,
 706                               is_last_subvol, subvol);
 707                 if (ret < 0)
 708                         goto out;
 709
 710                 /* done with this subvol, so add it to the clone sources */
 711                 add_clone_source(&send, root_id);
 712
 713                 parent_root_id = 0;
 714                 full_send = 0;
 715         }
 716
 717         ret = 0;
 718
 719 out:
 720         free(subvol);
 721         free(snapshot_parent);
 722         free(send.clone_sources);
 723         if (send.mnt_fd >= 0)
 724                 close(send.mnt_fd);
 725         free(send.root_path);
 726         subvol_uuid_search_finit(&send.sus);
 727         return !!ret;
 728 }
 729
 730 const char * const cmd_send_usage[] = {
 731         "btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 732         "Send the subvolume(s) to stdout.",
 733         "Sends the subvolume(s) specified by <subvol> to stdout.",
 734         "By default, this will send the whole subvolume. To do an incremental",
 735         "send, use '-p <parent>'. If you want to allow btrfs to clone from",
 736         "any additional local snapshots, use '-c <clone-src>' (multiple times",
 737         "where applicable). You must not specify clone sources unless you",
 738         "guarantee that these snapshots are exactly in the same state on both",
 739         "sides, the sender and the receiver. It is allowed to omit the",
 740         "'-p <parent>' option when '-c <clone-src>' options are given, in",
 741         "which case 'btrfs send' will determine a suitable parent among the",
 742         "clone sources itself.",
 743         "\n",
 744         "-v               Enable verbose debug output. Each occurrence of",
 745         "                 this option increases the verbose level more.",
 746         "-e               If sending multiple subvols at once, use the new",
 747         "                 format and omit the end-cmd between the subvols.",
 748         "-p <parent>      Send an incremental stream from <parent> to",
 749         "                 <subvol>.",
 750         "-c <clone-src>   Use this snapshot as a clone source for an ",
 751         "                 incremental send (multiple allowed)",
 752         "-f <outfile>     Output is normally written to stdout. To write to",
 753         "                 a file, use this option. An alternative would be to",
 754         "                 use pipes.",
 755         NULL
 756 };