mogstored

   1 #!/usr/bin/perl
   2
   3 eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
   4     if 0; # not running under some shell
   5
   6 #
   7 # MogileFS storage node daemon
   8 #  (perlbal front-end)
   9 #
  10 # (c) 2004, Brad Fitzpatrick, <brad@danga.com>
  11 # (c) 2006-2007, Six Apart, Ltd.
  12
  13 use strict;
  14 use lib 'lib';
  15 use Mogstored::HTTPServer;
  16
  17 use IO::Socket::INET;
  18 use POSIX qw(WNOHANG);
  19 use Perlbal 1.73;
  20 use FindBin qw($Bin $RealScript);
  21
  22 use Mogstored::HTTPServer::Perlbal;
  23 use Mogstored::HTTPServer::Lighttpd;
  24 use Mogstored::HTTPServer::None;
  25 use Mogstored::HTTPServer::Apache;
  26 use Mogstored::HTTPServer::Nginx;
  27 use Mogstored::SideChannelListener;
  28 use Mogstored::SideChannelClient;
  29
  30 my $selfexe = "$Bin/$RealScript";
  31
  32 # State:
  33 my %on_death;             # pid -> subref (to run when pid dies)
  34 my %devnum_to_device;     # mogile device number (eg. 'dev1' would be '1') -> os device path (eg. '/dev/rd0')
  35 my %osdevnum_to_device;   # os device number (fetched via stat(file)[0]) -> os device path (ec. '/dev/rd0')
  36 my %iostat_listeners;     # fd => SideChannel client: clients interested in iostat data.
  37 my $iostat_available = 1; # bool: iostat working.  assume working to start.
  38 my ($iostat_pipe_r, $iostat_pipe_w);  # pipes for talking to iostat process
  39
  40 # Config:
  41 my $opt_skipconfig;
  42 my $opt_daemonize;
  43 my $opt_config;
  44 my $opt_iostat = 1;  # default to on now
  45 my $max_conns = 10000;
  46 my $http_listen = "0.0.0.0:7500";
  47 my $mgmt_listen = "0.0.0.0:7501";
  48 my $docroot     = "/var/mogdata";
  49 my $default_config = "/etc/mogilefs/mogstored.conf";
  50 my $server      = $ENV{MOGSTORED_SERVER_TYPE} || "perlbal";
  51 my $serverbin   = "";
  52 my $pidfile     = undef;
  53
  54 # Rename binary in process list to make init scripts saner
  55 $0 = "mogstored";
  56
  57 my %config_opts = (
  58                    'iostat'       => \$opt_iostat,
  59                    's|skipconfig' => \$opt_skipconfig,
  60                    'daemonize|d'  => \$opt_daemonize,
  61                    'config=s'     => \$opt_config,
  62                    'httplisten=s' => \$http_listen,
  63                    'mgmtlisten=s' => \$mgmt_listen,
  64                    'docroot=s'    => \$docroot,
  65                    'maxconns=i'   => \$max_conns,
  66                    'server=s'     => \$server,
  67                    'serverbin=s'  => \$serverbin,
  68                    'pidfile=s'    => \$pidfile,
  69                    );
  70 usage() unless Getopt::Long::GetOptions(%config_opts);
  71
  72 die "Unknown server type.  Valid options: --server={perlbal,lighttpd,apache,nginx,none}"
  73     unless $server =~ /^perlbal|lighttpd|apache|nginx|none$/;
  74
  75 $opt_config = $default_config if ! $opt_config && -e $default_config;
  76 load_config_file($opt_config => \%config_opts) if $opt_config && !$opt_skipconfig;
  77
  78 # initialize basic required Perlbal machinery, for any HTTP server
  79 my $perlbal_init = qq{
  80    CREATE SERVICE mogstored
  81      SET role = web_server
  82      SET docroot = $docroot
  83
  84     # don't listen... this is just a stub service.
  85     CREATE SERVICE mgmt
  86        SET role = management
  87      ENABLE mgmt
  88 };
  89 $perlbal_init .= "\nSERVER pidfile = $pidfile" if defined($pidfile);
  90 Perlbal::run_manage_commands($perlbal_init , sub { print STDERR "$_[0]\n"; });
  91
  92 # start HTTP server
  93 my $httpsrv_class = "Mogstored::HTTPServer::" . ucfirst($server);
  94 my $httpsrv       = $httpsrv_class->new(
  95                                         listen   => $http_listen,
  96                                         docroot  => $docroot,
  97                                         maxconns => $max_conns,
  98                                         bin      => $serverbin,
  99                                         );
 100
 101 # Configure Perlbal HTTP listener after daemonization since it can create a
 102 # kqueue on *BSD.  kqueue descriptors are automatically invalidated on fork(),
 103 # making them unusable after daemonize.  For non-Perlbal, starting the
 104 # server before daemonization improves error reporting as daemonization
 105 # redirects stdout/stderr to /dev/null.
 106 $httpsrv->start if $server ne "perlbal";
 107
 108 if ($opt_daemonize) {
 109     $httpsrv->pre_daemonize;
 110     Perlbal::daemonize();
 111 } else {
 112     print "Running.\n";
 113 }
 114
 115 # It is now safe for Perlbal to create a kqueue
 116 $httpsrv->start if $server eq "perlbal";
 117
 118 $httpsrv->post_daemonize;
 119
 120 # kill our children processes on exit:
 121 my $parent_pid = $$;
 122
 123 $SIG{TERM} = $SIG{INT} = sub {
 124     return unless $$ == $parent_pid; # don't let this be inherited
 125     kill 'TERM', grep { $_ } keys %on_death;
 126     POSIX::_exit(0);
 127 };
 128
 129 setup_iostat_pipes();
 130 start_disk_usage_process();
 131 start_iostat_process() if $opt_iostat;
 132 harvest_dead_children();  # every 2 seconds, it reschedules itself
 133 setup_sidechannel_listener();
 134
 135 # now start the main loop
 136 Perlbal::run();
 137
 138 ############################################################################
 139
 140 sub usage {
 141     my $note = shift;
 142     $note = $note ? "NOTE: $note\n\n" : "";
 143
 144     die "${note}Usage: mogstored [OPTS]
 145
 146 OPTS:
 147  --daemonize  -d        Daemonize
 148  --config=<file>        Set config file (default is /etc/mogilefs/mogstored.conf)
 149  --httplisten=<ip:port> IP/Port HTTP server listens on
 150  --mgmtlisten=<ip:port> IP/Port management/sidechannel listens on
 151  --docroot=<path>       Docroot above device mount points.  Defaults to /var/mogdata
 152  --maxconns=<number>    Number of simultaneous clients to serve. Default 10000
 153 ";
 154
 155 }
 156
 157 # accessor for SideChannelClient:
 158 sub Mogstored::iostat_available {
 159     return $iostat_available;
 160 }
 161
 162 sub load_config_file {
 163     my ($conffile, $opts) = @_;
 164
 165     # parse the mogstored config file, which is just lines of comments and
 166     # "key = value" lines, where keys are just the same as command line
 167     # options.
 168     die "Config file $opt_config doesn't exist.\n" unless -e $conffile;
 169     open my $fh, $conffile or die "Couldn't open config file for reading: $!";
 170     while (<$fh>) {
 171         s/\#.*//;
 172         next unless /\S/;
 173         if (/SERVER max_connect/i || /CREATE SERVICE/i) {
 174             usage("Your $opt_config file is the old syntax.  The new format is simply lines of <key> = <value> where keys are the same as mogstored's command line options.");
 175
 176         }
 177         die "Unknown config syntax: $_\n" unless /^\s*(\w+)\s*=\s*(.+?)\s*$/;
 178         my ($key, $val) = ($1, $2);
 179         my $dest;
 180         foreach my $ck (keys %$opts) {
 181             next unless $ck =~ /^$key\b/;
 182             $dest = $opts->{$ck};
 183         }
 184         die "Unknown config setting: $key\n" unless $dest;
 185         $$dest = $val;
 186     }
 187 }
 188
 189 sub harvest_dead_children {
 190     my $dead = waitpid(-1, WNOHANG);
 191     if ($dead > 0) {
 192         my $code = delete $on_death{$dead};
 193         $code->() if $code;
 194     }
 195     Danga::Socket->AddTimer(2, \&harvest_dead_children);
 196 }
 197
 198 sub Mogstored::on_pid_death {
 199     my ($class, $pid, $code) = @_;
 200     $on_death{$pid} = $code;
 201 }
 202
 203 # returns $pid of child, if parent, else runs child.
 204 sub start_disk_usage_process {
 205     my $child = fork;
 206     unless (defined $child) {
 207         Perlbal::log('crit', "Fork error creating disk usage tracking process");
 208         return undef;
 209     }
 210
 211     # if we're the parent.
 212     if ($child) {
 213         $on_death{$child} = sub {
 214             start_disk_usage_process();  # start a new one
 215         };
 216         return $child;
 217     }
 218
 219     require Mogstored::ChildProcess::DiskUsage;
 220     my $class = "Mogstored::ChildProcess::DiskUsage";
 221     $class->pre_exec_init;
 222     $class->exec;
 223 }
 224
 225 sub Mogstored::iostat_subscribe {
 226     my ($class, $sock) = @_;
 227     $iostat_listeners{fileno($sock->sock)} = $sock;
 228 }
 229
 230 sub Mogstored::iostat_unsubscribe {
 231     my ($class, $sock) = @_;
 232     my $fdno = fileno($sock->sock);
 233     return unless defined $fdno;
 234     delete $iostat_listeners{$fdno};
 235 }
 236
 237 # to be honest, I have no clue why this exists.  I just had to move it
 238 # around for multi-server refactoring, and I felt better not
 239 # understanding it but preserving than killing it.  in particular, why
 240 # is this "graceful"?  (gets called from SideChannelClient's
 241 # die_gracefully)
 242 sub Mogstored::on_sidechannel_die_gracefully {
 243     if ($$ == $parent_pid) {
 244         kill 'TERM', grep { $_ } keys %on_death;
 245     }
 246 }
 247
 248 sub setup_sidechannel_listener {
 249     Mogstored::SideChannelListener->new($mgmt_listen);
 250 }
 251
 252 my $iostat_read_buf = "";
 253 sub setup_iostat_pipes {
 254     pipe ($iostat_pipe_r, $iostat_pipe_w);
 255     IO::Handle::blocking($iostat_pipe_r, 0);
 256     IO::Handle::blocking($iostat_pipe_w, 0);
 257
 258     Danga::Socket->AddOtherFds(fileno($iostat_pipe_r), sub {
 259         read_from_iostat_child();
 260     });
 261 }
 262
 263 sub start_iostat_process {
 264     my $pid = fork;
 265     unless (defined $pid) {
 266         warn "Fork for iostat failed: $!";
 267         return;
 268     }
 269
 270     if ($pid) {
 271         # Parent
 272         $on_death{$pid} = sub {
 273             # Try a final read from data on the socket.
 274             # Note that this doesn't internally loop... so it might miss data.
 275             read_from_iostat_child();
 276             # Kill any buffer so partial reads don't hurt us later.
 277             drain_iostat_child_pipe();
 278             start_iostat_process();
 279         };
 280         return;
 281     }
 282
 283     require Mogstored::ChildProcess::IOStat;
 284     my $class = "Mogstored::ChildProcess::IOStat";
 285     $class->pre_exec_init;
 286     $class->exec;
 287 }
 288
 289 sub Mogstored::get_iostat_writer_pipe { $iostat_pipe_w }
 290
 291 sub drain_iostat_child_pipe {
 292     my $data;
 293     while (1) {
 294         last unless sysread($iostat_pipe_r, $data, 10240) > 0;
 295     }
 296     $iostat_read_buf = '';
 297 }
 298
 299 # (runs in parent event-loop process)
 300 sub read_from_iostat_child {
 301     my $data;
 302     my $rv = sysread($iostat_pipe_r, $data, 10240);
 303     return unless $rv && $rv > 0;
 304
 305     $iostat_read_buf .= $data;
 306
 307     # only write complete lines to sockets (in case for some reason we get
 308     # a partial read and child process dies...)
 309     while ($iostat_read_buf =~ s/(.+)\r?\n//) {
 310         my $line = $1;
 311         foreach my $out_sock (values %iostat_listeners) {
 312             # where $line will be like "dev53\t53.23" or a "." to signal end of a group of devices.
 313             # stop writing to the socket if the listener isn't picking it up.
 314             next if $out_sock->{write_buf_size};
 315             $out_sock->write("$line\n");
 316         }
 317     }
 318 }
 319
 320 # Local Variables:
 321 # mode: perl
 322 # c-basic-indent: 4
 323 # indent-tabs-mode: nil
 324 # End:
 325
 326 __END__
 327
 328 =head1 NAME
 329
 330 mogstored -- MogileFS storage daemon
 331
 332 =head1 USAGE
 333
 334 This is the MogileFS storage daemon, which is just an HTTP server that
 335 supports PUT, DELETE, etc.  It's actually a wrapper around L<Perlbal>,
 336 doing all the proper Perlbal config for you.
 337
 338 In addition, it monitors disk usage, I/O activity, etc, which are
 339 checked from the L<MogileFS tracker|mogilefsd>.
 340
 341 =head1 AUTHORS
 342
 343 Brad Fitzpatrick E<lt>brad@danga.comE<gt>
 344
 345 Mark Smith E<lt>junior@danga.comE<gt>
 346
 347 Jonathan Steinert E<lt>jsteinert@sixapart.comE<gt>
 348
 349 =head1 ENVIRONMENT
 350
 351 =over 4
 352
 353 =item PERLBAL_XS_HEADERS
 354
 355 If defined and 0, Perlbal::XS::HTTPHeaders will not be used, if
 356 present.  Otherwise, it will be enabled by default, if installed and
 357 loadable.
 358
 359 =back
 360
 361 =head1 COPYRIGHT
 362
 363  Copyright 2004, Danga Interactive
 364  Copyright 2005-2006, Six Apart Ltd.
 365
 366 =head1 LICENSE
 367
 368 Same terms as Perl itself.  Artistic/GPLv2, at your choosing.
 369
 370 =head1 SEE ALSO
 371
 372 L<MogileFS::Overview> -- high level overview of MogileFS
 373
 374 L<mogilefsd> -- MogileFS daemon
 375
 376 L<http://danga.com/mogilefs/>