cmd/dcs-localdcs/localdcs.go

   1 package main
   2
   3 import (
   4         "flag"
   5         "fmt"
   6         "io/ioutil"
   7         "log"
   8         "net/http"
   9         "os"
  10         "os/exec"
  11         "os/user"
  12         "path/filepath"
  13         "strconv"
  14         "strings"
  15         "syscall"
  16         "time"
  17 )
  18
  19 var (
  20         unpackedPath = flag.String("unpacked_path",
  21                 "/tmp/dcs-hacking",
  22                 "Path to the unpacked sources")
  23         localdcsPath = flag.String("localdcs_path",
  24                 "~/.config/dcs-localdcs",
  25                 "Directory in which to keep state for dcs-localdcs (TLS certificates, PID files, etc.)")
  26         stop = flag.Bool("stop",
  27                 false,
  28                 "Whether to stop the currently running localdcs instead of starting a new one")
  29         listenPackageImporter = flag.String("listen_package_importer",
  30                 "localhost:21010",
  31                 "listen address ([host]:port) for dcs-package-importer")
  32         listenIndexBackend = flag.String("listen_index_backend",
  33                 "localhost:28081",
  34                 "listen address ([host]:port) for dcs-index-backend")
  35         listenSourceBackend = flag.String("listen_source_backend",
  36                 "localhost:28082",
  37                 "listen address ([host]:port) for dcs-source-backend")
  38         listenWeb = flag.String("listen_web",
  39                 "localhost:28080",
  40                 "listen address ([host]:port) for dcs-web")
  41         listenWebTLS = flag.String("listen_web_tls",
  42                 "",
  43                 "listen address ([host]:port) for dcs-web (TLS)")
  44 )
  45
  46 func installBinaries() error {
  47         cmd := exec.Command("go", "install", "-ldflags", "-X github.com/Debian/dcs/cmd/dcs-web/common.Version=git", "github.com/Debian/dcs/cmd/...")
  48         cmd.Stderr = os.Stderr
  49         log.Printf("Compiling and installing binaries: %v\n", cmd.Args)
  50         return cmd.Run()
  51 }
  52
  53 func help(binary string) error {
  54         err := exec.Command(binary, "-help").Run()
  55         if exiterr, ok := err.(*exec.ExitError); ok {
  56                 status, ok := exiterr.Sys().(syscall.WaitStatus)
  57                 if !ok {
  58                         log.Panicf("cannot run on this platform: exec.ExitError.Sys() does not return syscall.WaitStatus")
  59                 }
  60                 // -help results in exit status 2, so that’s expected.
  61                 if status.ExitStatus() == 2 {
  62                         return nil
  63                 }
  64         }
  65         return err
  66 }
  67
  68 func verifyBinariesAreExecutable() error {
  69         binaries := []string{
  70                 "dcs-package-importer",
  71                 "dcs-index-backend",
  72                 "dcs-source-backend",
  73                 "dcs-web",
  74                 "dcs-compute-ranking",
  75         }
  76         log.Printf("Verifying binaries are executable: %v\n", binaries)
  77         for _, binary := range binaries {
  78                 if err := help(binary); err != nil {
  79                         return err
  80                 }
  81         }
  82         return nil
  83 }
  84
  85 func compileStaticAssets() error {
  86         cmd := exec.Command("make")
  87         cmd.Stderr = os.Stderr
  88         cmd.Dir = "static"
  89         log.Printf("Compiling static assets: %v\n", cmd.Args)
  90         return cmd.Run()
  91 }
  92
  93 // recordResource appends a line to a file in -localnet_dir so that we can
  94 // clean up resources (tempdirs, pids) when being called with -stop later.
  95 func recordResource(rtype, value string) error {
  96         f, err := os.OpenFile(filepath.Join(*localdcsPath, rtype+"s"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
  97         if err != nil {
  98                 return err
  99         }
 100         defer f.Close()
 101         _, err = fmt.Fprintf(f, "%s\n", value)
 102         return err
 103 }
 104
 105 func kill() error {
 106         pidsFile := filepath.Join(*localdcsPath, "pids")
 107         if _, err := os.Stat(pidsFile); os.IsNotExist(err) {
 108                 return fmt.Errorf("-stop specified, but no localdcs instance found in -localdcs_path=%q", *localdcsPath)
 109         }
 110
 111         pidsBytes, err := ioutil.ReadFile(pidsFile)
 112         if err != nil {
 113                 return fmt.Errorf("Could not read %q: %v", pidsFile, err)
 114         }
 115         pids := strings.Split(string(pidsBytes), "\n")
 116         for _, pidline := range pids {
 117                 if pidline == "" {
 118                         continue
 119                 }
 120                 pid, err := strconv.Atoi(pidline)
 121                 if err != nil {
 122                         return fmt.Errorf("Invalid line in %q: %v", pidsFile, err)
 123                 }
 124
 125                 process, err := os.FindProcess(pid)
 126                 if err != nil {
 127                         log.Printf("Could not find process %d: %v", pid, err)
 128                         continue
 129                 }
 130                 if err := process.Kill(); err != nil {
 131                         log.Printf("Could not kill process %d: %v", pid, err)
 132                 }
 133         }
 134
 135         os.Remove(pidsFile)
 136
 137         return nil
 138 }
 139
 140 func launchInBackground(binary string, args ...string) error {
 141         // TODO: redirect stderr into a file
 142         cmd := exec.Command(binary, args...)
 143         cmd.Stderr = os.Stderr
 144         // Put the robustirc servers into a separate process group, so that they
 145         // survive when robustirc-localnet terminates.
 146         cmd.SysProcAttr = &syscall.SysProcAttr{
 147                 Setpgid: true,
 148         }
 149         if err := cmd.Start(); err != nil {
 150                 return fmt.Errorf("Could not start %q: %v", binary, err)
 151         }
 152         if err := recordResource("pid", strconv.Itoa(cmd.Process.Pid)); err != nil {
 153                 return fmt.Errorf("Could not record pid of %q: %v", binary, err)
 154         }
 155         return nil
 156 }
 157
 158 func httpReachable(addr string) error {
 159         // Poll the configured listening port to see if the server started up successfully.
 160         running := false
 161         var err error
 162         for try := 0; !running && try < 20; try++ {
 163                 _, err = http.Get("http://" + addr)
 164                 if err != nil {
 165                         time.Sleep(250 * time.Millisecond)
 166                         continue
 167                 }
 168
 169                 // Any HTTP response is okay.
 170                 return nil
 171         }
 172         return err
 173 }
 174
 175 func feed(pkg, file string) error {
 176         f, err := os.Open(file)
 177         if err != nil {
 178                 return err
 179         }
 180         defer f.Close()
 181         url := fmt.Sprintf("http://%s/import/%s/%s", *listenPackageImporter, pkg, filepath.Base(file))
 182         request, err := http.NewRequest("PUT", url, f)
 183         if err != nil {
 184                 return err
 185         }
 186         resp, err := http.DefaultClient.Do(request)
 187         if err != nil {
 188                 return err
 189         }
 190         defer resp.Body.Close()
 191
 192         if resp.StatusCode != http.StatusOK {
 193                 return fmt.Errorf("Unexpected HTTP status: got %d, want %d", resp.StatusCode, http.StatusOK)
 194         }
 195
 196         return nil
 197 }
 198
 199 func importTestdata() error {
 200         testdataFiles := make(map[string][]string)
 201         if err := filepath.Walk("testdata/pool", func(path string, info os.FileInfo, err error) error {
 202                 if info.IsDir() {
 203                         return nil
 204                 }
 205                 dir := filepath.Dir(path)
 206                 testdataFiles[dir] = append(testdataFiles[dir], path)
 207                 return nil
 208         }); err != nil {
 209                 return err
 210         }
 211         // e.g. testdataFiles = map[
 212         //   testdata/pool/main/i/i3-wm:[
 213         //     testdata/pool/main/i/i3-wm/i3-wm_4.5.1-2.debian.tar.gz
 214         //     testdata/pool/main/i/i3-wm/i3-wm_4.5.1-2.dsc
 215         //     testdata/pool/main/i/i3-wm/i3-wm_4.5.1.orig.tar.bz2]
 216         //   testdata/pool/main/z/zsh:[
 217         //     testdata/pool/main/z/zsh/zsh_5.2-3.debian.tar.xz
 218         //     testdata/pool/main/z/zsh/zsh_5.2-3.dsc
 219         //     testdata/pool/main/z/zsh/zsh_5.2.orig.tar.xz]]
 220         numPackages := 0
 221         for _, files := range testdataFiles {
 222                 var dsc string
 223                 var rest []string
 224                 for _, file := range files {
 225                         if filepath.Ext(file) == ".dsc" {
 226                                 dsc = file
 227                         } else {
 228                                 rest = append(rest, file)
 229                         }
 230                 }
 231                 if dsc == "" {
 232                         continue
 233                 }
 234                 numPackages++
 235                 // e.g.:
 236                 // dsc "testdata/pool/main/i/i3-wm/i3-wm_4.5.1-2.dsc"
 237                 // rest [
 238                 //   testdata/pool/main/i/i3-wm/i3-wm_4.5.1-2.debian.tar.gz
 239                 //   testdata/pool/main/i/i3-wm/i3-wm_4.5.1.orig.tar.bz2]
 240                 pkg := filepath.Base(dsc)
 241                 pkg = pkg[:len(pkg)-len(filepath.Ext(pkg))]
 242                 log.Printf("Importing package %q (files %v, dsc %s)\n", pkg, rest, dsc)
 243                 for _, file := range append(rest, dsc) {
 244                         if err := feed(pkg, file); err != nil {
 245                                 return err
 246                         }
 247                 }
 248         }
 249
 250         // Wait with merging until all packages were imported, but up to 5s max.
 251         for try := 0; try < 5; try++ {
 252                 resp, err := http.Get("http://" + *listenPackageImporter + "/metrics")
 253                 if err != nil {
 254                         time.Sleep(250 * time.Millisecond)
 255                         continue
 256                 }
 257                 body, err := ioutil.ReadAll(resp.Body)
 258                 resp.Body.Close()
 259                 if err != nil {
 260                         return err
 261                 }
 262                 if strings.Contains(string(body), fmt.Sprintf("package_indexes_successful %d", numPackages)) {
 263                         break
 264                 }
 265                 time.Sleep(1 * time.Second)
 266         }
 267
 268         _, err := http.Get("http://" + *listenPackageImporter + "/merge")
 269         if err != nil {
 270                 return err
 271         }
 272         // Wait for up to 20s until the merge completed
 273         for try := 0; try < 20; try++ {
 274                 resp, err := http.Get("http://" + *listenPackageImporter + "/metrics")
 275                 if err != nil {
 276                         time.Sleep(250 * time.Millisecond)
 277                         continue
 278                 }
 279                 body, err := ioutil.ReadAll(resp.Body)
 280                 resp.Body.Close()
 281                 if err != nil {
 282                         return err
 283                 }
 284                 if strings.Contains(string(body), "merges_successful 1") {
 285                         break
 286                 }
 287                 time.Sleep(1 * time.Second)
 288         }
 289         return nil
 290 }
 291
 292 func main() {
 293         flag.Parse()
 294
 295         if len(*localdcsPath) >= 2 && (*localdcsPath)[:2] == "~/" {
 296                 usr, err := user.Current()
 297                 if err != nil {
 298                         log.Fatalf("Cannot expand -localdcs_path: %v", err)
 299                 }
 300                 *localdcsPath = strings.Replace(*localdcsPath, "~/", usr.HomeDir+"/", 1)
 301         }
 302
 303         if err := os.MkdirAll(*localdcsPath, 0700); err != nil {
 304                 log.Fatalf("Could not create directory %q for dcs-localdcs state: %v", *localdcsPath, err)
 305         }
 306
 307         if *stop {
 308                 if err := kill(); err != nil {
 309                         log.Fatalf("Could not stop localdcs: %v", err)
 310                 }
 311                 return
 312         }
 313
 314         if _, err := os.Stat(filepath.Join(*localdcsPath, "pids")); !os.IsNotExist(err) {
 315                 log.Fatalf("There already is a localdcs instance running. Either use -stop or specify a different -localdcs_path")
 316         }
 317
 318         if err := os.MkdirAll(*unpackedPath, 0755); err != nil {
 319                 log.Fatalf("Could not create directory %q for unpacked files/index: %v", *unpackedPath, err)
 320         }
 321
 322         if err := installBinaries(); err != nil {
 323                 log.Fatalf("Compiling and installing binaries failed: %v", err)
 324         }
 325
 326         if err := verifyBinariesAreExecutable(); err != nil {
 327                 log.Fatalf("Could not find all required binaries: %v", err)
 328         }
 329
 330         if err := compileStaticAssets(); err != nil {
 331                 log.Fatalf("Compiling static assets failed: %v", err)
 332         }
 333
 334         if _, err := os.Stat(filepath.Join(*localdcsPath, "key.pem")); os.IsNotExist(err) {
 335                 log.Printf("Generating TLS certificate\n")
 336                 if err := generatecert(*localdcsPath); err != nil {
 337                         log.Fatalf("Could not generate TLS certificate: %v", err)
 338                 }
 339         }
 340
 341         rankingPath := filepath.Join(*localdcsPath, "ranking.json")
 342         if stat, err := os.Stat(rankingPath); err != nil || time.Since(stat.ModTime()) > 7*24*time.Hour {
 343                 log.Printf("Computing ranking data\n")
 344                 cmd := exec.Command(
 345                         "dcs-compute-ranking",
 346                         "-output_path="+rankingPath)
 347                 cmd.Env = append(os.Environ(),
 348                         "TMPDIR="+*localdcsPath)
 349                 cmd.Stderr = os.Stderr
 350                 if err := cmd.Run(); err != nil {
 351                         log.Fatalf("Could not compute ranking data: %v", err)
 352                 }
 353         } else {
 354                 log.Printf("Recent-enough rankings file %q found, not re-generating (delete to force)\n", rankingPath)
 355         }
 356
 357         // Start package importer and import testdata/
 358         if err := launchInBackground(
 359                 "dcs-package-importer",
 360                 "-debug_skip",
 361                 "-varz_avail_fs=",
 362                 "-tls_cert_path="+filepath.Join(*localdcsPath, "cert.pem"),
 363                 "-tls_key_path="+filepath.Join(*localdcsPath, "key.pem"),
 364                 "-unpacked_path="+*unpackedPath,
 365                 "-listen_address="+*listenPackageImporter); err != nil {
 366                 log.Fatal(err.Error())
 367         }
 368         if err := httpReachable(*listenPackageImporter); err != nil {
 369                 log.Fatalf("dcs-package-importer not reachable: %v", err)
 370         }
 371         if err := importTestdata(); err != nil {
 372                 log.Fatalf("Could not import testdata/: %v", err)
 373         }
 374
 375         if err := launchInBackground(
 376                 "dcs-index-backend",
 377                 "-varz_avail_fs=",
 378                 "-index_path="+*unpackedPath+"/full.idx",
 379                 "-tls_cert_path="+filepath.Join(*localdcsPath, "cert.pem"),
 380                 "-tls_key_path="+filepath.Join(*localdcsPath, "key.pem"),
 381                 "-listen_address="+*listenIndexBackend,
 382                 "-tls_require_client_auth=false"); err != nil {
 383                 log.Fatal(err.Error())
 384         }
 385
 386         // TODO: check for healthiness
 387
 388         log.Printf("dcs-index-backend running at https://%s\n", *listenIndexBackend)
 389
 390         if err := launchInBackground(
 391                 "dcs-source-backend",
 392                 "-varz_avail_fs=",
 393                 "-unpacked_path="+*unpackedPath,
 394                 "-ranking_data_path="+rankingPath,
 395                 "-tls_cert_path="+filepath.Join(*localdcsPath, "cert.pem"),
 396                 "-tls_key_path="+filepath.Join(*localdcsPath, "key.pem"),
 397                 "-listen_address="+*listenSourceBackend,
 398                 "-tls_require_client_auth=false"); err != nil {
 399                 log.Fatal(err.Error())
 400         }
 401
 402         // TODO: check for healthiness
 403
 404         log.Printf("dcs-source-backend running at https://%s\n", *listenSourceBackend)
 405
 406         if err := launchInBackground(
 407                 "dcs-web",
 408                 "-varz_avail_fs=",
 409                 "-headroom_percentage=0",
 410                 "-template_pattern=cmd/dcs-web/templates/*",
 411                 "-static_path=static/",
 412                 "-source_backends="+*listenSourceBackend,
 413                 "-tls_cert_path="+filepath.Join(*localdcsPath, "cert.pem"),
 414                 "-tls_key_path="+filepath.Join(*localdcsPath, "key.pem"),
 415                 "-listen_address="+*listenWeb,
 416                 "-listen_address_tls="+*listenWebTLS); err != nil {
 417                 log.Fatal(err.Error())
 418         }
 419
 420         listenAddresses := "http://" + *listenWeb
 421         if *listenWebTLS != "" {
 422                 listenAddresses = listenAddresses + " and https://" + *listenWebTLS
 423         }
 424         log.Printf("dcs-web running at %s\n", listenAddresses)
 425 }