cmd/source-backend/source-backend.go

   1 // vim:ts=4:sw=4:noexpandtab
   2 package main
   3
   4 import (
   5         "encoding/json"
   6         "flag"
   7         "fmt"
   8         // This is a forked version of codesearch/regexp which returns the results
   9         // in a structure instead of printing to stdout/stderr directly.
  10         "github.com/Debian/dcs/cmd/dcs-web/varz"
  11         "github.com/Debian/dcs/ranking"
  12         "github.com/Debian/dcs/regexp"
  13         "io"
  14         "log"
  15         "net/http"
  16         "os"
  17         "path"
  18         "strconv"
  19         "strings"
  20 )
  21
  22 var (
  23         listenAddress = flag.String("listen_address", ":28082", "listen address ([host]:port)")
  24         unpackedPath  = flag.String("unpacked_path",
  25                 "/dcs-ssd/unpacked/",
  26                 "Path to the unpacked sources")
  27 )
  28
  29 type SourceReply struct {
  30         // The number of the last used filename, needed for pagination
  31         LastUsedFilename int
  32
  33         AllMatches []regexp.Match
  34 }
  35
  36 // Serves a single file for displaying it in /show
  37 func File(w http.ResponseWriter, r *http.Request) {
  38         r.ParseForm()
  39         filename := r.Form.Get("file")
  40
  41         log.Printf("requested filename *%s*\n", filename)
  42         // path.Join calls path.Clean so we get the shortest path without any "..".
  43         absPath := path.Join(*unpackedPath, filename)
  44         log.Printf("clean, absolute path is *%s*\n", absPath)
  45         if !strings.HasPrefix(absPath, *unpackedPath) {
  46                 http.Error(w, "Path traversal is bad, mhkay?", http.StatusForbidden)
  47                 return
  48         }
  49
  50         file, err := os.Open(absPath)
  51         if err != nil {
  52                 http.Error(w, fmt.Sprintf(`Could not open file "%s"`, absPath), http.StatusNotFound)
  53                 return
  54         }
  55         defer file.Close()
  56
  57         io.Copy(w, file)
  58 }
  59
  60 func Source(w http.ResponseWriter, r *http.Request) {
  61         r.ParseForm()
  62         textQuery := r.Form.Get("q")
  63         limit, err := strconv.ParseInt(r.Form.Get("limit"), 10, 0)
  64         if err != nil {
  65                 log.Printf("%s\n", err)
  66                 return
  67         }
  68         filenames := r.Form["filename"]
  69         re, err := regexp.Compile(textQuery)
  70         if err != nil {
  71                 log.Printf("%s\n", err)
  72                 return
  73         }
  74
  75         log.Printf("query: text = %s, regexp = %s\n", textQuery, re)
  76
  77         rankingopts := ranking.RankingOptsFromQuery(r.URL.Query())
  78
  79         querystr := ranking.NewQueryStr(textQuery)
  80
  81         // Create one Goroutine per filename, which means all IO will be done in
  82         // parallel.
  83         output := make(chan []regexp.Match)
  84         for _, filename := range filenames {
  85                 go func(filename string) {
  86                         // TODO: figure out how to safely clone a dcs/regexp
  87                         re, err := regexp.Compile(textQuery)
  88                         if err != nil {
  89                                 log.Printf("%s\n", err)
  90                                 return
  91                         }
  92
  93                         grep := regexp.Grep{
  94                                 Regexp: re,
  95                                 Stdout: os.Stdout,
  96                                 Stderr: os.Stderr,
  97                         }
  98
  99                         output <- grep.File(path.Join(*unpackedPath, filename))
 100                 }(filename)
 101         }
 102
 103         fmt.Printf("done, now getting the results\n")
 104
 105         // TODO: also limit the number of matches per source-package, not only per file
 106         var reply SourceReply
 107         for idx, filename := range filenames {
 108                 fmt.Printf("…in %s\n", filename)
 109                 matches := <-output
 110                 for idx, match := range matches {
 111                         if limit > 0 && idx == 5 {
 112                                 // TODO: we somehow need to signal that there are more results
 113                                 // (if there are more), so that the user can expand this.
 114                                 break
 115                         }
 116                         fmt.Printf("match: %s\n", match)
 117                         match.Ranking = ranking.PostRank(rankingopts, &match, &querystr)
 118                         match.Path = match.Path[len(*unpackedPath):]
 119                         reply.AllMatches = append(reply.AllMatches, match)
 120                 }
 121                 if limit > 0 && int64(len(reply.AllMatches)) >= limit {
 122                         reply.LastUsedFilename = idx
 123                         break
 124                 }
 125         }
 126         jsonFiles, err := json.Marshal(&reply)
 127         if err != nil {
 128                 log.Printf("%s\n", err)
 129                 return
 130         }
 131         _, err = w.Write(jsonFiles)
 132         if err != nil {
 133                 log.Printf("%s\n", err)
 134                 return
 135         }
 136
 137 }
 138
 139 func main() {
 140         flag.Parse()
 141         fmt.Println("Debian Code Search source-backend")
 142
 143         http.HandleFunc("/source", Source)
 144         http.HandleFunc("/file", File)
 145         http.HandleFunc("/varz", varz.Varz)
 146         log.Fatal(http.ListenAndServe(*listenAddress, nil))
 147 }