cmd/dcs-compute-ranking/popcon.go

   1 package main
   2
   3 import (
   4         "bufio"
   5         "compress/gzip"
   6         "fmt"
   7         "log"
   8         "net/http"
   9         "regexp"
  10         "strconv"
  11         "strings"
  12
  13         "github.com/stapelberg/godebiancontrol"
  14 )
  15
  16 var (
  17         asciiMatch = regexp.MustCompile(`^(?P<package>[A-Za-z0-9-.+_]+)(:(?P<arch>[a-z0-9]+))?$`)
  18 )
  19
  20 func popconInstallations(binaryPackages []godebiancontrol.Paragraph) (map[string]float32, error) {
  21         binaryToSource := make(map[string]string)
  22         for _, pkg := range binaryPackages {
  23                 source, ok := pkg["Source"]
  24                 if !ok {
  25                         source = pkg["Package"]
  26                 }
  27                 idx := strings.Index(source, " ")
  28                 if idx > -1 {
  29                         source = source[:idx]
  30                 }
  31                 binaryToSource[pkg["Package"]] = source
  32         }
  33
  34         installations := make(map[string]float32)
  35
  36         // Modeled after UDD’s popcon_gatherer.py:
  37         // https://anonscm.debian.org/cgit/collab-qa/udd.git/tree/udd/popcon_gatherer.py?id=9db1e97eff32691f4df03d1b9ee8a9290a91fc7a
  38         url := "http://popcon.debian.org/all-popcon-results.txt.gz"
  39         resp, err := http.Get(url)
  40         if err != nil {
  41                 return nil, err
  42         }
  43         if resp.StatusCode != http.StatusOK {
  44                 return nil, fmt.Errorf("HTTP Status %s for URL %q, expected OK", resp.Status, url)
  45         }
  46         defer resp.Body.Close()
  47         reader, err := gzip.NewReader(resp.Body)
  48         if err != nil {
  49                 return nil, err
  50         }
  51         scanner := bufio.NewScanner(reader)
  52         for scanner.Scan() {
  53                 line := scanner.Text()
  54                 if !strings.HasPrefix(line, "Package: ") {
  55                         continue
  56                 }
  57                 // line is e.g. “Package: 0ad                              269  1380   121     4”
  58                 fields := strings.Fields(line)
  59                 // 2016/03/28 16:30:58 fields = ["Package:" "zyne" "16" "14" "3" "0"]
  60                 if len(fields) != 6 {
  61                         log.Printf("Skipping line %q: expected %d fields, got %d\n", fields, 6, len(fields))
  62                         continue
  63                 }
  64                 var insts int64
  65                 for _, field := range fields[2:] {
  66                         converted, err := strconv.ParseInt(field, 0, 64)
  67                         if err != nil {
  68                                 log.Printf("Could not convert field %q to integer: %v", field, err)
  69                                 continue
  70                         }
  71                         insts += converted
  72                 }
  73                 matches := asciiMatch.FindStringSubmatch(fields[1])
  74                 if len(matches) == 0 {
  75                         if *verbose {
  76                                 fmt.Printf("%q is not a valid package name, skipping\n", fields[1])
  77                         }
  78                         continue
  79                 }
  80                 binaryPackage := matches[1]
  81
  82                 sourcePackage, ok := binaryToSource[binaryPackage]
  83                 if !ok {
  84                         if *verbose {
  85                                 fmt.Printf("Could not find package %q in binary→source mapping, skipping\n", binaryPackage)
  86                         }
  87                         continue
  88                 }
  89                 installations[sourcePackage] += float32(insts)
  90         }
  91         return installations, scanner.Err()
  92 }