build/submit_telemetry_data.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import print_function
   6
   7 import datetime
   8 import json
   9 import logging
  10 import os
  11 import sys
  12
  13 import requests
  14 import voluptuous
  15 import voluptuous.humanize
  16
  17 from mozbuild.telemetry import (
  18     schema as build_telemetry_schema,
  19     verify_statedir,
  20 )
  21
  22 BUILD_TELEMETRY_URL = "https://incoming.telemetry.mozilla.org/{endpoint}"
  23 SUBMIT_ENDPOINT = "submit/eng-workflow/build/1/{ping_uuid}"
  24 STATUS_ENDPOINT = "status"
  25
  26
  27 def delete_expired_files(directory, days=30):
  28     """Discards files in a directory older than a specified number
  29     of days
  30     """
  31     now = datetime.datetime.now()
  32     for filename in os.listdir(directory):
  33         filepath = os.path.join(directory, filename)
  34
  35         ctime = os.path.getctime(filepath)
  36         then = datetime.datetime.fromtimestamp(ctime)
  37
  38         if (now - then) > datetime.timedelta(days=days):
  39             os.remove(filepath)
  40
  41     return
  42
  43
  44 def check_edge_server_status(session):
  45     """Returns True if the Telemetry Edge Server
  46     is ready to accept data
  47     """
  48     status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
  49     response = session.get(status_url)
  50     if response.status_code != 200:
  51         return False
  52     return True
  53
  54
  55 def send_telemetry_ping(session, data, ping_uuid):
  56     """Sends a single build telemetry ping to the
  57     edge server, returning the response object
  58     """
  59     resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
  60     url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
  61     response = session.post(url, json=data)
  62
  63     return response
  64
  65
  66 def submit_telemetry_data(outgoing, submitted):
  67     """Sends information about `./mach build` invocations to
  68     the Telemetry pipeline
  69     """
  70     with requests.Session() as session:
  71         # Confirm the server is OK
  72         if not check_edge_server_status(session):
  73             logging.error('Error posting to telemetry: server status is not "200 OK"')
  74             return 1
  75
  76         for filename in os.listdir(outgoing):
  77             path = os.path.join(outgoing, filename)
  78
  79             if os.path.isdir(path) or not path.endswith(".json"):
  80                 logging.info("skipping item {}".format(path))
  81                 continue
  82
  83             ping_uuid = os.path.splitext(filename)[0]  # strip ".json" to get ping UUID
  84
  85             try:
  86                 with open(path, "r") as f:
  87                     data = json.load(f)
  88
  89                 # Verify the data matches the schema
  90                 voluptuous.humanize.validate_with_humanized_errors(
  91                     data, build_telemetry_schema
  92                 )
  93
  94                 response = send_telemetry_ping(session, data, ping_uuid)
  95                 if response.status_code != 200:
  96                     msg = "response code {code} sending {uuid} to telemetry: {body}".format(
  97                         body=response.content,
  98                         code=response.status_code,
  99                         uuid=ping_uuid,
 100                     )
 101                     logging.error(msg)
 102                     continue
 103
 104                 # Move from "outgoing" to "submitted"
 105                 os.rename(
 106                     os.path.join(outgoing, filename), os.path.join(submitted, filename)
 107                 )
 108
 109                 logging.info("successfully posted {} to telemetry".format(ping_uuid))
 110
 111             except ValueError as ve:
 112                 # ValueError is thrown if JSON cannot be decoded
 113                 logging.exception("exception parsing JSON at %s: %s" % (path, str(ve)))
 114                 os.remove(path)
 115
 116             except voluptuous.Error as e:
 117                 # Invalid is thrown if some data does not fit
 118                 # the correct Schema
 119                 logging.exception("invalid data found at %s: %s" % (path, e.message))
 120                 os.remove(path)
 121
 122             except Exception as e:
 123                 logging.error("exception posting to telemetry " "server: %s" % str(e))
 124                 break
 125
 126     delete_expired_files(submitted)
 127
 128     return 0
 129
 130
 131 if __name__ == "__main__":
 132     if len(sys.argv) != 2:
 133         print("usage: python submit_telemetry_data.py <statedir>")
 134         sys.exit(1)
 135
 136     statedir = sys.argv[1]
 137
 138     try:
 139         outgoing, submitted, telemetry_log = verify_statedir(statedir)
 140
 141         # Configure logging
 142         logging.basicConfig(
 143             filename=telemetry_log,
 144             format="%(asctime)s %(message)s",
 145             level=logging.DEBUG,
 146         )
 147
 148         sys.exit(submit_telemetry_data(outgoing, submitted))
 149
 150     except Exception as e:
 151         # Handle and print messages from `statedir` verification
 152         print(e.message)
 153         sys.exit(1)