build/submit_telemetry_data.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import print_function
   6
   7 import datetime
   8 import json
   9 import logging
  10 import os
  11 import sys
  12
  13 import requests
  14 import voluptuous
  15 import voluptuous.humanize
  16
  17 from mozbuild.telemetry import (
  18     schema as build_telemetry_schema,
  19     verify_statedir,
  20 )
  21
  22 BUILD_TELEMETRY_URL = 'https://incoming.telemetry.mozilla.org/{endpoint}'
  23 SUBMIT_ENDPOINT = 'submit/eng-workflow/build/1/{ping_uuid}'
  24 STATUS_ENDPOINT = 'status'
  25
  26
  27 def delete_expired_files(directory, days=30):
  28     '''Discards files in a directory older than a specified number
  29     of days
  30     '''
  31     now = datetime.datetime.now()
  32     for filename in os.listdir(directory):
  33         filepath = os.path.join(directory, filename)
  34
  35         ctime = os.path.getctime(filepath)
  36         then = datetime.datetime.fromtimestamp(ctime)
  37
  38         if (now - then) > datetime.timedelta(days=days):
  39             os.remove(filepath)
  40
  41     return
  42
  43
  44 def check_edge_server_status(session):
  45     '''Returns True if the Telemetry Edge Server
  46     is ready to accept data
  47     '''
  48     status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
  49     response = session.get(status_url)
  50     if response.status_code != 200:
  51         return False
  52     return True
  53
  54
  55 def send_telemetry_ping(session, data, ping_uuid):
  56     '''Sends a single build telemetry ping to the
  57     edge server, returning the response object
  58     '''
  59     resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
  60     url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
  61     response = session.post(url, json=data)
  62
  63     return response
  64
  65
  66 def submit_telemetry_data(outgoing, submitted):
  67     '''Sends information about `./mach build` invocations to
  68     the Telemetry pipeline
  69     '''
  70     with requests.Session() as session:
  71         # Confirm the server is OK
  72         if not check_edge_server_status(session):
  73             logging.error('Error posting to telemetry: server status is not "200 OK"')
  74             return 1
  75
  76         for filename in os.listdir(outgoing):
  77             path = os.path.join(outgoing, filename)
  78
  79             if os.path.isdir(path) or not path.endswith('.json'):
  80                 logging.info('skipping item {}'.format(path))
  81                 continue
  82
  83             ping_uuid = os.path.splitext(filename)[0]  # strip ".json" to get ping UUID
  84
  85             try:
  86                 with open(path, 'r') as f:
  87                     data = json.load(f)
  88
  89                 # Verify the data matches the schema
  90                 voluptuous.humanize.validate_with_humanized_errors(
  91                     data, build_telemetry_schema
  92                 )
  93
  94                 response = send_telemetry_ping(session, data, ping_uuid)
  95                 if response.status_code != 200:
  96                     msg = 'response code {code} sending {uuid} to telemetry: {body}'.format(
  97                         body=response.content,
  98                         code=response.status_code,
  99                         uuid=ping_uuid,
 100                     )
 101                     logging.error(msg)
 102                     continue
 103
 104                 # Move from "outgoing" to "submitted"
 105                 os.rename(os.path.join(outgoing, filename),
 106                           os.path.join(submitted, filename))
 107
 108                 logging.info('successfully posted {} to telemetry'.format(ping_uuid))
 109
 110             except ValueError as ve:
 111                 # ValueError is thrown if JSON cannot be decoded
 112                 logging.exception('exception parsing JSON at %s: %s'
 113                                   % (path, str(ve)))
 114                 os.remove(path)
 115
 116             except voluptuous.Error as e:
 117                 # Invalid is thrown if some data does not fit
 118                 # the correct Schema
 119                 logging.exception('invalid data found at %s: %s'
 120                                   % (path, e.message))
 121                 os.remove(path)
 122
 123             except Exception as e:
 124                 logging.error('exception posting to telemetry '
 125                               'server: %s' % str(e))
 126                 break
 127
 128     delete_expired_files(submitted)
 129
 130     return 0
 131
 132
 133 if __name__ == '__main__':
 134     if len(sys.argv) != 2:
 135         print('usage: python submit_telemetry_data.py <statedir>')
 136         sys.exit(1)
 137
 138     statedir = sys.argv[1]
 139
 140     try:
 141         outgoing, submitted, telemetry_log = verify_statedir(statedir)
 142
 143         # Configure logging
 144         logging.basicConfig(filename=telemetry_log,
 145                             format='%(asctime)s %(message)s',
 146                             level=logging.DEBUG)
 147
 148         sys.exit(submit_telemetry_data(outgoing, submitted))
 149
 150     except Exception as e:
 151         # Handle and print messages from `statedir` verification
 152         print(e.message)
 153         sys.exit(1)