1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import print_function
15 import voluptuous
.humanize
17 from mozbuild
.telemetry
import (
18 schema
as build_telemetry_schema
,
22 BUILD_TELEMETRY_URL
= "https://incoming.telemetry.mozilla.org/{endpoint}"
23 SUBMIT_ENDPOINT
= "submit/eng-workflow/build/1/{ping_uuid}"
24 STATUS_ENDPOINT
= "status"
27 def delete_expired_files(directory
, days
=30):
28 """Discards files in a directory older than a specified number
31 now
= datetime
.datetime
.now()
32 for filename
in os
.listdir(directory
):
33 filepath
= os
.path
.join(directory
, filename
)
35 ctime
= os
.path
.getctime(filepath
)
36 then
= datetime
.datetime
.fromtimestamp(ctime
)
38 if (now
- then
) > datetime
.timedelta(days
=days
):
44 def check_edge_server_status(session
):
45 """Returns True if the Telemetry Edge Server
46 is ready to accept data
48 status_url
= BUILD_TELEMETRY_URL
.format(endpoint
=STATUS_ENDPOINT
)
49 response
= session
.get(status_url
)
50 if response
.status_code
!= 200:
55 def send_telemetry_ping(session
, data
, ping_uuid
):
56 """Sends a single build telemetry ping to the
57 edge server, returning the response object
59 resource_url
= SUBMIT_ENDPOINT
.format(ping_uuid
=str(ping_uuid
))
60 url
= BUILD_TELEMETRY_URL
.format(endpoint
=resource_url
)
61 response
= session
.post(url
, json
=data
)
66 def submit_telemetry_data(outgoing
, submitted
):
67 """Sends information about `./mach build` invocations to
68 the Telemetry pipeline
70 with requests
.Session() as session
:
71 # Confirm the server is OK
72 if not check_edge_server_status(session
):
73 logging
.error('Error posting to telemetry: server status is not "200 OK"')
76 for filename
in os
.listdir(outgoing
):
77 path
= os
.path
.join(outgoing
, filename
)
79 if os
.path
.isdir(path
) or not path
.endswith(".json"):
80 logging
.info("skipping item {}".format(path
))
83 ping_uuid
= os
.path
.splitext(filename
)[0] # strip ".json" to get ping UUID
86 with
open(path
, "r") as f
:
89 # Verify the data matches the schema
90 voluptuous
.humanize
.validate_with_humanized_errors(
91 data
, build_telemetry_schema
94 response
= send_telemetry_ping(session
, data
, ping_uuid
)
95 if response
.status_code
!= 200:
96 msg
= "response code {code} sending {uuid} to telemetry: {body}".format(
97 body
=response
.content
,
98 code
=response
.status_code
,
104 # Move from "outgoing" to "submitted"
106 os
.path
.join(outgoing
, filename
), os
.path
.join(submitted
, filename
)
109 logging
.info("successfully posted {} to telemetry".format(ping_uuid
))
111 except ValueError as ve
:
112 # ValueError is thrown if JSON cannot be decoded
113 logging
.exception("exception parsing JSON at %s: %s" % (path
, str(ve
)))
116 except voluptuous
.Error
as e
:
117 # Invalid is thrown if some data does not fit
119 logging
.exception("invalid data found at %s: %s" % (path
, e
.message
))
122 except Exception as e
:
123 logging
.error("exception posting to telemetry " "server: %s" % str(e
))
126 delete_expired_files(submitted
)
131 if __name__
== "__main__":
132 if len(sys
.argv
) != 2:
133 print("usage: python submit_telemetry_data.py <statedir>")
136 statedir
= sys
.argv
[1]
139 outgoing
, submitted
, telemetry_log
= verify_statedir(statedir
)
143 filename
=telemetry_log
,
144 format
="%(asctime)s %(message)s",
148 sys
.exit(submit_telemetry_data(outgoing
, submitted
))
150 except Exception as e
:
151 # Handle and print messages from `statedir` verification