python/google/appengine/tools/queue_xml_parser.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Performs translation of queue.xml to queue.yaml."""
  18
  19 from xml.etree import ElementTree
  20
  21 from google.appengine.tools import xml_parser_utils
  22 from google.appengine.tools.app_engine_config_exception import AppEngineConfigException
  23
  24 PUSH_QUEUE_TAGS = (
  25     'rate', 'bucket-size', 'max-concurrent-requests', 'target')
  26
  27 PUSH_QUEUE_RETRY_PARAMS = (
  28     'task-age-limit',
  29     'min-backoff-seconds',
  30     'max-backoff-seconds',
  31     'max-doublings')
  32
  33 RETRY_PARAMETER_TAGS = ('task-retry-limit',) + PUSH_QUEUE_RETRY_PARAMS
  34
  35 BAD_MODE_ERROR_MESSAGE = (
  36     'Mode, if specified, must be either push or pull'
  37     ' (defaults to push) for queue entries. Bad value'
  38     " '%s' in <queue> entry with name '%s'")
  39
  40 PULL_QUEUE_ERROR_MESSAGE = (
  41     'The element <%s> is not defined for pull '
  42     "queues; bad <queue> entry with name '%s'")
  43 RETRY_PARAM_ERROR_MESSAGE = (
  44     'The element <%s> in <retry-parameters> is not '
  45     "defined for pull queue with name '%s'")
  46
  47
  48 def GetQueueYaml(unused_application, queue_xml_str):
  49   queue_xml = QueueXmlParser().ProcessXml(queue_xml_str)
  50   return queue_xml.ToYaml()
  51
  52
  53 class QueueXmlParser(object):
  54   """Provides logic for walking down XML tree and pulling data."""
  55
  56   def ProcessXml(self, xml_str):
  57     """Parses XML string and returns object representation of relevant info.
  58
  59     Args:
  60       xml_str: The XML string.
  61     Returns:
  62       A QueueXml object containing information about task queue
  63       specifications from the XML.
  64     Raises:
  65       AppEngineConfigException: In case of malformed XML or illegal inputs.
  66     """
  67
  68     try:
  69       self.errors = []
  70       xml_root = ElementTree.fromstring(xml_str)
  71
  72       if xml_parser_utils.GetTag(xml_root) != 'queue-entries':
  73         raise AppEngineConfigException('Root tag must be <queue-entries>')
  74
  75       self.queue_xml = QueueXml()
  76       self.queue_xml.queues = []
  77       self.queue_xml.total_storage_limit = xml_parser_utils.GetChildNodeText(
  78           xml_root, 'total-storage-limit')
  79       for child in xml_parser_utils.GetNodes(xml_root, 'queue'):
  80         self.ProcessQueueNode(child)
  81
  82       if self.errors:
  83         raise AppEngineConfigException('\n'.join(self.errors))
  84
  85       return self.queue_xml
  86
  87     except ElementTree.ParseError:
  88       raise AppEngineConfigException('Bad input -- not valid XML')
  89
  90   def ProcessQueueNode(self, node):
  91     """Processes XML <queue> nodes into Queue objects.
  92
  93     The following information is parsed out:
  94       name
  95       mode: can be either push or pull
  96       retry-parameters:
  97         task-retry-limit
  98     ---- push queues only ----
  99         task-age-limit
 100         min-backoff-seconds
 101         max-back-off-seconds
 102         max-doubling
 103       bucket-size
 104       max-concurrent-requests
 105       rate: how often tasks are processed on this queue.
 106       target: version of application on which tasks on this queue will be
 107         invoked.
 108     ---- pull queues only ----
 109       acl: access control list - lists user and writer email addresses.
 110
 111     Args:
 112       node: Current <queue> XML node being processed.
 113     """
 114
 115     name = xml_parser_utils.GetChildNodeText(node, 'name')
 116     if not name:
 117       self.errors.append('Must specify a name for each <queue> entry')
 118       return
 119
 120
 121     mode = xml_parser_utils.GetChildNodeText(node, 'mode', 'push')
 122
 123     if mode not in ('push', 'pull'):
 124       self.errors.append(BAD_MODE_ERROR_MESSAGE % (mode, name))
 125       return
 126     if mode == 'pull':
 127       queue = PullQueue()
 128       queue.name = name
 129       self._ProcessPullQueueNode(node, queue)
 130     else:
 131       queue = PushQueue()
 132       queue.name = name
 133       self._ProcessPushQueueNode(node, queue)
 134
 135     self.queue_xml.queues.append(queue)
 136
 137   def _ProcessPushQueueNode(self, node, queue):
 138     if xml_parser_utils.GetChild(node, 'acl') is not None:
 139       self.errors.append(
 140           'The element <acl> is not defined for push '
 141           "queues; bad <queue> entry with name '%s'" % queue.name)
 142     for tag in PUSH_QUEUE_TAGS:
 143       field_name = tag.replace('-', '_')
 144       setattr(queue, field_name, xml_parser_utils.GetChildNodeText(node, tag))
 145     self._ProcessRetryParametersNode(node, queue)
 146
 147   def _ProcessPullQueueNode(self, node, queue):
 148     """Populates PullQueue-specific fields from parsed XML."""
 149     for tag in PUSH_QUEUE_TAGS:
 150       if xml_parser_utils.GetChild(node, tag) is not None:
 151         self.errors.append(PULL_QUEUE_ERROR_MESSAGE % (tag, queue.name))
 152
 153     acl_node = xml_parser_utils.GetChild(node, 'acl')
 154
 155     if acl_node is not None:
 156       queue.acl = Acl()
 157       queue.acl.user_emails = [
 158           sub_node.text for sub_node in
 159           xml_parser_utils.GetNodes(acl_node, 'user-email')]
 160       queue.acl.writer_emails = [
 161           sub_node.text for sub_node in
 162           xml_parser_utils.GetNodes(acl_node, 'writer-email')]
 163     else:
 164       queue.acl = None
 165
 166     self._ProcessRetryParametersNode(node, queue)
 167
 168   def _ProcessRetryParametersNode(self, node, queue):
 169     """Pulls information out of <retry-parameters> node."""
 170     retry_parameters_node = xml_parser_utils.GetChild(
 171         node, 'retry-parameters')
 172     if retry_parameters_node is None:
 173       queue.retry_parameters = None
 174       return
 175     retry_parameters = RetryParameters()
 176     queue.retry_parameters = retry_parameters
 177     retry_parameters.task_retry_limit = xml_parser_utils.GetChildNodeText(
 178         retry_parameters_node, 'task-retry-limit')
 179
 180     for tag in PUSH_QUEUE_RETRY_PARAMS:
 181
 182       if xml_parser_utils.GetChild(retry_parameters_node, tag) is not None:
 183         if isinstance(queue, PullQueue):
 184           self.errors.append(RETRY_PARAM_ERROR_MESSAGE % (tag, queue.name))
 185         else:
 186           setattr(
 187               retry_parameters,
 188               tag.replace('-', '_'),
 189               xml_parser_utils.GetChildNodeText(retry_parameters_node, tag))
 190
 191
 192 class QueueXml(object):
 193
 194   def __init__(self):
 195     self.queues = []
 196     self.total_storage_limit = None
 197
 198   def ToYaml(self):
 199     statements = []
 200     if self.total_storage_limit:
 201       statements.append('total_storage_limit: %s\n' % self.total_storage_limit)
 202     statements.append('queue:')
 203     for queue in self.queues:
 204       statements += queue.GetYamlStatementsList()
 205
 206     return '\n'.join(statements) + '\n'
 207
 208
 209 class Queue(object):
 210
 211   def GetYamlStatementsList(self):
 212     statements = ['- name: %s' % self.name]
 213     statements += self.GetAdditionalYamlStatementsList()
 214
 215     if self.retry_parameters:
 216       statements += self.retry_parameters.GetYamlStatementsList()
 217
 218     return statements
 219
 220
 221 class PushQueue(Queue):
 222
 223   def GetAdditionalYamlStatementsList(self):
 224     statements = ['  mode: push']
 225     fields = (tag.replace('-', '_') for tag in PUSH_QUEUE_TAGS)
 226     for field in fields:
 227       field_value = getattr(self, field)
 228       if field_value:
 229         statements.append('  %s: %s' % (field, field_value))
 230
 231     return statements
 232
 233
 234 class PullQueue(Queue):
 235
 236   def GetAdditionalYamlStatementsList(self):
 237     statements = ['  mode: pull']
 238     if self.acl:
 239       statements += self.acl.GetYamlStatementsList()
 240     return statements
 241
 242
 243 class Acl(object):
 244
 245   def GetYamlStatementsList(self):
 246     statements = ['  acl:']
 247     statements += ['  - user_email: %s' % user_email for user_email in
 248                    self.user_emails]
 249     statements += ['  - writer_email: %s' % writer_email for writer_email in
 250                    self.writer_emails]
 251     return statements
 252
 253
 254 class RetryParameters(object):
 255
 256   def GetYamlStatementsList(self):
 257     statements = ['  retry_parameters:']
 258     field_names = (tag.replace('-', '_') for tag in RETRY_PARAMETER_TAGS)
 259     for field in field_names:
 260       field_value = getattr(self, field, None)
 261       if field_value:
 262         statements.append('    %s: %s' % (field, field_value))
 263     return statements