python/google/appengine/ext/mapreduce/parameters.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Parameters to control Mapreduce."""
  18
  19 __all__ = ["CONFIG_NAMESPACE",
  20            "config"]
  21
  22 from google.appengine.api import lib_config
  23
  24 CONFIG_NAMESPACE = "mapreduce"
  25
  26
  27 class _ConfigDefaults(object):
  28   """Default configs.
  29
  30   Do not change parameters whose names begin with _.
  31
  32   SHARD_MAX_ATTEMPTS: Max attempts to execute a shard before giving up.
  33
  34   TASK_MAX_ATTEMPTS: Max attempts to execute a task before dropping it. Task
  35     is any taskqueue task created by MR framework. A task is dropped
  36     when its X-AppEngine-TaskExecutionCount is bigger than this number.
  37     Dropping a task will cause abort on the entire MR job.
  38
  39   TASK_MAX_DATA_PROCESSING_ATTEMPTS:
  40     Max times to execute a task when previous task attempts failed during
  41     data processing stage. An MR work task has three major stages:
  42     initial setup, data processing, and final checkpoint.
  43     Setup stage should be allowed to be retried more times than data processing
  44     stage: setup failures are caused by unavailable GAE services while
  45     data processing failures are mostly due to user function error out on
  46     certain input data. Thus, set TASK_MAX_ATTEMPTS higher than this parameter.
  47
  48   QUEUE_NAME: Default queue for MR.
  49
  50   SHARD_COUNT: Default shard count.
  51
  52   PROCESSING_RATE_PER_SEC: Default rate of processed entities per second.
  53
  54   BASE_PATH : Base path of mapreduce and pipeline handlers.
  55   """
  56
  57   SHARD_MAX_ATTEMPTS = 4
  58
  59
  60   TASK_MAX_ATTEMPTS = 31
  61
  62   TASK_MAX_DATA_PROCESSING_ATTEMPTS = 11
  63
  64   QUEUE_NAME = "default"
  65
  66   SHARD_COUNT = 8
  67
  68
  69   PROCESSING_RATE_PER_SEC = 1000000
  70
  71
  72   BASE_PATH = "/_ah/mapreduce"
  73
  74
  75
  76
  77   _SLICE_DURATION_SEC = 15
  78
  79
  80   _LEASE_GRACE_PERIOD = 1
  81
  82
  83   _REQUEST_EVENTUAL_TIMEOUT = 10 * 60 + 30
  84
  85
  86   _CONTROLLER_PERIOD_SEC = 2
  87
  88
  89 config = lib_config.register(CONFIG_NAMESPACE, _ConfigDefaults.__dict__)
  90
  91
  92
  93
  94
  95 _DEFAULT_PIPELINE_BASE_PATH = config.BASE_PATH + "/pipeline"
  96
  97 _GCS_URLFETCH_TIMEOUT_SEC = 30