class/System.Core/System.Linq/ParallelExecuter.cs

   1 #if NET_4_0
   2 //
   3 // ParallelExecuter.cs
   4 //
   5 // Author:
   6 //       Jérémie "Garuma" Laval <jeremie.laval@gmail.com>
   7 //
   8 // Copyright (c) 2010 Jérémie "Garuma" Laval
   9 //
  10 // Permission is hereby granted, free of charge, to any person obtaining a copy
  11 // of this software and associated documentation files (the "Software"), to deal
  12 // in the Software without restriction, including without limitation the rights
  13 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 // copies of the Software, and to permit persons to whom the Software is
  15 // furnished to do so, subject to the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be included in
  18 // all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  23 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  24 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  25 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  26 // THE SOFTWARE.
  27
  28 using System;
  29 using System.Threading;
  30 using System.Threading.Tasks;
  31 using System.Collections;
  32 using System.Collections.Generic;
  33 using System.Collections.Concurrent;
  34
  35 namespace System.Linq
  36 {
  37         // TODO: Refactory each of the Process method into one big entity
  38         // Check CancellationToken.Canceled parameter in the Task's action body too
  39         internal static class ParallelExecuter
  40         {
  41                 internal static QueryOptions CheckQuery<T> (QueryBaseNode<T> startingNode)
  42                 {
  43                         return CheckQuery<T> (startingNode, false);
  44                 }
  45
  46                 internal static QueryOptions CheckQuery<T> (QueryBaseNode<T> startingNode, bool blocking)
  47                 {
  48                         return CheckQuery (startingNode, GetBestWorkerNumber (blocking));
  49                 }
  50
  51                 internal static QueryOptions CheckQuery<T> (QueryBaseNode<T> startingNode, int partitionCount)
  52                 {
  53                         QueryCheckerVisitor visitor = new QueryCheckerVisitor (partitionCount);
  54                         startingNode.Visit (visitor);
  55
  56                         return visitor.Options;
  57                 }
  58
  59                 // QueryOptions.ImplementerToken = QueryOptions.ImplementerToken.Chain (myOperatorSource);
  60                 internal static CancellationToken Chain (this CancellationToken self, CancellationTokenSource other)
  61                 {
  62                         CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource (self, other.Token);
  63                         return linked.Token;
  64                 }
  65
  66                 internal static int GetBestWorkerNumber ()
  67                 {
  68                         return GetBestWorkerNumber (false);
  69                 }
  70
  71                 internal static int GetBestWorkerNumber (bool blocking)
  72                 {
  73                         return blocking ? Environment.ProcessorCount + 1 : Environment.ProcessorCount;
  74                 }
  75
  76                 internal static Task[] Process<TSource, TElement> (QueryBaseNode<TSource> node, Action<TElement> call,
  77                                                                    Func<QueryBaseNode<TSource>, QueryOptions, IList<IEnumerable<TElement>>> acquisitionFunc,
  78                                                                    QueryOptions options)
  79                 {
  80                         return Process<TSource, TElement> (node, call, acquisitionFunc, null, options);
  81                 }
  82
  83                 internal static Task[] Process<TSource, TElement> (QueryBaseNode<TSource> node, Action<TElement> call,
  84                                                                    Func<QueryBaseNode<TSource>, QueryOptions, IList<IEnumerable<TElement>>> acquisitionFunc,
  85                                                                    Action endAction,
  86                                                                    QueryOptions options)
  87                 {
  88                         return Process<TSource, TElement> (node, (e, i) => call (e), acquisitionFunc, (i) => endAction (), options);
  89                 }
  90
  91                 internal static Task[] Process<TSource, TElement> (QueryBaseNode<TSource> node, Action<TElement, int> call,
  92                                                                    Func<QueryBaseNode<TSource>, QueryOptions, IList<IEnumerable<TElement>>> acquisitionFunc,
  93                                                                    Action<int> endAction,
  94                                                                    QueryOptions options)
  95                 {
  96                         IList<IEnumerable<TElement>> enumerables = acquisitionFunc (node, options);
  97
  98                         Task[] tasks = new Task[enumerables.Count];
  99
 100                         for (int i = 0; i < tasks.Length; i++) {
 101                                 int index = i;
 102                                 tasks[i] = Task.Factory.StartNew (() => {
 103                                         foreach (TElement item in enumerables[index]) {
 104                                                 // This is from specific operators
 105                                                 if (options.ImplementerToken.IsCancellationRequested)
 106                                                         break;
 107                                                 if (options.Token.IsCancellationRequested)
 108                                                         throw new OperationCanceledException (options.Token);
 109
 110                                                 call (item, index);
 111                                         }
 112                                         if (endAction != null)
 113                                                 endAction (index);
 114                                   }, options.Token);
 115                         }
 116
 117                         return tasks;
 118                 }
 119
 120                 internal static void ProcessAndBlock<T> (QueryBaseNode<T> node, Action<T> call)
 121                 {
 122                         QueryOptions options = CheckQuery (node, true);
 123
 124                         Task[] tasks = Process (node, call, (n, o) => n.GetEnumerables (o), options);
 125                         Task.WaitAll (tasks, options.Token);
 126                 }
 127
 128                 internal static Action ProcessAndCallback<T> (QueryBaseNode<T> node, Action<T> call,
 129                                                               Action callback, QueryOptions options)
 130                 {
 131                         Task[] tasks = Process (node, call, (n, o) => n.GetEnumerables (o), options);
 132                         if (callback != null)
 133                                 Task.Factory.ContinueWhenAll (tasks,  (_) => callback ());
 134
 135                         return () => Task.WaitAll (tasks, options.Token);
 136                 }
 137
 138                 internal static Action ProcessAndCallback<T> (QueryBaseNode<T> node, Action<KeyValuePair<long, T>, int> call,
 139                                                               Action callback, QueryOptions options)
 140                 {
 141                         return ProcessAndCallback<T> (node, call, null, callback, options);
 142                 }
 143
 144                 internal static Action ProcessAndCallback<T> (QueryBaseNode<T> node, Action<KeyValuePair<long, T>, int> call,
 145                                                               Action<int> endAction,
 146                                                               Action callback, QueryOptions options)
 147                 {
 148                         Task[] tasks = Process (node, call, (n, o) => n.GetOrderedEnumerables (o), endAction, options);
 149                         if (callback != null)
 150                                 Task.Factory.ContinueWhenAll (tasks,  (_) => callback ());
 151
 152                         return () => Task.WaitAll (tasks, options.Token);
 153                 }
 154
 155                 internal static void ProcessAndAggregate<T, U> (QueryBaseNode<T> node,
 156                                                     Func<U> seedFunc,
 157                                                     Func<U, T, U> localCall,
 158                                                     Action<IList<U>> call)
 159                 {
 160                         QueryOptions options = CheckQuery (node, true);
 161
 162                         IList<IEnumerable<T>> enumerables = node.GetEnumerables (options);
 163                         U[] locals = new U[enumerables.Count];
 164                         Task[] tasks = new Task[enumerables.Count];
 165
 166                         bool init = false;
 167                         if (seedFunc != null) {
 168                                 for (int i = 0; i < locals.Length; i++)
 169                                         locals[i] = seedFunc ();
 170                                 init = true;
 171                         }
 172
 173                         for (int i = 0; i < tasks.Length; i++) {
 174                                 int index = i;
 175                                 tasks[i] = Task.Factory.StartNew (() => {
 176                                         foreach (T item in enumerables[index]) {
 177                                                 // This is from specific operators
 178                                                 if (options.ImplementerToken.IsCancellationRequested)
 179                                                         break;
 180                                                 if (options.Token.IsCancellationRequested)
 181                                                         throw new OperationCanceledException (options.Token);
 182
 183                                                 if (!init) {
 184                                                         init = true;
 185                                                         // HACK: TODO: omfwtfitsomuchsucks
 186                                                         locals[index] = (U)(object)item;
 187                                                         continue;
 188                                                 }
 189
 190                                                 U acc = locals[index];
 191                                                 locals[index] = localCall (acc, item);
 192                                         }
 193                                 }, options.Token);
 194                         }
 195
 196                         Task.WaitAll (tasks, options.Token);
 197
 198                         if (call != null)
 199                                 call (locals);
 200                 }
 201         }
 202 }
 203 #endif