From 7e0f1d69fa805fa6e0fe4c1b3db7661af19c0119 Mon Sep 17 00:00:00 2001 From: milde Date: Mon, 31 Aug 2009 10:56:44 +0000 Subject: [PATCH] no need to keep text and code source examples in SVN git-svn-id: http://svn.berlios.de/svnroot/repos/pylit/trunk@111 fb71aa59-6827-0410-b536-ee2229a4f8e3 --- rstdocs/examples/99bottles.py | 114 ---- rstdocs/examples/iterqueue.py | 1003 ------------------------------ rstdocs/examples/iterqueue_speed_test.py | 100 --- rstdocs/examples/iterqueue_test.py | 207 ------ rstdocs/examples/simplestates.py | 198 ------ rstdocs/examples/simplestates_test.py | 642 ------------------- rstdocs/examples/testfile_literate.py | 161 ----- rstdocs/examples/testmod_literate.py | 154 ----- 8 files changed, 2579 deletions(-) delete mode 100644 rstdocs/examples/99bottles.py delete mode 100644 rstdocs/examples/iterqueue.py delete mode 100644 rstdocs/examples/iterqueue_speed_test.py delete mode 100644 rstdocs/examples/iterqueue_test.py delete mode 100644 rstdocs/examples/simplestates.py delete mode 100644 rstdocs/examples/simplestates_test.py delete mode 100644 rstdocs/examples/testfile_literate.py delete mode 100644 rstdocs/examples/testmod_literate.py diff --git a/rstdocs/examples/99bottles.py b/rstdocs/examples/99bottles.py deleted file mode 100644 index 8fdb69b..0000000 --- a/rstdocs/examples/99bottles.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -# :Copyright: 2007 Riccardo Murri, Guenter Milde. -# Released under the terms of the GNU General Public License -# (v. 2 or later) - -# 99bottles.py -# ============ -# Introductory Example to Literate Programming -# ++++++++++++++++++++++++++++++++++++++++++++ -# -# Print the famous `99 bottles of beer` song lyrics -# -# -# This is used as an introductory example to literate programming -# in the `LiteratePrograms.org Wiki`_. -# -# The lyrics -# ---------- -# -# We take the lyrics from the Wikipedia_, which says in the -# `99 bottles of beer`_ entry: -# -# The verse format is very formulaic, and can be expressed as follows: -# -# From 99 down to 1:: - -verse_template = """ - bottles of beer on the wall - bottles of beer! -Take one down, pass it around - bottles of beer on the wall!""" - -# There is much variation in the final verse. One common final verse (which -# could potentially cause an infinite-loop motif) is:: - -final_verse = """ -No bottles of beer on the wall! -No bottles of beer! -Go to the store and buy some more -99 bottles of beer on the wall!""" - -# The Python program -# ------------------ -# -# There are a countless number of ways to implement a program that prints the -# whole song in Python. The following examples uses a `for` loop and the -# `replace` method of string objects. -# -# Basic version -# -# Count down from 99 to 1 and print the verses:: - -def print_verses_1(start_number=99): - for number in xrange(start_number, 0, -1): - verse = verse_template.replace("", str(number)) - print verse.replace("", str(number-1 or "No")) - -# Consider the singular case -# -# There is one problem left, we should check whether to print 'bottles' or -# 'bottle'. -# -# An improved version will replace the "bottles" with a construct that -# takes into account the actual number of bottles:: - -def print_verses_2(start_number=99): - for number in xrange(start_number, 0, -1): - verse = verse_template.replace("", str(number)) - verse = verse.replace("bottles", "bottle" + plural_suffix(number)) - print verse.replace("", str(number-1 or "No")) - -# where an auxiliary function returns the matching suffix (or not):: - -def plural_suffix(number): - if number != 1: - return "s" - else: - return "" - -# Still, the last line come out wrong, as here we have bottles. To -# treat this case we either could split the last line and treat it differently, -# or use a modified template as e.g. :: - -verse_template_2 = """ - bottle of beer on the wall - bottle of beer! -Take one down, pass it around - bottle of beer on the wall!""" - -# together with:: - -def print_verses_3(start_number=99): - for number in xrange(start_number, 0, -1): - verse = verse_template_2.replace("", str(number)) - verse = verse.replace("", plural_suffix(number), 2) - verse = verse.replace("", plural_suffix(number-1), 1) - print verse.replace("", str(number-1 or "No")) - - -# Command line use -# ---------------- -# -# Print the lyrics if this script is called from the command line:: - -if __name__ == "__main__": - print_verses_3() - print final_verse - - -# .. _Wikipedia: http://en.wikipedia.org -# .. _99 bottles of beer: http://en.wikipedia.org/wiki/99_Bottles_of_Beer -# .. _LiteratePrograms.org Wiki: -# http://en.literateprograms.org/LiteratePrograms:Welcome diff --git a/rstdocs/examples/iterqueue.py b/rstdocs/examples/iterqueue.py deleted file mode 100644 index 5d860b6..0000000 --- a/rstdocs/examples/iterqueue.py +++ /dev/null @@ -1,1003 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf8 -*- - -# ************************************ -# Extending Iterators for use as Queue -# ************************************ -# -# :Version: 0.2 -# :Date: 2007-01-15 -# :Copyright: 2005, 2007 Guenter Milde. -# Released under the terms of the GNU General Public License -# (v. 2 or later) -# :Changelog: 2005-06-29 Initial version -# 2007-01-07 literate version, more examples -# :Abstract: There are many variants of "rich iterators" with varying -# efficiency, conventions, naming, and behaviour. This survey will -# compare them and provide a case for the inclusion of a "rich -# iterator wrapper" to the Python Standard Library -# -# .. contents:: -# -# :: - -"""iterqueue: mutable iterators - -Classes for "extended iterators" with methods to let iterators be used as -queue - - `push` or - `append left` -- push back values - `peek` -- get a value without "using it up" - `__nonzero__` -- test for empty iterator - -""" - -# Imports -# -# The `itertools` module provides a set of building blocks for the work with -# iterators (but misses a class for "mutable" iterators). :: - -import itertools - -# The `collections` module with the efficient double-sided queue was -# introduced in Python 2.4. The following construct provides a minimal -# compatibility definition if it is not available:: - -try: - from collections import deque -except ImportError: - class deque(list): - def appendleft(self, value): - self.insert(0, value) - - -# Iterables and Iterators -# ======================= -# -# Iterables and iterators are defined by the iterator protocol as laid out in -# the section on `Iterator Types`_ in the Python Library Reference`: -# -# Iterables: -# One method needs to be defined for container objects to provide iteration -# support: -# -# :__iter__(): Return an iterator object. [...] If a container supports -# different types of iteration, additional methods can be -# provided to specifically request iterators for those -# iteration types. [...] -# -# :: - -def is_iterable(object): - """Check if the argument is iterable""" - return hasattr(object, "__iter__") and is_iterator(iter(object)) - -# Iterators: -# The *iterator objects* themselves are required to support the following -# two methods, which together form the *iterator protocol*: -# -# :__iter__(): Return the iterator object itself. This is required to allow -# both containers and iterators to be used with the `for` and -# `in` statements... -# -# :next(): Return the next item from the container. If there are no further -# items, raise the `StopIteration` exception. -# -# [...] once an iterator's next() method raises `StopIteration`, -# it will continue to do so on subsequent calls. Implementations -# that do not obey this property are deemed broken. -# -# Check if an object is an iterator:: - -def is_iterator(object): - """check if the argument is an iterator""" - if not hasattr(object, "__iter__"): - return False - return (object.__iter__() is object) and hasattr(object, "next") - - -# Try it: -# -# >>> import iterqueue -# >>> iterqueue.is_iterator(23) -# False -# >>> iterqueue.is_iterator(iter(range(3))) -# True -# -# The iterator protocol was primarily designed to be the *minimum* necessary -# to work in `for statements`, translating (behind the scene):: - -# | for item in iterable: -# | -# -# into the equivalent of:: - -# | iterator = iter(iterable) -# | while 1: -# | try: -# | item = iterator.next() -# | except StopIteration: break -# | -# -# -# To add iterator behaviour to your classes, define an `__iter__` method which -# returns an object with a `next` method. If the class defines `next`, then -# `__iter__` can just return `self`. (`tutorial chapter on iterators`_) -# -# Python's *generators* provide a convenient way to implement the iterator -# protocol. Generator objects are returned by *generator functions* (functions -# with the ``yield`` keyword, new in 2.3) and *generator expressions* (new in -# 2.4). -# -# .. _`Iterator Types`: -# http://docs.python.org/library/stdtypes.html#iterator-types -# .. _`tutorial chapter on iterators`: -# http://docs.python.org/tutorial/classes.html#iterators -# -# Limitations of iterator objects -# =============================== -# -# Most built-in Python iterator objects (including generator objects) are -# non-mutable (except the call to the `next` method). They "produce the data -# just in time", which is fast and memory efficient. -# -# However: -# -# 1. In some occasions, it is important to -# -# * find out whether an iterator is empty or -# * to "peek" at a data value -# -# without advancing the iterator. -# -# 2. In a state machine, an iterator holding the input values can be passed -# around to the state handling functions. If a state handler realises that -# a value should be processed by another state handler, it needs to -# "push it back". -# -# 3. One might want modify the object iterated over in a `for` statement. -# -# Generally, the object in a `for` statement can not be changed inside the -# loop. -# -# >>> from collections import deque -# >>> it = deque(range(3)) -# >>> for v in it: -# ... print v, -# ... if v == 1: -# ... it.appendleft("eins") -# ... -# Traceback (most recent call last): -# File "doctest.py", line 1248, in __run -# compileflags, 1) in test.globs -# File "", line 1, in ? -# for v in it: -# RuntimeError: deque mutated during iteration -# -# Pushing the limits -# ==================== -# -# There are many ways to live with the limits of iterators. Most often it -# helps to get a true understanding of their nature and try to count for it in -# the code. However, the "never ending" discussion and varying recipes for -# enhanced iterators show the ongoing public demand. This is why I argue for -# the inclusion of a 'rich iterator' wrapper class into the standard library -# based on the _`standardisation argument` in the itertools_ module. -# -# Standardisation helps avoid the readability and reliability problems which -# arise when many different individuals create their own slightly varying -# implementations, each with their own quirks and naming conventions. -# -# -# .. _itertools: http://docs.python.org/library/itertools.html -# -# Recode to work with iterators as they are -# ----------------------------------------- -# -# The most straightforward way is to translate code like -# -# >>> def print_first(l): -# ... if not l: -# ... print "list empty" -# ... else: -# ... print l[0] -# ... -# >>> print_first([1, 2]) -# 1 -# >>> print_first([]) -# list empty -# -# into something in the line of -# -# >>> def print_next(it): -# ... try: -# ... value = it.next() -# ... except StopIteration: -# ... print "list empty" -# ... else: -# ... print value -# ... -# >>> print_next(iter([1, 2])) -# 1 -# >>> print_next(iter([])) -# list empty -# -# In a `for` statement, the `else` keyword can be utilised to call an -# expression (or a block) if the end of the iterator is reached: -# -# >>> def find_five(iterable): -# ... for i in iterable: -# ... if i == 5: -# ... print "5 found" -# ... break -# ... else: -# ... print "5 not found" -# -# If the loop is aborted, the else clause is skipped -# -# >>> find_five(range(7)) -# 5 found -# -# Otherwise it prints its message: -# -# >>> find_five(range(3)) -# 5 not found -# -# However, there might be cases where this is not applicable and a test for -# the emptiness or a peek at the first value without advancing the iterator -# would enable much cleaner code. -# -# Use a container object -# ---------------------- -# -# One could wrap e.g. a generator into a `list` or `collections.deque` to add -# random access as well as extensibility. -# -# >>> que = deque(xrange(3)) -# >>> que.appendleft("foo") -# >>> print que -# deque(['foo', 0, 1, 2]) -# -# However, it will put all iterator values into memory which becomes a problem -# for large iterators (and is non-feasible for unlimited iterators). -# -# Also, iterating in a `for` statement will loose the rich behaviour. Instead -# a construct with a `while` statement is needed, e.g: -# -# >>> que = deque(range(3)) -# >>> while que: -# ... v = que.popleft() -# ... print v, -# ... if v == 1: -# ... que.appendleft("eins") -# ... -# 0 1 eins 2 -# -# Use a rich iterator -# ------------------- -# -# If the argument of a `for` statement is an iterator (whose `__iter__` -# method returns `self`), it is available unchanged inside the loop. A *rich -# iterator* provides additional methods besides the ones required for the -# iterator protocol. -# -# State Reporting Iterator -# ~~~~~~~~~~~~~~~~~~~~~~~~ -# -# An iterator that returns an indicator "full or empty" (values waiting or -# not) when converted to Boolean will be called *state reporting iterator*:: - -def is_state_reporting(object): - return hasattr(object, "__nonzero__") or hasattr(object, "__len__") - -# Peekable Iterator -# ~~~~~~~~~~~~~~~~~ -# -# An iterator that provides a `peek` method will be called a -# *peekable iterator*:: - -def is_peekable(object): - return hasattr(object, "peek") - - -# Push Iterator -# ~~~~~~~~~~~~~ -# -# An iterator that provides for push-back will be called *push-iterator*:: - -def is_pushable(object): - return hasattr(object, "appendleft") or hasattr(object, "push") - -# Push iterators can be easily extended with `peek` and test of emptiness -# (see `PushIterator`_). -# -# Iterator Queue -# ~~~~~~~~~~~~~~ -# An iterator that also provides methods for appending and extending will be -# called *iterator_queue*. -# -# Methods that need access from the "right" side or knowledge of the length of -# the iterator are not included in the iterator_queue specification as they -# clash with the "just in time" acquisition of the values that give iterators -# time and memory advantage over sequences. :: - -def is_iterator_queue(object): - return (is_state_reporting(object) - and hasattr(object, "append") - and hasattr(object, "appendleft") - and hasattr(object, "extend") - and hasattr(object, "extendleft") - and hasattr(object, "clear") - and hasattr(object, "rotate") - ) - -# Rich iterator examples -# ====================== -# -# The following examples are the result of a net survey and own ideas. -# The will be compared and profiled in this paper. -# -# All of them are iterator-wrappers:: - -def is_iterator_wrapper(obj): - """Try if obj can wrap an iterator""" - try: - it = obj(xrange(1)) - except: - return False - try: - return is_iterator(it) - except: - return False - -# Xiterwrapper -# ------------ -# -# Tom Andersson suggested in the Python list an `xiterable protocol`__ for -# extended iterables and a wrapper to convert a "traditional" iterator to an -# extended version -# -# __ http://mail.python.org/pipermail/python-list/2006-January/360162.html -# -# :: - -def xiter(iterable): - if (hasattr(iterable, "__xiter__")): - return iterable.__xiter__() - else: - return xiterwrapper(iter(iterable)) - -class xiterwrapper(object): - def __init__(self, it): - self.it = it - self.advance() - def hasNext(self): - return hasattr(self, "_next") - def next(self): - try: - cur = self._next - self.advance() - return cur - except AttributeError: - raise StopIteration - def peek(self): - try: - return self._next - except AttributeError: - raise StopIteration - def advance(self): - try: - self._next = self.it.next() - except StopIteration: - if (hasattr(self, "_next")): - del self._next - def __xiter__(self): - return self - def __iter__(self): - return self - -# Usage -# ~~~~~ -# -# >>> import iterqueue -# >>> it = iterqueue.xiter(xrange(3)) -# >>> iterqueue.is_iterator(it) -# True -# >>> iterqueue.is_peekable(it) -# True -# >>> iterqueue.is_pushable(it) -# False -# -# We add the __nonzero__ method for a non-destructive test of waiting values -# to add the state reporting feature:: - - __nonzero__ = hasNext - -# >>> iterqueue.is_state_reporting(it) -# True -# -# Adding a `push` method is not possible without major changes to the code. -# -# IteratorWrapper BFL -# ------------------- -# -# In a `post on python-3000`__ Guido van Rossum argued against inclusion of an -# "emptiness" test in the iterator protocol, as "that's just not something -# that generators can be expected to support" and hence would exclude -# generators from the definition of an iterator. -# -# __ http://mail.python.org/pipermail/python-3000/2006-March/000058.html -# -# ... you can always write a helper class that takes an iterator and -# returns an object that represents the same iterator, but sometimes -# buffers one element. But the buffering violates the coroutine-ish -# properties of generators, so it should not be the only (or even the -# default) way to access generators. -# ... -# -# Here's a sample wrapper (untested) -# -# :: - -class IteratorWrapperBFL(object): - def __init__(self, it): - self.it = iter(it) - self.buffer = None - self.buffered = False - self.exhausted = False - def __iter__(self): - return self - def next(self): - if self.buffered: - value = self.buffer - self.buffered = False - self.buffer = None - return value - if self.exhausted: - raise StopIteration() - try: - return self.it.next() - except StopIteration: - self.exhausted = True - raise - def __nonzero__(self): - if self.buffered: - return True - if self.exhausted: - return False - try: - self.buffer = self.it.next() - except StopIteration: - self.exhausted = True - return False - self.buffered = True - return True - -# This example provides an "emptiness" test but no peek or push-back: -# -# >>> it = iterqueue.IteratorWrapperBFL(xrange(3)) -# >>> iterqueue.is_state_reporting(it) -# True -# -# Peeking could be easily added, though:: - - def peek(self): - self.buffer = self.next() - self.buffered = True - return self.buffer - -# >>> iterqueue.is_peekable(it) -# True -# -# IteratorWrapper DD -# ------------------ -# -# Daniel Dittmar wrote on Di 22 Jul. 2003 on comp.lang.python -# -# It shouldn't be too difficult to write an iterator wrapper class that does -# exactly what you want (not tested):: - -class IteratorWrapperDD: - def __init__ (self, iterArg): - iterArg = iter (iterArg) - try: - self.firstElement = iterArg.next () - self.isEmpty = false - self.next = self.returnFirstElement - self.baseIter = iterArg - except StopIteration: - self.isEmpty = true - self.next = self.throwStopIteration - - def returnFirstElement(self): - self.next = self.baseIter.next - return self.firstElement - - def throwStopIteration(self): - raise StopIteration - - -# PushIterator -# ------------ -# -# In the slides to the `Effective Python Programming`_ OSCON 2005 tutorial by -# Anthony Baxter, I found a genially simple example for an iterator with a -# `push` method. -# -# .. _`Effective Python Programming`: -# http://www.interlink.com.au/anthony/tech/talks/OSCON2005/effective_r27.pdf -# -# :: - -class PushIterator: - def __init__(self, iterable): - """Store iterator as data argument and set up cache""" - self.it = iter(iterable) - self.cache = [] - - def __iter__(self): - return self - - def next(self): - """Return next value (from cache or iterator)""" - if self.cache: - return self.cache.pop() - return self.it.next() - - def push(self, value): - """Push back one value (will become the `next` value)""" - self.cache.append(value) - -# Once `push` is defined, it is easy to add `peek` and `__nonzero__`. -# -# The question arises, what should be returned by `peek()` if the iterator is -# empty. The easiest option is to raise `StopIteration`, but this might come -# unhandy in some cases. My proposal is to add an optional `default` argument, -# which is returned in case the iterator is empty. (As there is no sensible -# default value for the `default` argument, it cannot be implemented as -# keyword arg, instead an argument list is used):: - - def peek(self, *defaults): - """Return next value but do not advance the iterator""" - try: - value = self.next() - except StopIteration: - if defaults: - return defaults[0] - raise - self.push(value) - return value - - def __nonzero__(self): - """Test whether the iterator is empty""" - try: - self.peek() - except StopIteration: - return False - return True - -# An alias makes the class more compatible with `collections.deque` :: - - appendleft = push - -# Optimisation of `peek` and `__nonzero__` is is left out in favour of -# improved clarity. -# -# Usage -# ~~~~~ -# -# Create an instance from an iterable object: -# -# >>> it = iterqueue.PushIterator(range(4)) -# -# Test for values: -# -# >>> bool(it) -# True -# -# Have a peek ... -# -# >>> it.peek(None) -# 0 -# -# the value is still there: -# -# >>> it.next() -# 0 -# -# See what is left -# -# >>> [i for i in it] -# [1, 2, 3] -# -# It should be empty now: -# -# >>> bool(it) -# False -# -# So a peek will return the default: -# -# >>> print it.peek(None) -# None -# -# PushIt -# ------------- -# -# The wrapping of an iterator in a class leads to performance loss, as every -# call to `next()` is a relatively costly function call. -# -# Remapping of self.next leads to a more more efficient implementation -# of the PushIterator for the case that `peek` or `push` is called far -# less frequently than `next` ('normal' iterating with occasional peek or -# backtrack). :: - -class PushIt(PushIterator): - def __init__(self, iterable): - self.it = iter(iterable) - self.cache = [] - self.next = self.it.next - - def _next(self): - """Return next element. Try cache first.""" - if self.cache: - return self.cache.pop() - self.next = self.it.next - return self.next() - - def push(self, value): - """Push back one value to the iterator""" - self.cache.append(value) - self.next = self._next - - def peek(self): - """Return next value but do not advance the iterator""" - if self.cache: - return self.cache[-1] - value = self.it.next() - self.push(value) - return value - - -# IterQueue -# --------- -# -# The `IterQueue` class adds iterator behaviour to a double-ended queue:: - -class IterQueue(deque): - """Iterator object that is also a queue""" - def __iter__(self): - return self - def next(self): - try: - return self.popleft() - except IndexError: - raise StopIteration - # - def peek(self): - """Return next value but do not advance the iterator""" - value = self.next() - self.appendleft(value) - return value - - -# Usage -# ~~~~~ -# -# Creating an instance wraps an iterable in an iterator queue -# -# >>> it = iterqueue.IterQueue(xrange(3)) -# -# which is an iterator according to the iterator protocol with "queue" -# methods -# -# >>> iterqueue.is_iterator_queue(it) -# True -# -# We can test whether there is data in the iterator or get the length of it: -# -# >>> bool(it) -# True -# >>> len(it) -# 3 -# -# It is possible to modify this iterator in the middle of a `for` statement: -# -# >>> for v in it: -# ... print v, -# ... if v == 1: -# ... it.appendleft("eins") -# ... -# 0 1 eins 2 -# -# As iteration is done on the object itself and not on a copy, it is exhausted -# now: -# -# >>> print it -# deque([]) -# -# (the iterator advertises itself as `deque`, as we did not override the -# `__repr__` method) -# -# We can make up for this :: - - def __repr__(self): - return "" - -# but there might be other problems left... -# -# -# Problems -# ~~~~~~~~ -# -# Converting an iterable to a `collections.deque` object creates a list of all -# values in the memory, loosing the memory saving advantages of generator -# objects with "just in time" production of the data. -# -# Printing (and probably other uses as well) "use up" the iterator -# -# >>> it = iterqueue.IterQueue(range(3)) -# >>> print it -# deque([0, 1, 2]) -# >>> print it -# deque([]) -# -# -# IQueue -# ------ -# -# The following class implements an iterator queue that is -# -# * memory efficient, as generators are kept as generators -# -# * mostly compatible to `collections.deque` (offering all methods of a -# `deque` for appends) -# -# It does not offer -# -# * random access to the values, nor -# -# * pop from the right end, -# -# as this would require to convert the iterator to a sequence loosing the -# memory-saving advantage. -# -# Iterating over instances is less fast, as the next() method is redefined -# (a function call is needed for every step). Implementing in C would help -# to improve speed. -# -# But, -# -# itertools.queue() was rejected because it didn't fit naturally into -# applications -- you had to completely twist your logic around just to -# accommodate it. Besides, it is already simple to iterate over a list -# while appending items to it as needed. -# -# -- Raymond Hettinger 03-13-05 http://www.codecomments.com/message423138.html -# -# However, both, the speed increase as well as the `standardisation argument`_ -# given for the `itertools` hold also in this case -# Maybe IQueue should become a collections candidate? -# -# :: - -class IQueue: - """Iterator with "on-line extensibility" - - An iterator with methods to append or extend it from left or right - (even while the iterator is in use). - - Can be conceived as a mixture of `itertools.chain` and - `collections.deque`. - - As `next` is redefined, there is a performance loss when iterating - over large iterators. - """ - - def __init__(self, *iterables): - """Convert `iterables` to a queue object""" - self.iterators = deque(iterables) - # - def __iter__(self): - return self - - def next(self): - while True: - try: - return self.iterators[0].next() - except AttributeError: # convert iterable to iterator - self.iterators[0] = iter(self.iterators[0]) - except StopIteration: # switch to next iterator - del(self.iterators[0]) - except IndexError: # all iterators exhausted - raise StopIteration - # - def append(self, value): - """append `value` to self - - The value is wrapped in an iterable and - appended to the queue of iterables - """ - self.iterators.append(iter((value,))) - # - def appendleft(self, value): - """Prepend one (scalar) value to the iterator. - - The value is wrapped in an iterable and - inserted at the first position in the list of iterables - """ - self.iterators.appendleft(iter((value,))) - # - def clear(self): - """Remove all elements from the iterator. - """ - self.iterators.clear() - # - def extend(self, iterable): - """append `iterable` to self""" - self.iterators.append(iter(iterable)) - # - def extendleft(self, iterable): - """prepend `iterable` to self""" - self.iterators.appendleft(iter(iterable)) - # - def peek(self): - """Return the next value without advancing the iterator - - Yield next value but push back a copy of the result. - This way you may "peak" at an iterator without loss. - - Raises `StopIteration` if the iterator is empty. - """ - value = self.next() - self.iterators.appendleft(iter((value,))) - return value - # - def rotate(self, n=1): - """append the next `n` values to the end of the iterator - - Similar to `container.deque.rotate`, but - * negative `n` leads to error - * a list of the `n` rotated values is returned - """ - result = list(itertools.islice(self, n)) - self.iterators.append(result) - return result - - # - def __repr__(self): - """Return a string representation""" - return "IQueue(%s)" % repr(self.iterators) - # - def __nonzero__(self): - """Test for a non-zero length of the iterator""" - if len(self.iterators) > 1: - return True - try: - self.peek() - except StopIteration: - return False - return True - -# XIter -# ----- -# -# The `XIter` class is an optimised version of the `IQueue` for the -# case when appending of a value is a done less frequently than calling `next`. -# It does so by aliasing next to the underlying iterators `next` method in -# case there is only one iterator in the `iterables` chain. -# -# :: - -class XIter: - """'Mutable iterator' class""" - def __init__(self, *iterables): - self.iterators = deque(iter(i) for i in iterables) - if len(self.iterators) is 1: - self.next = self.iterators[0].next - else: - self.next = self._next # iterate over argument - # - def __iter__(self): return self # "I am an iterator!" - # - def _next(self): - """get next in turn if there are more than one iterators""" - try: - return self.iterators[0].next() - except StopIteration: - del(self.iterators[0]) # switch to next iterator - assert len(self.iterators) >= 1 - if len(self.iterators) is 1: - self.next = self.iterators[0].next - return self.next() - # - def append(self, element): - """append `element` to self""" - self.iterators.append(iter((element,))) - self.next = self._next # iterate over cache - # - def appendleft(self, element): - """prepend `element` to self""" - self.iterators.appendleft(iter((element,))) - self.next = self._next # iterate over cache - # - def extend(self, iterable): - """append `iterable` to self""" - self.iterators.append(iter(iterable)) - self.next = self._next # iterate over cache - # - def extendleft(self, iterable): - """prepend `iterable` to self""" - self.iterators.appendleft(iter(iterable)) - self.next = self._next # iterate over cache - - # - def peek(self): - """Return the next value without advancing the iterator - - Yield next value but push back a copy of the result. - This way you may "peak" at an iterator without loss. - - Raises `StopIteration` if the iterator is empty. - """ - value = self.next() - self.appendleft(value) - return value - # - def rotate(self, n=1): - """append the next `n` values to the end of the iterator - - Similar to `container.deque.rotate`, but - * negative `n` leads to error - * a list of the `n` rotated values is returned - """ - result = list(itertools.islice(self, n)) - self.iterators.append(result) - return result - - # - def __repr__(self): - return "XIter(%s)" % repr(self.iterators) - # - def __nonzero__(self): - """Test for a non-zero length of the iterator""" - if len(self.iterators) > 1: - return True - try: - self.peek() - except StopIteration: - return False - return True - - -# Some optimisation could be done adapting a `round-robin example`__ posted by -# R. Hettinger on 2004-04-30 15:58 in comp.lang.python -# -# __ http://sourceforge.net/tracker/index.php?func=detail&aid=756253&group_id=5470&atid=305470 -# -# :: - -## -# For the record, here a simple and efficient roundrobin task -# server based on collections.deque: -# -# def roundrobin(*iterables): -# pending = deque(iter(i).next for i in iterables) -# gettask, scheduletask = pending.popleft, pending.append -# while pending: -# task = gettask() -# try: -# yield task() -# except StopIteration: -# continue -# scheduletask(task) -# -# for value in roundrobin('abc', 'd', 'efgh'): -# print value - - -# Do a doctest if the module is run in nosetests:: - -def test(): - import doctest - doctest.testmod() diff --git a/rstdocs/examples/iterqueue_speed_test.py b/rstdocs/examples/iterqueue_speed_test.py deleted file mode 100644 index fa8aa5c..0000000 --- a/rstdocs/examples/iterqueue_speed_test.py +++ /dev/null @@ -1,100 +0,0 @@ -# iterqueue_speed_test.py -# ======================= -# Profiling the iterqueue extended iterator classes -# ------------------------------------------------- -# -# :: - -import sys, os, itertools -from timeit import Timer -import iterqueue -from iterqueue_test import wrappers, peekables, pushables, \ - state_reporters, iqueues - - -def print_iterator_lists(): - print "Wrappers" - print " ","\n ".join(wrapper.__name__ for wrapper in wrappers) - print "Peekables" - print " ","\n ".join(peekable.__name__ for peekable in peekables) - print "Pushables" - print " ","\n ".join(pushable.__name__ for pushable in pushables) - print "State Reporters" - print " ","\n ".join(state_reporter.__name__ - for state_reporter in state_reporters) - print "Iterator Queues" - print " ","\n ".join(iqueue.__name__ for iqueue in iqueues) - - -#print_iterator_lists() - -# use cases (benchmarks) -# ~~~~~~~~~~~~~~~~~~~~~~ -# -# :: - -def loop(iterator): - """baseline: empty `for` loop""" - for _ in iterator: - pass - -def peek_in_loop(iterator): - """peek in every loop""" - for _ in iterator: - try: - iterator.peek() - except StopIteration: - pass - -def peek_before_loop(iterator): - """peek at first value once, then loop""" - try: - iterator.peek() - except StopIteration: - pass - for _ in iterator: - pass - -def bool_in_loop(iterator): - """test for values in every loop""" - for _ in iterator: - bool(iterator) - -def bool_before_loop(iterator): - """test for values once, then loop""" - bool(iterator) - for _ in iterator: - pass - - -def time_benchmark(fun, wrappers, iterator): - """profile benchmark `fun` with `iterator` wrapped in `wrappers`""" - - print fun.__doc__, "(%s)"%iterator - setup = "import iterqueue_speed_test\nimport iterqueue" - benchmark = "iterqueue_speed_test.%s(iterqueue.%s(%s))" - stmts = [benchmark%(fun.__name__, wrapper.__name__, iterator) - for wrapper in wrappers] - timers = [Timer(stmt=stmt, setup=setup) for stmt in stmts] - - t_i = [min(timer.repeat(number=1, repeat=3)) for timer in timers] - - results = ["%.5f s %s"%(t, wrapper.__name__) - for t, wrapper in zip(t_i, wrappers)] - results.sort() - print "\n".join(results) - -# time_benchmark(loop, iterqueue.XIter, xrange(1000)) -# -# :: - -time_benchmark(loop, wrappers, xrange(1000)) -print -time_benchmark(peek_before_loop, peekables, xrange(1000)) -print -time_benchmark(peek_in_loop, peekables, xrange(1000)) -print -time_benchmark(bool_before_loop, state_reporters, xrange(1000)) -print -time_benchmark(bool_in_loop, state_reporters, xrange(1000)) - diff --git a/rstdocs/examples/iterqueue_test.py b/rstdocs/examples/iterqueue_test.py deleted file mode 100644 index 1da7d0f..0000000 --- a/rstdocs/examples/iterqueue_test.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf8 -*- - -# iterqueue_test.py -# ***************** -# Test the iterator wrappers from iterqueue.py -# ============================================ -# -# .. contents:: -# -# :: - -import sys, itertools -import iterqueue -from iterqueue import * - -# Get and sort the wrapper classes -# -------------------------------- -# -# List all iterator wrapper objects:: - -wrappers = [obj for obj in iterqueue.__dict__.values() - if is_iterator_wrapper(obj)] -# print "\n".join(repr(wrapper) for wrapper in wrappers) - -# List iterator wrappers that provide a `peek` method:: - -peekables = [obj for obj in wrappers if is_peekable(obj)] -# print "Peekables" -# print "\n".join(repr(peekable) for peekable in peekables) - -# List iterator wrappers that provide a `push` method:: - -pushables = [obj for obj in wrappers if is_pushable(obj)] -# print "Pushables" -# print "\n".join(repr(pushable) for pushable in pushables) - -# List iterator wrappers that provide a test for "values available":: - -state_reporters = [obj for obj in wrappers if is_state_reporting(obj)] -# print "State Reporters" -# print "\n".join(repr(state_reporter) for state_reporter in state_reporters) - -# List iterator wrappers that implement the "queue" methods:: - -iqueues = [obj for obj in wrappers if is_iterator_queue(obj)] -# print "Iterator Queues" -# print "\n".join(repr(iqueue) for iqueue in iqueues) - - -# Test Wrappers -# ------------- -# -# Test the basic iterator features of an iterator wrapper. :: - -class Test_Wrappers: - """Test the wrapping of iterator wrappers""" - def wrap_ok(self, wrapper, base): - """iterating over the wrapper should return the same - as iterating over base - """ - print wrapper - assert list(wrapper(base)) == list(base) - assert [i for i in wrapper(base)] == [i for i in base] - - def test_wrappers(self, base=xrange(3)): - for wrapper in wrappers: - yield self.wrap_ok, wrapper, base - - -# Test Peekables -# -------------- -# -# :: - -class Test_Peekables: - """Test the peek method of iterator wrappers""" - def peek_ok(self, wrapper, base): - """peek() should return next value but not advance the iterator""" - print wrapper - print wrapper.peek - it = wrapper(base) - it.peek() - first = it.peek() - print first - assert first == 0 - # peek() must not "use up" values - result = list(it) - print result - assert result == list(base) - - def test_peekables(self, base=xrange(3)): - """Test generator for peekable iterator wrappers""" - for wrapper in peekables: - yield self.peek_ok, wrapper, base - -# Test Pushables -# -------------- -# -# :: - -class Test_Pushables: - """Test the push method of iterator wrappers""" - - def push_ok(self, wrapper, base): - """push(value) shall prepend `value` to iterator""" - print wrapper.push - it = wrapper(base) - it.push(9) - result = list(it) - print result - assert result == [9] + list(base) - - def push_while_iterating_ok(self, wrapper): - """push shall work even in an iteration loop""" - print wrapper - it = wrapper(xrange(3)) - result = [] - for i in it: - if i == 1: - it.push("xx") - result.append(i) - assert result == [0, 1, 'xx', 2] - - def test_pushables(self, base=xrange(3)): - """Test generator for pushable iterator wrappers""" - for wrapper in pushables: - if not hasattr(wrapper, "push"): - wrapper.push = wrapper.appendleft - yield self.push_ok, wrapper, base - yield self.push_while_iterating_ok, wrapper - - -# Test Iterator Queue -# ------------------- -# -# :: - -class TestIteratorQueue: - """Test the queueing methods of iterator queues""" - # - def extend_ok(self, wrapper, base): - """extend(iterable) shall append `iterable` to iterator""" - print wrapper - it = wrapper(base) - it.extend([9]) - assert list(it) == list(base) + [9] - - def extendleft_ok(self, wrapper, base): - """extendleft(iterable) shall prepend `iterable` to iterator""" - print wrapper - it = wrapper(base) - it.extendleft([9]) - result = [i for i in it] - print result - assert result == [9] + list(base) - - def append_ok(self, wrapper, base): - """append(value) shall append `value` to iterator""" - print wrapper - it = wrapper(base) - it.append(9) - result = list(it) - print result - assert result == list(base) + [9] - - def test_iqueues(self, base=xrange(3)): - """Test generator for iterator-queue wrappers""" - for wrapper in iqueues: - yield self.extend_ok, wrapper, base - yield self.extendleft_ok, wrapper, base - yield self.append_ok, wrapper, base - - -# Test State Reporters -# -------------------- -# -# :: - -class Test_StateReporters: - """Test the state reporting when converted to bool""" - def bool_ok(self, wrapper): - """Empty iterator should evaluate to False - Non-empty iterator should evaluate to True - the evaluation should not advance the iterator - """ - base = xrange(3) # make sure it is not empty! - it0 = wrapper([]) - assert bool(it0) == False - assert list(it0) == [] - it1 = wrapper(base) - assert bool(it1) == True - assert list(it1) == list(base) - assert bool(wrapper(iter([]))) == False - assert bool(wrapper(iter([1]))) == True - - def test_iqueues(self): - """Test generator for iterator-queue wrappers""" - for wrapper in state_reporters: - yield self.bool_ok, wrapper - - -if __name__ == "__main__": - import nose - # this doesnot show any effect :-( - nose.configure(["test.py", "--detailed-errors"]) - nose.runmodule() # requires nose 0.9.1 diff --git a/rstdocs/examples/simplestates.py b/rstdocs/examples/simplestates.py deleted file mode 100644 index d27bbb0..0000000 --- a/rstdocs/examples/simplestates.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python -# -*- coding: iso-8859-1 -*- - -# simplestates.py -# *************** -# Generic state machine class using iterators -# +++++++++++++++++++++++++++++++++++++++++++ -# -# :Version: 0.2 -# :Date: 2006-12-01 -# :Copyright: 2006 Guenter Milde. -# Released under the terms of the GNU General Public License -# (v. 2 or later) -# -# Detailed documentation of this class and the design rationales (including -# tested variants) is available in the file simplestates-test.py.txt -# :: - -"""Simple generic state machine class using iterators - -Usage -===== - -Example: A two-state machine sorting numbers in the categories - "< 3" and ">= 3". - -Preparation ------------ - -Import the basic class:: - ->>> from simplestates import SimpleStates - -Subclass and add state handlers: - ->>> class StateExample(SimpleStates): -... def high_handler_generator(self): -... result = [] -... for token in self.data_iterator: -... if token <= 3: -... self.state = "low" -... yield result -... result = [] -... else: -... result.append(token) -... yield result -... def low_handler_generator(self): -... result = [] -... for token in self.data_iterator: -... if token > 3: -... self.state = "high" -... yield result -... result = [] -... else: -... result.append(token) -... yield result - - -Set up an instance of the StateExample machine with some test data:: - ->>> testdata = [1, 2, 3, 4, 5, 4, 3, 2, 1] ->>> testmachine = StateExample(testdata, state="low") - ->>> print [name for name in dir(testmachine) if name.endswith("generator")] -['high_handler_generator', 'low_handler_generator'] - - -Running -------- - -Iterating over the state machine yields the results of state processing:: - ->>> for result in testmachine: -... print result, -... -[1, 2, 3] [5, 4] [2, 1] - -For a correct working sort algorithm, we would expect:: - - [1, 2, 3] [4, 5, 4] [3, 2, 1] - -However, to achieve this a backtracking algorithm is needed. See iterqueue.py -and simplestates-test.py for an example. - - -The `__call__` method returns a list of results. It is used if you call -an instance of the class:: - ->>> testmachine() -[[1, 2, 3], [5, 4], [2, 1]] - -""" - -# Abstract State Machine Class -# ============================ -# -# :: - -class SimpleStates: - """generic state machine acting on iterable data - - Class attributes: - - state -- name of the current state (next state_handler method called) - state_handler_generator_suffix -- common suffix of generator functions - returning a state-handler iterator - """ - state = 'start' - state_handler_generator_suffix = "_handler_generator" - -# Initialisation -# -------------- -# -# * sets the data object to the `data` argument. -# -# * remaining keyword arguments are stored as class attributes (or methods, if -# they are function objects) overwriting class defaults (a neat little trick -# I found somewhere on the net) -# -# ..note: This is the same as `self.__dict__.update(keyw)`. However, -# the "Tutorial" advises to confine the direct use of `__dict__` -# to post-mortem analysis or the like... -# -# :: - - def __init__(self, data, **keyw): - """data -- iterable data object - (list, file, generator, string, ...) - **keyw -- all remaining keyword arguments are - stored as class attributes - """ - self.data = data - for (key, value) in keyw.iteritems(): - setattr(self, key, value) - - - - - -# Iteration over class instances -# ------------------------------ -# -# The special `__iter__` method returns an iterator. This allows to use -# a class instance directly in an iteration loop. We define it as is a -# generator method that sets the initial state and then iterates over the -# data calling the state methods:: - - def __iter__(self): - """Generate and return an iterator - - * ensure `data` is an iterator - * convert the state generators into iterators - * (re) set the state attribute to the initial state - * pass control to the active states state_handler - which should call and process self.data_iterator.next() - """ - self.data_iterator = iter(self.data) - self._initialize_state_generators() - # now start the iteration - while True: - yield getattr(self, self.state)() - -# a helper function generates state handlers from generators. It is called by -# the `__iter__` method above:: - - def _initialize_state_generators(self): - """Generic function to initialise state handlers from generators - - functions whose name matches `[^_]_handler_generator` will - be converted to iterators and their `.next()` method stored as - `self.`. - """ - suffix = self.state_handler_generator_suffix - shg_names = [name for name in dir(self) - if name.endswith(suffix) - and not name.startswith("_")] - for name in shg_names: - shg = getattr(self, name) - setattr(self, name[:-len(suffix)], shg().next) - -# Use instances like functions -# ---------------------------- -# -# To allow use of class instances as callable objects, we add a `__call__` -# method:: - - def __call__(self): - """Iterate over state-machine and return results as a list""" - return [token for token in self] - -# Command line usage -# ================== -# -# running this script does a doctest:: - -if __name__ == "__main__": - import doctest - doctest.testmod() diff --git a/rstdocs/examples/simplestates_test.py b/rstdocs/examples/simplestates_test.py deleted file mode 100644 index ef8a43c..0000000 --- a/rstdocs/examples/simplestates_test.py +++ /dev/null @@ -1,642 +0,0 @@ -#!/usr/bin/env python -# -*- coding: iso-8859-1 -*- - -# simplestates_test.py: Test the simplestates.py generic state machine -# ******************************************************************** -# -# :Status: draft -# :Date: 2006-12-01 -# :Copyright: 2006 Guenter Milde. -# Released under the terms of the GNU General Public License -# (v. 2 or later) -# -# .. default-role:: literal -# -# .. contents:: :depth: 2 -# -# -# Abstract State Machine Class -# ============================ -# -# First version of an abstract state machine -# :: - -class SimpleStates1: - """generic state machine acting on iterable data - - Class attributes - init_state -- name of the first state_handler method - """ - init_state = 'start' - -# Initialisation -# -# * sets the data object to the `data` argument. -# * remaining keyword arguments are stored as class attributes (or methods, if -# they are function objects) overwriting class defaults (a neat little trick -# I found somewhere on the net) -# * the `state_handler` attribute is set to the method named in `init_state` -# -# :: - - def __init__(self, data, **keyw): - """data -- iterable data object - (list, file, generator, string, ...) - **keyw -- all remaining keyword arguments are - stored as class attributes - """ - self.data = data - self.__dict__.update(keyw) - -# The special `__iter__` method returns an iterator_. This allows to use -# a class instance directly in an iteration loop. We define it as is a -# generator_ method that sets the initial state and then iterates over the -# data calling the state methods:: - - def __iter__(self): - self.state_handler = getattr(self, self.init_state) - for token in self.data: - yield self.state_handler(token) - -# To use class instances as callable objects, we add a `__call__` method:: - - def __call__(self): - """Run state-machine and return tokens as a list""" - return [token for token in self] - -# Example 1: A two-state machine sorting numbers -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# Our small example state machine subclasses the `SimpleStates1` class:: - -class Example1(SimpleStates1): - """A silly example two-state machine sorting numbers - in the categories "low" (< 3) and "high" (>= 3). - """ - -# It will be completed by two state methods and a `__str__` method. -# -# State Methods -# ------------- -# -# State methods are functions that are called to iterate over the data. They -# will typically -# -# * test the data token for a change of state indicator -# * return the data token after some processing -# -# In our example, the `low` method switches to `high` (and calls it with the -# data token), if token is bigger than 3. If not, it returns "l(token)":: - - def low(self, token): - # print "low(", token, ")", - if token > 3: - self.state_handler = self.high - # backtracking - return self.state_handler(token) - return "l(%d)"%token - -# The `high` method switches to `low`, if token is bigger than 3. If not, it -# returns "h(token)":: - - def high(self, token): - # print "high(", token, ")", - if token <= 3: - self.state_handler = self.low - # backtracking - return self.state_handler(token) - return "h(%d)"%token - -# Conversion of the class instance to a string is done by joining the list -# that is returned by a call to the instance with spaces:: - - def __str__(self): - return " ".join(self()) - -# Test -# ---- -# -# Testing is done with the nose_ test framework. This will collect and -# execute all test functions and methods (basically everything that starts or -# ends with "[Tt]est"). This is similar to the more known "py.test". -# -# .. _nose: http://somethingaboutorange.com/mrl/projects/nose/ -# -# We set up some test data:: - -testdata = [1, 2, 3, 4, 5, 4, 3, 2, 1] - -# and define a test function:: - -def test_Example1(): - statemachine = Example1(testdata, init_state='low') - for result in statemachine: - print result, - print - - # Calling an instance should return a list of results - print statemachine() - assert statemachine() == ['l(1)','l(2)','l(3)', # low numbers - 'h(4)','h(5)','h(4)', # high numbers - 'l(3)','l(2)','l(1)'] # low again - - # Converting to a string should call the __str__ method:: - print str(statemachine) - assert str(statemachine) == "l(1) l(2) l(3) h(4) h(5) h(4) l(3) l(2) l(1)" - -# Discussion -# ---------- -# -# The sorting works as expected. However, as the state handlers get the data -# token by token, acting on subsequent tokens or tests that combine the -# knowledge of several tokens are hard to achieve. -# -# An example would be a state handler that sums up the data tokens and -# returns the result if it exceeds a threshold. -# -# -# -# Varied State Machine Class Template -# =================================== -# -# The second version of an abstract state machine converts the test data to an -# iterator which is shared by the state methods. -# -# There is no need to pass this on via arguments, as class methods share the -# class instances attributes (class variables). -# -# We subclass our first version and modify to our needs:: - -class SimpleStates2(SimpleStates1): - """second version of the abstract state machine class - """ - -# We add the initialisation of the data to the `__iter__` method. The data is -# transformed into an iterator_ first. :: - - def __iter__(self): - self.data_iterator = iter(self.data) - self.state_handler = getattr(self, self.init_state) - # do not pass data tokens as argument - # (state methods should call self.data_iterator.next() instead) - while True: - yield self.state_handler() - -# Iterators "use up" the data, so the state methods will always get a "new" -# token until the data is fully "used up" and `StopIteration` is raised -# aborting the iteration. -# -# Doing the conversion from iterable to iterator in `__iter__` and not in -# `__init__` allows repeated iteration over the class instance (if the data is -# a list or a file and not already a generator) as the "used up" generator is -# replaced by a new one. -# -# Example 2: Another two-state machine sorting numbers -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# Our small example state machine subclasses the `SimpleStates2` class -# and adds 2 methods as state handlers. :: - -class Example2(SimpleStates2): - """An example two-state machine sorting numbers - in the categories "low" (< 3) and "high" (>= 3). - """ - -# State methods -# ------------- -# -# This time, the state methods will get the data tokens not as argument but -# take them from the `data_iterator`. Note that *backtracking* is impossible -# with a standard iterator. See below for the problem this causes for our -# sorting algorithm. :: - - def low(self): - # print "low(", token, ")", - token = self.data_iterator.next() - if token > 3: - self.state_handler = self.high - return "l(%d)"%token - def high(self): - # print "high(", token, ")", - token = self.data_iterator.next() - if token <= 3: - self.state_handler = self.low - return "h(%d)"%token - -# Test -# ---- -# -# Define a second test function:: - -def test_Example2(): - statemachine = Example2(testdata, init_state='low') - -# Calling an instance should return a list of results. However, as -# we cannot backtrack on a normal iterator, the result is not as we expected: -# There is a "hysteresis" the "switch triggering" token is always processed by -# the "old" state:: - - print statemachine() - assert statemachine() == ['l(1)', 'l(2)', 'l(3)', # low numbers - 'l(4)', 'h(5)', 'h(4)', # high numbers - 'h(3)', 'l(2)', 'l(1)'] # low numbers - -# Discussion -# ---------- -# -# Missing backtracks break our number sorting machine. The remedy -# is the use of an iterator with an appendleft() method (known from the -# dqueue() standard class). We will come to this in `Example 4`__ -# -# __ `Example 4: A two-state machine with generators and backtracking`_ -# -# OTOH, as the state methods do the fetching of data tokens themself, a state -# handler that sums up the data tokens and returns the result if it exceeds a -# threshold would be easy to implement. We will do this in our next example -# using state handler generators. -# -# -# State Machine class using state_handler generators -# ================================================== -# -# The variations in `StateMachine2` complicate the StateMachine design. They -# makes sense, however, if we use generated iterators to handle the states. -# No changes are needed to the abstract base class, so that Example 3 can -# build on `StateMachine2`:: - -class Example3(SimpleStates2): - -# Example 3: A two-state machine with state handler generators -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# State Generators -# ---------------- -# -# State Generators generate and return an iterator that will handle the next -# data token(s) if its .next() method is called. This is easily achieved with a -# for loop over self.data and the `yield` keyword. -# :: - - def high_handler_generator(self): - """Return an iterator, whose next() method - returns "h(token)" and switches to `low`, if token > 3 - """ - for token in self.data_iterator: - if token <= 3: - self.state_handler = self.low - yield "h(%d)"%token - # - def low_handler_generator(self): - """Return an iterator, whose next() method sums up data tokens. - If the sum exceeds 8, it is returned and the state - switches to `high`. - """ - sum = 0 - for token in self.data_iterator: - sum += token - if sum > 8: - self.state_handler = self.high - yield "s=%d"%sum - sum = 0 # next iteration continues here - # no more tokens but sum not reached - yield "p=%d"%sum # partial sum - -# The iterator must instantiate the state-iterators before starting the -# iteration loop:: - - def __iter__(self): - """Generate and return an iterator - - * convert `data` to an iterator - * convert the state generators into iterators - * (re) set the state_handler attribute to the init-state - * pass control to the active states state_handler - which should call and process self.data_iterator.next() - """ - self.data_iterator = iter(self.data) - self.high = self.high_handler_generator().next - self.low = self.low_handler_generator().next - # init state - self.state_handler = getattr(self, self.init_state) - # now start the iteration, aborts if data is empty - while True: - yield self.state_handler() - -# Test -# ------- -# -# Again define a test function that gets an instance of the Example3 class :: - -def test_Example3(): - statemachine = Example3(testdata, init_state='low') - -# Calling statemachine() should iterate over the test data and return the -# processed values as list:: - - print statemachine() - assert statemachine() == ['s=10','h(5)','h(4)','h(3)', 'p=3'] - -# Backtracking -# ============ -# -# the iterqueue module provides an "extensible" iterator with, e.g., -# an `appendleft` method to push back values:: - -from iterqueue import XIter - -# Thus we can prepend a non-processed data item -# to the data iterator for use by the next state handler -# -# Example 4: A two-state machine with generators and backtracking -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# Again we start from the `SimpleStates2` base class:: - -class Example4(SimpleStates2): - """two-state machine with generators and backtracking - """ - -# Let the iterator wrap the data in an XIter instance with `appendleft` -# method:: - - def __iter__(self): - """Generate and return an iterator - - * convert `data` to an iterator queue - * convert the state generators into iterators - * (re) set the state_handler attribute to the init-state - * pass control to the active states state_handler - which should call and process self.data_iterator.next() - """ - self.data_iterator = XIter(self.data) # queue with `appendleft` method - self.high = self.high_handler_generator().next - self.low = self.low_handler_generator().next - self.state_handler = getattr(self, self.init_state) - # now start the iteration - while True: - yield self.state_handler() - -# Add state method generators that use the "backtracking" feature:: - - def high_handler_generator(self): - """Return an iterator, whose next() method - returns "h(token)" and switches to `low`, if token > 3 - """ - for token in self.data_iterator: - # print "high got", token - if token <= 3: - # push back data token - self.data_iterator.appendleft(token) - # set the new state - self.state_handler = self.low - # return non-value indicating the state switch - yield None - else: - yield "h(%d)"%token - # - def low_handler_generator(self): - """Return an iterator, whose next() method - returns "l(token)" and switches to `high`, if token <=3 - """ - for token in self.data_iterator: - # print "low got", token - if token > 3: - self.data_iterator.appendleft(token) # push back - # set and run the new state - self.state_handler = self.high - # alternatively, return the token processed by the new - # state handler - yield self.state_handler() - else: - yield "l(%d)"%token - -# The `__str__` converter should ignore the switch-indicator:: - - def __str__(self): - tokens = [token for token in self() if token != None] - return " ".join(tokens) - -# Test -# ------- -# -# Again define a test function. This time with an instance of the Example4 -# class :: - -def test_Example4(): - statemachine = Example4(testdata, init_state='low') - -# Calling statemachine() should iterate over the test data and return the -# processed values as list. If the state of the machine changes, the special -# "non-value" `None` is returned. :: - - print statemachine() # only printed if something goes wrong - assert statemachine() == ['l(1)', 'l(2)', 'l(3)', - 'h(4)', 'h(5)', 'h(4)', None, # switch indicator - 'l(3)', 'l(2)', 'l(1)'] - -# Converting to a string should skip the `None` values:: - - print statemachine - assert str(statemachine) == "l(1) l(2) l(3) h(4) h(5) h(4) l(3) l(2) l(1)" - -# Discussion -# ---------- -# -# The `XIter` class allows backtracking also in a state machine with state -# handlers acting on a common iterator object. The "high" and "low" handlers -# demonstrate two possible actions for the state-transition with backtrack: -# Either call the new state handler from the current one -# (like the `low_handler_generator`) or return a "non-value" that signifies -# that processing the data token did not produce any output data. -# -# Using generators made the state handlers shorter and (once the concept of a -# generator is clear) easier. Further advantages of the generator concept are -# -# * internal variables are easily saved over subsequent invocations -# * no function-call overhead (not relevant in this example but maybe for a -# state machine that has to process long data lists. -# -# -# Converting all state method generators with a generic function -# ============================================================== -# -# In `Example4`, we had to redefine the `__iter__` method to convert the -# method state generators into iterators. It would be nice if this could be -# done in the base class. -# -# `SimpleStates3` adds a generic function for this task that relies on a -# simple naming convention: functions whose name matches -# `_handler_generator` should be converted to iterators and their -# `.next()` method stored as ``. -# :: - -class SimpleStates5(SimpleStates2): - """generic state machine acting on iterable data - """ - def _initialize_state_generators(self): - """Generic function to initialise state handlers from generators - - functions whose name matches `[^_]_handler_generator` should - be converted to iterators and their `.next()` method stored as - ``. - """ - suffix = "_handler_generator" - shg_names = [name for name in dir(self) - if name.endswith(suffix) - and not name.startswith("_")] - for name in shg_names: - shg = getattr(self, name) - print shg - setattr(self, name[:-len(suffix)], shg().next) - - - def __iter__(self): - """Generate and return an iterator - - * convert `data` to an iterator queue - * convert the state generators into iterators - * (re) set the state_handler attribute to the init-state - * pass control to the active states state_handler - which should call and process self.data_iterator.next() - """ - self.data_iterator = XIter(self.data) # queue with `appendleft` method - self._initialize_state_generators() - self.state_handler = getattr(self, self.init_state) - # now start the iteration - while True: - yield self.state_handler() - -# Example 5 -# ~~~~~~~~~ -# -# The next example combines the state handlers from Example 4 and the new -# class.:: - -class Example5(Example4, SimpleStates5): - """one more example""" - pass - -# Test -# ---- -# -# A function that has the generator-suffix but is prefixed with an underscore, -# should be skipped by the `_initialize_state_generators` method:: - -class Test_SimpleStates5: - E5 = Example5(testdata) - E5._bogus_handler_generator = "low" - def test_initialize_state_generators(self): - self.E5._initialize_state_generators() - -# A test function. This time with an instance of the Example5 class :: - -def test_Example5(): - statemachine = Example5(testdata, init_state='low') - print statemachine.__dict__ - -# Calling statemachine() should iterate over the test data and return the -# processed values as list. If the state of the machine changes, the special -# "non-value" `None` is returned. :: - - print statemachine() # only printed if something goes wrong - assert statemachine() == ['l(1)', 'l(2)', 'l(3)', - 'h(4)', 'h(5)', 'h(4)', None, # switch indicator - 'l(3)', 'l(2)', 'l(1)'] - -# Converting to a string should skip the `None` values:: - - print statemachine - assert str(statemachine) == "l(1) l(2) l(3) h(4) h(5) h(4) l(3) l(2) l(1)" - -# Putting it together -# =================== -# -# The file `simplestates.py` contains the full definition of the `SimpleStates5` -# class in a self-contained version. -# -# Example 6 -# ~~~~~~~~~ -# -# The final Example is used to test whether we have put it together well. It -# subclasses SimpleStates and adds state method generators for "high" and -# "low":: - -import simplestates -class Example6(simplestates.SimpleStates): - """two-state machine with generators and backtracking - """ - def high_handler_generator(self): - """Return an iterator, whose next() method - returns "h(token)" and switches to `low`, if token > 3 - """ - for token in self.data_iterator: - # print "high got", token - if token <= 3: - # push back data token - self.data_iterator.appendleft(token) - # set the new state - self.state_handler = self.low - # return the token processed by the new state handler - yield self.state_handler() - else: - yield "h(%d)"%token - # - def low_handler_generator(self): - """Return an iterator, whose next() method - returns "l(token)" and switches to `high`, if token <=3 - """ - for token in self.data_iterator: - # print "low got", token - if token > 3: - self.data_iterator.appendleft(token) # push back - # set and run the new state - self.state_handler = self.high - # return the token processed by the new state handler - yield self.state_handler() - else: - yield "l(%d)"%token - -# Test -# ---- -# -# In order not to make it dependent on the iterqueue module, the final -# `SimpleStates` does not wrap the data in an XIter instance. This step should -# be done at the instantiation of a state machine. :: - -def test_Example5(): - statemachine = Example5(XIter(testdata), init_state='low') - print statemachine.__dict__ - -# Calling statemachine() should iterate over the test data and return the -# processed values as list:: - - print statemachine() # only printed if something goes wrong - # reset the data iterator as it is "used up" now - statemachine.data = XIter(testdata) - assert statemachine() == ['l(1)', 'l(2)', 'l(3)', - 'h(4)', 'h(5)', 'h(4)', None, - 'l(3)', 'l(2)', 'l(1)'] - -# Index -# ===== -# -# -# :_`generator`: A function with a `yield` keyword. Calling this function will -# return an iterator_ -# -# :_`iterator`: An object with a `next()` method. Calling `.next()` -# will (typically) return one data token (list element, line in -# a file, ...). If there is no more data the `StopIteration` -# exception is raised. -# -# Command line usage -# ================== -# -# running this script should explore it in the "nose" test framework:: - -if __name__ == "__main__": - import nose, doctest - # first run any doctests - doctest.testmod() - # then run nose on this module - nose.runmodule() # requires nose 0.9.1 diff --git a/rstdocs/examples/testfile_literate.py b/rstdocs/examples/testfile_literate.py deleted file mode 100644 index 3411ae0..0000000 --- a/rstdocs/examples/testfile_literate.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -# -*- coding: iso-8859-1 -*- - -# :Date: $Date$ -# :Version: SVN-Revision $Revision$ -# :URL: $URL: $ -# :Copyright: 2006 Guenter Milde. -# Released without any warranty under the terms of the -# GNU General Public License (v. 2 or later) -# -# testfile_literate -# ================= -# -# -# Import -# ------ -# -# As this module is not loaded when the file is tested with -# ``pylit --doctest``, the first doctest must import it before any of its -# objects can be used. -# An elegant solution is to give a usage example in the module's docstring:: - -""" -This is the "testfile_literate" module. - -It supplies one function, `factorial()` that returns the factorial of its -argument, e.g.: - ->>> import testfile_literate ->>> testfile_literate.factorial(5) -120 - -""" - -__docformat__ = 'restructuredtext' - - -# :Beware: as the docstring is not parsed as separate unit but as part of the -# file, there must be a blank line also after the last doctest block. -# Otherwise `doctest` expects ``"""`` to be part of the output. -# -# Alternatives for easier access to the defined objects are: -# -# >>> import testfile_literate as fac -# >>> fac.factorial(5) -# 120 -# -# >>> from testfile_literate import * -# >>> factorial(5) -# 120 -# -# This imports the *code source* of the literal script: -# -# * ``testfile_literate.py`` must be present in the PYTHONPATH or the current -# working directory. -# -# * The doctest examples in the file argument to ``pylit --doctest`` (be it -# text source or code source) are tested with the code definitions in the -# last saved version of the code source. -# -# -# factorial -# --------- -# -# The functions docstring can be kept concise and additional discussion -# reffered to the text part of the literate source:: - -def factorial(n): - """Return the factorial of `n`, an exact integer >= 0. - - >>> [factorial(n) for n in range(6)] - [1, 1, 2, 6, 24, 120] - >>> factorial(30) - 265252859812191058636308480000000L - - Factorials of floats are OK, but the float must be an exact integer: - - >>> factorial(30.0) - 265252859812191058636308480000000L - - """ - - import math - if not n >= 0: - raise ValueError("n must be >= 0") - if math.floor(n) != n: - raise ValueError("n must be exact integer") - if n+1 == n: # catch a value like 1e300 - raise OverflowError("n too large") - result = 1 - factor = 2 - while factor <= n: - result *= factor - factor += 1 - return result - - -# Discussion and test -# ~~~~~~~~~~~~~~~~~~~ -# -# `factorial()` accepts input as int, long or float: -# -# >>> factorial(30) -# 265252859812191058636308480000000L -# >>> factorial(30L) -# 265252859812191058636308480000000L -# >>> factorial(30.0) -# 265252859812191058636308480000000L -# -# However, the float must be an exact integer and it must also not be -# ridiculously large: -# -# >>> factorial(30.1) -# Traceback (most recent call last): -# ... -# ValueError: n must be exact integer -# -# >>> factorial(1e100) -# Traceback (most recent call last): -# ... -# OverflowError: n too large -# -# The factorial of neagive values is not defined: -# -# >>> factorial(-1) -# Traceback (most recent call last): -# ... -# ValueError: n must be >= 0 -# -# The type of the return value depends on the size of the result. -# -# If the result is small enough to fit in an int, return an int. -# Else return a long: -# -# >>> [factorial(n) for n in range(6)] -# [1, 1, 2, 6, 24, 120] -# >>> [factorial(long(n)) for n in range(6)] -# [1, 1, 2, 6, 24, 120] -# >>> factorial(30) -# 265252859812191058636308480000000L -# >>> factorial(30L) -# 265252859812191058636308480000000L -# -# -# Default Action -# -------------- -# -# The script is tested by calling ``pylit --doctest testfile_literate``. -# -# This is especially handy for scripts that should performe some default -# action other than a self test, e.g. -# -# Print the first 10 natural numbers and their factorials if -# called as `__main__` (i.e. from the command line):: - -if __name__ == "__main__": - print "n n!" - for n in range(10): - print n, factorial(n) - - diff --git a/rstdocs/examples/testmod_literate.py b/rstdocs/examples/testmod_literate.py deleted file mode 100644 index 491ebba..0000000 --- a/rstdocs/examples/testmod_literate.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python -# -*- coding: iso-8859-1 -*- - -# :Date: $Date$ -# :Version: SVN-Revision $Revision$ -# :URL: $URL: $ -# :Copyright: 2006 Guenter Milde. -# Released without any warranty under the terms of the -# GNU General Public License (v. 2 or later) -# -# testmod_literate -# ================ -# -# The module docstring should give a concise description of the working, -# details are in the literate source so the docstrings are not bloated:: - -""" -This is the "testmod_literate" module. - -It supplies one function, `factorial()`. For example, - ->>> factorial(5) -120 - -""" - -__docformat__ = 'restructuredtext' - - -# **Beware:** as the docstring is not parsed as separate unit but as part of -# the file, there must be a blank line also after the last doctest block. -# Otherwise `doctest` expects ``"""`` to be part of the output. -# -# -# factorial -# --------- -# The functions docstring can be kept concise and additional discussion -# reffered to the text part of the literate source:: - -def factorial(n): - """Return the factorial of `n`, an exact integer >= 0. - - >>> [factorial(n) for n in range(6)] - [1, 1, 2, 6, 24, 120] - >>> factorial(30) - 265252859812191058636308480000000L - - Factorials of floats are OK, but the float must be an exact integer: - - >>> factorial(30.0) - 265252859812191058636308480000000L - - """ - - import math - if not n >= 0: - raise ValueError("n must be >= 0") - if math.floor(n) != n: - raise ValueError("n must be exact integer") - if n+1 == n: # catch a value like 1e300 - raise OverflowError("n too large") - result = 1 - factor = 2 - while factor <= n: - result *= factor - factor += 1 - return result - - -# Discussion and test -# ~~~~~~~~~~~~~~~~~~~ -# -# `factorial()` accepts input as int, long or float: -# -# >>> factorial(30) -# 265252859812191058636308480000000L -# >>> factorial(30L) -# 265252859812191058636308480000000L -# >>> factorial(30.0) -# 265252859812191058636308480000000L -# -# However, the float must be an exact integer and it must also not be -# ridiculously large: -# -# >>> factorial(30.1) -# Traceback (most recent call last): -# ... -# ValueError: n must be exact integer -# -# >>> factorial(1e100) -# Traceback (most recent call last): -# ... -# OverflowError: n too large -# -# The factorial of neagive values is not defined: -# -# >>> factorial(-1) -# Traceback (most recent call last): -# ... -# ValueError: n must be >= 0 -# -# The type of the return value depends on the size of the result. -# -# If the result is small enough to fit in an int, return an int. -# Else return a long: -# -# >>> [factorial(n) for n in range(6)] -# [1, 1, 2, 6, 24, 120] -# >>> [factorial(long(n)) for n in range(6)] -# [1, 1, 2, 6, 24, 120] -# >>> factorial(30) -# 265252859812191058636308480000000L -# >>> factorial(30L) -# 265252859812191058636308480000000L -# -# -# Self Test -# --------- -# -# The traditional test function parses the docstrings of all objects in this -# module. It misses doctests in comments:: - -def _test(): - import doctest - doctest.testmod() - -# Test all doctest blocks (both in docstrings and in text parts (well -# formatted comments) if the module is called as `__main__` (i.e. from the -# command line):: - -def _test_all_doctests(): - import pylit, sys - pylit.run_doctest(infile=sys.argv[0], txt2code=False, - globs=sys.modules.get('__main__').__dict__) - -# Doctests can still be disabled or commented - make sure they are not -# recognized as text block (no double colon here): -# -# # a silly doctest -# # >>> False -# # True -# -# or (with non-canonical comments):: - -# a silly doctest -#>>> False -#True - -# Do a self test:: - -if __name__ == "__main__": - #_test() - _test_all_doctests() - -- 2.11.4.GIT