Add accounts tempaltes
[frozenviper.git] / Django-1.2.1 / django / contrib / gis / utils / layermapping.py
blobcec198979683baa5c2bc6dc5c3444aeb666c97ed
1 # LayerMapping -- A Django Model/OGR Layer Mapping Utility
2 """
3 The LayerMapping class provides a way to map the contents of OGR
4 vector files (e.g. SHP files) to Geographic-enabled Django models.
6 For more information, please consult the GeoDjango documentation:
7 http://geodjango.org/docs/layermapping.html
8 """
9 import sys
10 from datetime import date, datetime
11 from decimal import Decimal
12 from django.core.exceptions import ObjectDoesNotExist
13 from django.db import connections, DEFAULT_DB_ALIAS
14 from django.contrib.gis.db.models import GeometryField
15 from django.contrib.gis.gdal import CoordTransform, DataSource, \
16 OGRException, OGRGeometry, OGRGeomType, SpatialReference
17 from django.contrib.gis.gdal.field import \
18 OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime
19 from django.db import models, transaction
20 from django.contrib.localflavor.us.models import USStateField
22 # LayerMapping exceptions.
23 class LayerMapError(Exception): pass
24 class InvalidString(LayerMapError): pass
25 class InvalidDecimal(LayerMapError): pass
26 class InvalidInteger(LayerMapError): pass
27 class MissingForeignKey(LayerMapError): pass
29 class LayerMapping(object):
30 "A class that maps OGR Layers to GeoDjango Models."
32 # Acceptable 'base' types for a multi-geometry type.
33 MULTI_TYPES = {1 : OGRGeomType('MultiPoint'),
34 2 : OGRGeomType('MultiLineString'),
35 3 : OGRGeomType('MultiPolygon'),
36 OGRGeomType('Point25D').num : OGRGeomType('MultiPoint25D'),
37 OGRGeomType('LineString25D').num : OGRGeomType('MultiLineString25D'),
38 OGRGeomType('Polygon25D').num : OGRGeomType('MultiPolygon25D'),
41 # Acceptable Django field types and corresponding acceptable OGR
42 # counterparts.
43 FIELD_TYPES = {
44 models.AutoField : OFTInteger,
45 models.IntegerField : (OFTInteger, OFTReal, OFTString),
46 models.FloatField : (OFTInteger, OFTReal),
47 models.DateField : OFTDate,
48 models.DateTimeField : OFTDateTime,
49 models.EmailField : OFTString,
50 models.TimeField : OFTTime,
51 models.DecimalField : (OFTInteger, OFTReal),
52 models.CharField : OFTString,
53 models.SlugField : OFTString,
54 models.TextField : OFTString,
55 models.URLField : OFTString,
56 USStateField : OFTString,
57 models.XMLField : OFTString,
58 models.SmallIntegerField : (OFTInteger, OFTReal, OFTString),
59 models.PositiveSmallIntegerField : (OFTInteger, OFTReal, OFTString),
62 # The acceptable transaction modes.
63 TRANSACTION_MODES = {'autocommit' : transaction.autocommit,
64 'commit_on_success' : transaction.commit_on_success,
67 def __init__(self, model, data, mapping, layer=0,
68 source_srs=None, encoding=None,
69 transaction_mode='commit_on_success',
70 transform=True, unique=None, using=DEFAULT_DB_ALIAS):
71 """
72 A LayerMapping object is initialized using the given Model (not an instance),
73 a DataSource (or string path to an OGR-supported data file), and a mapping
74 dictionary. See the module level docstring for more details and keyword
75 argument usage.
76 """
77 # Getting the DataSource and the associated Layer.
78 if isinstance(data, basestring):
79 self.ds = DataSource(data)
80 else:
81 self.ds = data
82 self.layer = self.ds[layer]
84 self.using = using
85 self.spatial_backend = connections[using].ops
87 # Setting the mapping & model attributes.
88 self.mapping = mapping
89 self.model = model
91 # Checking the layer -- intitialization of the object will fail if
92 # things don't check out before hand.
93 self.check_layer()
95 # Getting the geometry column associated with the model (an
96 # exception will be raised if there is no geometry column).
97 if self.spatial_backend.mysql:
98 transform = False
99 else:
100 self.geo_field = self.geometry_field()
102 # Checking the source spatial reference system, and getting
103 # the coordinate transformation object (unless the `transform`
104 # keyword is set to False)
105 if transform:
106 self.source_srs = self.check_srs(source_srs)
107 self.transform = self.coord_transform()
108 else:
109 self.transform = transform
111 # Setting the encoding for OFTString fields, if specified.
112 if encoding:
113 # Making sure the encoding exists, if not a LookupError
114 # exception will be thrown.
115 from codecs import lookup
116 lookup(encoding)
117 self.encoding = encoding
118 else:
119 self.encoding = None
121 if unique:
122 self.check_unique(unique)
123 transaction_mode = 'autocommit' # Has to be set to autocommit.
124 self.unique = unique
125 else:
126 self.unique = None
128 # Setting the transaction decorator with the function in the
129 # transaction modes dictionary.
130 if transaction_mode in self.TRANSACTION_MODES:
131 self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode]
132 self.transaction_mode = transaction_mode
133 else:
134 raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)
136 if using is None:
137 pass
139 #### Checking routines used during initialization ####
140 def check_fid_range(self, fid_range):
141 "This checks the `fid_range` keyword."
142 if fid_range:
143 if isinstance(fid_range, (tuple, list)):
144 return slice(*fid_range)
145 elif isinstance(fid_range, slice):
146 return fid_range
147 else:
148 raise TypeError
149 else:
150 return None
152 def check_layer(self):
154 This checks the Layer metadata, and ensures that it is compatible
155 with the mapping information and model. Unlike previous revisions,
156 there is no need to increment through each feature in the Layer.
158 # The geometry field of the model is set here.
159 # TODO: Support more than one geometry field / model. However, this
160 # depends on the GDAL Driver in use.
161 self.geom_field = False
162 self.fields = {}
164 # Getting lists of the field names and the field types available in
165 # the OGR Layer.
166 ogr_fields = self.layer.fields
167 ogr_field_types = self.layer.field_types
169 # Function for determining if the OGR mapping field is in the Layer.
170 def check_ogr_fld(ogr_map_fld):
171 try:
172 idx = ogr_fields.index(ogr_map_fld)
173 except ValueError:
174 raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld)
175 return idx
177 # No need to increment through each feature in the model, simply check
178 # the Layer metadata against what was given in the mapping dictionary.
179 for field_name, ogr_name in self.mapping.items():
180 # Ensuring that a corresponding field exists in the model
181 # for the given field name in the mapping.
182 try:
183 model_field = self.model._meta.get_field(field_name)
184 except models.fields.FieldDoesNotExist:
185 raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name)
187 # Getting the string name for the Django field class (e.g., 'PointField').
188 fld_name = model_field.__class__.__name__
190 if isinstance(model_field, GeometryField):
191 if self.geom_field:
192 raise LayerMapError('LayerMapping does not support more than one GeometryField per model.')
194 # Getting the coordinate dimension of the geometry field.
195 coord_dim = model_field.dim
197 try:
198 if coord_dim == 3:
199 gtype = OGRGeomType(ogr_name + '25D')
200 else:
201 gtype = OGRGeomType(ogr_name)
202 except OGRException:
203 raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name)
205 # Making sure that the OGR Layer's Geometry is compatible.
206 ltype = self.layer.geom_type
207 if not (ltype.name.startswith(gtype.name) or self.make_multi(ltype, model_field)):
208 raise LayerMapError('Invalid mapping geometry; model has %s%s, '
209 'layer geometry type is %s.' %
210 (fld_name, (coord_dim == 3 and '(dim=3)') or '', ltype))
212 # Setting the `geom_field` attribute w/the name of the model field
213 # that is a Geometry. Also setting the coordinate dimension
214 # attribute.
215 self.geom_field = field_name
216 self.coord_dim = coord_dim
217 fields_val = model_field
218 elif isinstance(model_field, models.ForeignKey):
219 if isinstance(ogr_name, dict):
220 # Is every given related model mapping field in the Layer?
221 rel_model = model_field.rel.to
222 for rel_name, ogr_field in ogr_name.items():
223 idx = check_ogr_fld(ogr_field)
224 try:
225 rel_field = rel_model._meta.get_field(rel_name)
226 except models.fields.FieldDoesNotExist:
227 raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' %
228 (rel_name, rel_model.__class__.__name__))
229 fields_val = rel_model
230 else:
231 raise TypeError('ForeignKey mapping must be of dictionary type.')
232 else:
233 # Is the model field type supported by LayerMapping?
234 if not model_field.__class__ in self.FIELD_TYPES:
235 raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name)
237 # Is the OGR field in the Layer?
238 idx = check_ogr_fld(ogr_name)
239 ogr_field = ogr_field_types[idx]
241 # Can the OGR field type be mapped to the Django field type?
242 if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]):
243 raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' %
244 (ogr_field, ogr_field.__name__, fld_name))
245 fields_val = model_field
247 self.fields[field_name] = fields_val
249 def check_srs(self, source_srs):
250 "Checks the compatibility of the given spatial reference object."
252 if isinstance(source_srs, SpatialReference):
253 sr = source_srs
254 elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()):
255 sr = source_srs.srs
256 elif isinstance(source_srs, (int, basestring)):
257 sr = SpatialReference(source_srs)
258 else:
259 # Otherwise just pulling the SpatialReference from the layer
260 sr = self.layer.srs
262 if not sr:
263 raise LayerMapError('No source reference system defined.')
264 else:
265 return sr
267 def check_unique(self, unique):
268 "Checks the `unique` keyword parameter -- may be a sequence or string."
269 if isinstance(unique, (list, tuple)):
270 # List of fields to determine uniqueness with
271 for attr in unique:
272 if not attr in self.mapping: raise ValueError
273 elif isinstance(unique, basestring):
274 # Only a single field passed in.
275 if unique not in self.mapping: raise ValueError
276 else:
277 raise TypeError('Unique keyword argument must be set with a tuple, list, or string.')
279 #### Keyword argument retrieval routines ####
280 def feature_kwargs(self, feat):
282 Given an OGR Feature, this will return a dictionary of keyword arguments
283 for constructing the mapped model.
285 # The keyword arguments for model construction.
286 kwargs = {}
288 # Incrementing through each model field and OGR field in the
289 # dictionary mapping.
290 for field_name, ogr_name in self.mapping.items():
291 model_field = self.fields[field_name]
293 if isinstance(model_field, GeometryField):
294 # Verify OGR geometry.
295 val = self.verify_geom(feat.geom, model_field)
296 elif isinstance(model_field, models.base.ModelBase):
297 # The related _model_, not a field was passed in -- indicating
298 # another mapping for the related Model.
299 val = self.verify_fk(feat, model_field, ogr_name)
300 else:
301 # Otherwise, verify OGR Field type.
302 val = self.verify_ogr_field(feat[ogr_name], model_field)
304 # Setting the keyword arguments for the field name with the
305 # value obtained above.
306 kwargs[field_name] = val
308 return kwargs
310 def unique_kwargs(self, kwargs):
312 Given the feature keyword arguments (from `feature_kwargs`) this routine
313 will construct and return the uniqueness keyword arguments -- a subset
314 of the feature kwargs.
316 if isinstance(self.unique, basestring):
317 return {self.unique : kwargs[self.unique]}
318 else:
319 return dict((fld, kwargs[fld]) for fld in self.unique)
321 #### Verification routines used in constructing model keyword arguments. ####
322 def verify_ogr_field(self, ogr_field, model_field):
324 Verifies if the OGR Field contents are acceptable to the Django
325 model field. If they are, the verified value is returned,
326 otherwise the proper exception is raised.
328 if (isinstance(ogr_field, OFTString) and
329 isinstance(model_field, (models.CharField, models.TextField))):
330 if self.encoding:
331 # The encoding for OGR data sources may be specified here
332 # (e.g., 'cp437' for Census Bureau boundary files).
333 val = unicode(ogr_field.value, self.encoding)
334 else:
335 val = ogr_field.value
336 if len(val) > model_field.max_length:
337 raise InvalidString('%s model field maximum string length is %s, given %s characters.' %
338 (model_field.name, model_field.max_length, len(val)))
339 elif isinstance(ogr_field, OFTReal) and isinstance(model_field, models.DecimalField):
340 try:
341 # Creating an instance of the Decimal value to use.
342 d = Decimal(str(ogr_field.value))
343 except:
344 raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field.value)
346 # Getting the decimal value as a tuple.
347 dtup = d.as_tuple()
348 digits = dtup[1]
349 d_idx = dtup[2] # index where the decimal is
351 # Maximum amount of precision, or digits to the left of the decimal.
352 max_prec = model_field.max_digits - model_field.decimal_places
354 # Getting the digits to the left of the decimal place for the
355 # given decimal.
356 if d_idx < 0:
357 n_prec = len(digits[:d_idx])
358 else:
359 n_prec = len(digits) + d_idx
361 # If we have more than the maximum digits allowed, then throw an
362 # InvalidDecimal exception.
363 if n_prec > max_prec:
364 raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' %
365 (model_field.max_digits, model_field.decimal_places, max_prec))
366 val = d
367 elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(model_field, models.IntegerField):
368 # Attempt to convert any OFTReal and OFTString value to an OFTInteger.
369 try:
370 val = int(ogr_field.value)
371 except:
372 raise InvalidInteger('Could not construct integer from: %s' % ogr_field.value)
373 else:
374 val = ogr_field.value
375 return val
377 def verify_fk(self, feat, rel_model, rel_mapping):
379 Given an OGR Feature, the related model and its dictionary mapping,
380 this routine will retrieve the related model for the ForeignKey
381 mapping.
383 # TODO: It is expensive to retrieve a model for every record --
384 # explore if an efficient mechanism exists for caching related
385 # ForeignKey models.
387 # Constructing and verifying the related model keyword arguments.
388 fk_kwargs = {}
389 for field_name, ogr_name in rel_mapping.items():
390 fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name))
392 # Attempting to retrieve and return the related model.
393 try:
394 return rel_model.objects.get(**fk_kwargs)
395 except ObjectDoesNotExist:
396 raise MissingForeignKey('No ForeignKey %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs))
398 def verify_geom(self, geom, model_field):
400 Verifies the geometry -- will construct and return a GeometryCollection
401 if necessary (for example if the model field is MultiPolygonField while
402 the mapped shapefile only contains Polygons).
404 # Downgrade a 3D geom to a 2D one, if necessary.
405 if self.coord_dim != geom.coord_dim:
406 geom.coord_dim = self.coord_dim
408 if self.make_multi(geom.geom_type, model_field):
409 # Constructing a multi-geometry type to contain the single geometry
410 multi_type = self.MULTI_TYPES[geom.geom_type.num]
411 g = OGRGeometry(multi_type)
412 g.add(geom)
413 else:
414 g = geom
416 # Transforming the geometry with our Coordinate Transformation object,
417 # but only if the class variable `transform` is set w/a CoordTransform
418 # object.
419 if self.transform: g.transform(self.transform)
421 # Returning the WKT of the geometry.
422 return g.wkt
424 #### Other model methods ####
425 def coord_transform(self):
426 "Returns the coordinate transformation object."
427 SpatialRefSys = self.spatial_backend.spatial_ref_sys()
428 try:
429 # Getting the target spatial reference system
430 target_srs = SpatialRefSys.objects.get(srid=self.geo_field.srid).srs
432 # Creating the CoordTransform object
433 return CoordTransform(self.source_srs, target_srs)
434 except Exception, msg:
435 raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg)
437 def geometry_field(self):
438 "Returns the GeometryField instance associated with the geographic column."
439 # Use the `get_field_by_name` on the model's options so that we
440 # get the correct field instance if there's model inheritance.
441 opts = self.model._meta
442 fld, model, direct, m2m = opts.get_field_by_name(self.geom_field)
443 return fld
445 def make_multi(self, geom_type, model_field):
447 Given the OGRGeomType for a geometry and its associated GeometryField,
448 determine whether the geometry should be turned into a GeometryCollection.
450 return (geom_type.num in self.MULTI_TYPES and
451 model_field.__class__.__name__ == 'Multi%s' % geom_type.django)
453 def save(self, verbose=False, fid_range=False, step=False,
454 progress=False, silent=False, stream=sys.stdout, strict=False):
456 Saves the contents from the OGR DataSource Layer into the database
457 according to the mapping dictionary given at initialization.
459 Keyword Parameters:
460 verbose:
461 If set, information will be printed subsequent to each model save
462 executed on the database.
464 fid_range:
465 May be set with a slice or tuple of (begin, end) feature ID's to map
466 from the data source. In other words, this keyword enables the user
467 to selectively import a subset range of features in the geographic
468 data source.
470 step:
471 If set with an integer, transactions will occur at every step
472 interval. For example, if step=1000, a commit would occur after
473 the 1,000th feature, the 2,000th feature etc.
475 progress:
476 When this keyword is set, status information will be printed giving
477 the number of features processed and sucessfully saved. By default,
478 progress information will pe printed every 1000 features processed,
479 however, this default may be overridden by setting this keyword with an
480 integer for the desired interval.
482 stream:
483 Status information will be written to this file handle. Defaults to
484 using `sys.stdout`, but any object with a `write` method is supported.
486 silent:
487 By default, non-fatal error notifications are printed to stdout, but
488 this keyword may be set to disable these notifications.
490 strict:
491 Execution of the model mapping will cease upon the first error
492 encountered. The default behavior is to attempt to continue.
494 # Getting the default Feature ID range.
495 default_range = self.check_fid_range(fid_range)
497 # Setting the progress interval, if requested.
498 if progress:
499 if progress is True or not isinstance(progress, int):
500 progress_interval = 1000
501 else:
502 progress_interval = progress
504 # Defining the 'real' save method, utilizing the transaction
505 # decorator created during initialization.
506 @self.transaction_decorator
507 def _save(feat_range=default_range, num_feat=0, num_saved=0):
508 if feat_range:
509 layer_iter = self.layer[feat_range]
510 else:
511 layer_iter = self.layer
513 for feat in layer_iter:
514 num_feat += 1
515 # Getting the keyword arguments
516 try:
517 kwargs = self.feature_kwargs(feat)
518 except LayerMapError, msg:
519 # Something borked the validation
520 if strict: raise
521 elif not silent:
522 stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))
523 else:
524 # Constructing the model using the keyword args
525 is_update = False
526 if self.unique:
527 # If we want unique models on a particular field, handle the
528 # geometry appropriately.
529 try:
530 # Getting the keyword arguments and retrieving
531 # the unique model.
532 u_kwargs = self.unique_kwargs(kwargs)
533 m = self.model.objects.using(self.using).get(**u_kwargs)
534 is_update = True
536 # Getting the geometry (in OGR form), creating
537 # one from the kwargs WKT, adding in additional
538 # geometries, and update the attribute with the
539 # just-updated geometry WKT.
540 geom = getattr(m, self.geom_field).ogr
541 new = OGRGeometry(kwargs[self.geom_field])
542 for g in new: geom.add(g)
543 setattr(m, self.geom_field, geom.wkt)
544 except ObjectDoesNotExist:
545 # No unique model exists yet, create.
546 m = self.model(**kwargs)
547 else:
548 m = self.model(**kwargs)
550 try:
551 # Attempting to save.
552 m.save(using=self.using)
553 num_saved += 1
554 if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m))
555 except SystemExit:
556 raise
557 except Exception, msg:
558 if self.transaction_mode == 'autocommit':
559 # Rolling back the transaction so that other model saves
560 # will work.
561 transaction.rollback_unless_managed()
562 if strict:
563 # Bailing out if the `strict` keyword is set.
564 if not silent:
565 stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid)
566 stream.write('%s\n' % kwargs)
567 raise
568 elif not silent:
569 stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))
571 # Printing progress information, if requested.
572 if progress and num_feat % progress_interval == 0:
573 stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved))
575 # Only used for status output purposes -- incremental saving uses the
576 # values returned here.
577 return num_saved, num_feat
579 nfeat = self.layer.num_feat
580 if step and isinstance(step, int) and step < nfeat:
581 # Incremental saving is requested at the given interval (step)
582 if default_range:
583 raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.')
584 beg, num_feat, num_saved = (0, 0, 0)
585 indices = range(step, nfeat, step)
586 n_i = len(indices)
588 for i, end in enumerate(indices):
589 # Constructing the slice to use for this step; the last slice is
590 # special (e.g, [100:] instead of [90:100]).
591 if i+1 == n_i: step_slice = slice(beg, None)
592 else: step_slice = slice(beg, end)
594 try:
595 num_feat, num_saved = _save(step_slice, num_feat, num_saved)
596 beg = end
597 except:
598 stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice))
599 raise
600 else:
601 # Otherwise, just calling the previously defined _save() function.
602 _save()