2 # -*- coding: utf-8 -*-
4 # Copyright 2004-2006 Zuza Software Foundation
6 # This file is part of translate.
8 # translate is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # translate is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with translate; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 """manages projects and files and translations"""
24 from translate
.storage
import factory
25 from translate
.filters
import checks
26 from translate
.convert
import po2csv
27 from translate
.convert
import po2xliff
28 from translate
.convert
import xliff2po
29 from translate
.convert
import po2ts
30 from translate
.convert
import pot2po
31 from translate
.convert
import po2oo
32 from translate
.tools
import pocompile
33 from translate
.tools
import pogrep
34 from translate
.search
import match
35 from translate
.search
import indexer
36 from translate
.storage
import statsdb
, base
37 from Pootle
import statistics
38 from Pootle
import pootlefile
39 from translate
.storage
import versioncontrol
40 from jToolkit
import timecache
41 from jToolkit
import prefs
48 class RightsError(ValueError):
51 class InternalAdminSession
:
52 """A fake session used for doing internal admin jobs"""
54 self
.username
= "internal"
57 def localize(self
, message
):
60 def issiteadmin(self
):
63 class potimecache(timecache
.timecache
):
64 """Caches pootlefile objects, remembers time, and reverts back to statistics when necessary..."""
65 def __init__(self
, expiryperiod
, project
):
66 """initialises the cache to keep objects for the given expiryperiod, and point back to the project"""
67 timecache
.timecache
.__init
__(self
, expiryperiod
)
68 self
.project
= project
70 def __getitem__(self
, key
):
71 """[] access of items"""
72 if key
and not dict.__contains
__(self
, key
):
73 popath
= os
.path
.join(self
.project
.podir
, key
)
74 if os
.path
.exists(popath
):
75 # update the index to pofiles...
76 self
.project
.scanpofiles()
77 return timecache
.timecache
.__getitem
__(self
, key
)
79 def expire(self
, pofilename
):
80 """expires the given pofilename by recreating it (holding only stats)"""
81 timestamp
, currentfile
= dict.__getitem
__(self
, pofilename
)
82 if currentfile
.pomtime
is not None:
83 # use the currentfile.pomtime as a timestamp as well, so any modifications will extend its life
84 if time
.time() - currentfile
.pomtime
> self
.expiryperiod
.seconds
:
85 self
.__setitem
__(pofilename
, pootlefile
.pootlefile(self
.project
, pofilename
))
87 class TranslationProject(object):
88 """Manages iterating through the translations in a particular project"""
90 index_directory
= ".translation_index"
92 def __init__(self
, languagecode
, projectcode
, potree
, create
=False):
93 self
.languagecode
= languagecode
94 self
.projectcode
= projectcode
96 self
.languagename
= self
.potree
.getlanguagename(self
.languagecode
)
97 self
.projectname
= self
.potree
.getprojectname(self
.projectcode
)
98 self
.projectdescription
= self
.potree
.getprojectdescription(self
.projectcode
)
99 self
.pofiles
= potimecache(15*60, self
)
100 self
.projectcheckerstyle
= self
.potree
.getprojectcheckerstyle(self
.projectcode
)
101 checkerclasses
= [checks
.projectcheckers
.get(self
.projectcheckerstyle
, checks
.StandardChecker
), checks
.StandardUnitChecker
]
102 self
.checker
= checks
.TeeChecker(checkerclasses
=checkerclasses
, errorhandler
=self
.filtererrorhandler
, languagecode
=languagecode
)
103 self
.fileext
= self
.potree
.getprojectlocalfiletype(self
.projectcode
)
105 # terminology matcher
106 self
.termmatcher
= None
107 self
.termmatchermtime
= None
109 self
.converttemplates(InternalAdminSession())
110 self
.podir
= potree
.getpodir(languagecode
, projectcode
)
111 if self
.potree
.hasgnufiles(self
.podir
, self
.languagecode
) == "gnu":
112 self
.filestyle
= "gnu"
114 self
.filestyle
= "std"
117 self
.readquickstats()
121 """reads the project preferences"""
122 self
.prefs
= prefs
.PrefsParser()
123 self
.prefsfile
= os
.path
.join(self
.podir
, "pootle-%s-%s.prefs" % (self
.projectcode
, self
.languagecode
))
124 if not os
.path
.exists(self
.prefsfile
):
125 prefsfile
= open(self
.prefsfile
, "w")
126 prefsfile
.write("# Pootle preferences for project %s, language %s\n\n" % (self
.projectcode
, self
.languagecode
))
128 self
.prefs
.parsefile(self
.prefsfile
)
131 """saves the project preferences"""
132 self
.prefs
.savefile()
134 def getrightnames(self
, session
):
135 """gets the available rights and their localized names"""
136 localize
= session
.localize
138 return [("view", localize("View")),
139 ("suggest", localize("Suggest")),
140 ("translate", localize("Translate")),
141 ("overwrite", localize("Overwrite")),
143 ("review", localize("Review")),
145 ("archive", localize("Archive")),
146 # l10n: This refers to generating the binary .mo file
147 ("pocompile", localize("Compile PO files")),
148 ("assign", localize("Assign")),
149 ("admin", localize("Administrate")),
150 ("commit", localize("Commit")),
153 def getrights(self
, session
=None, username
=None, usedefaults
=True):
154 """gets the rights for the given user (name or session, or not-logged-in if username is None)
155 if usedefaults is False then None will be returned if no rights are defined (useful for editing rights)"""
156 # internal admin sessions have all rights
157 if isinstance(session
, InternalAdminSession
):
158 return [right
for right
, localizedright
in self
.getrightnames(session
)]
159 if session
is not None and session
.isopen
and username
is None:
160 username
= session
.username
164 rightstree
= getattr(self
.prefs
, "rights", None)
165 if rightstree
is not None:
166 if rightstree
.__hasattr
__(username
):
167 rights
= rightstree
.__getattr
__(username
)
172 if username
== "nobody":
174 elif rightstree
is None:
175 if self
.languagecode
== "en":
176 rights
= "view, archive, pocompile"
178 rights
= self
.potree
.getdefaultrights()
180 rights
= getattr(rightstree
, "default", None)
183 rights
= [right
.strip() for right
in rights
.split(",")]
184 if session
is not None and session
.issiteadmin():
185 if "admin" not in rights
:
186 rights
.append("admin")
189 def getuserswithinterest(self
, session
):
190 """returns all the users who registered for this language and project"""
192 def usableuser(user
, userprefs
):
193 if user
in ["__dummy__", "default", "nobody"]:
195 return self
.languagecode
in getattr(userprefs
, "languages", [])
198 for username
, userprefs
in session
.loginchecker
.users
.iteritems():
199 if usableuser(username
, userprefs
):
200 # Let's build a nice descriptive name for use in the interface. It will
201 # contain both the username and the full name, if available.
202 name
= getattr(userprefs
, "name", None)
204 description
= "%s (%s)" % (name
, username
)
206 description
= username
207 setattr(userprefs
, "description", description
)
208 users
[username
] = userprefs
211 def getuserswithrights(self
):
212 """gets all users that have rights defined for this project"""
213 return [username
for username
, user_rights
in getattr(self
.prefs
, "rights", {}).iteritems()]
215 def setrights(self
, username
, rights
):
216 """sets the rights for the given username... (or not-logged-in if username is None)"""
217 if username
is None: username
= "nobody"
218 if isinstance(rights
, list):
219 rights
= ", ".join(rights
)
220 if not hasattr(self
.prefs
, "rights"):
221 self
.prefs
.rights
= prefs
.PrefNode(self
.prefs
, "rights")
222 self
.prefs
.rights
.__setattr
__(username
, rights
)
225 def delrights(self
, session
, username
):
226 """deletes teh rights for the given username"""
227 # l10n: Don't translate "nobody" or "default"
228 if username
== "nobody" or username
== "default":
229 # l10n: Don't translate "nobody" or "default"
230 raise RightsError(session
.localize('You cannot remove the "nobody" or "default" user'))
231 self
.prefs
.rights
.__delattr
__(username
)
234 def getgoalnames(self
):
235 """gets the goals and associated files for the project"""
236 goals
= getattr(self
.prefs
, "goals", {})
238 for goalname
, goalnode
in goals
.iteritems():
239 goallist
.append(goalname
.decode("utf-8"))
244 """gets the goal, goalnode tuples"""
245 goals
= getattr(self
.prefs
, "goals", {})
247 for goalname
, goalnode
in goals
.iteritems():
248 newgoals
[goalname
.decode("utf-8")] = goalnode
251 def getgoalfiles(self
, goalname
, dirfilter
=None, maxdepth
=None, includedirs
=True, expanddirs
=False, includepartial
=False):
252 """gets the files for the given goal, with many options!
253 dirfilter limits to files in a certain subdirectory
254 maxdepth limits to files / directories of a certain depth
255 includedirs specifies whether to return directory names
256 expanddirs specifies whether to expand directories and return all files in them
257 includepartial specifies whether to return directories that are not in the goal, but have files below maxdepth in the goal"""
259 return self
.getnogoalfiles(dirfilter
=dirfilter
, maxdepth
=maxdepth
, includedirs
=includedirs
, expanddirs
=expanddirs
)
260 goals
= getattr(self
.prefs
, "goals", {})
261 poext
= os
.extsep
+ self
.fileext
262 pathsep
= os
.path
.sep
263 unique
= lambda filelist
: dict.fromkeys(filelist
).keys()
264 for testgoalname
, goalnode
in self
.getgoals().iteritems():
265 if goalname
!= testgoalname
: continue
266 goalmembers
= getattr(goalnode
, "files", "")
267 goalmembers
= [goalfile
.strip() for goalfile
in goalmembers
.split(",") if goalfile
.strip()]
268 goaldirs
= [goaldir
for goaldir
in goalmembers
if goaldir
.endswith(pathsep
)]
269 goalfiles
= [goalfile
for goalfile
in goalmembers
if not goalfile
.endswith(pathsep
) and goalfile
in self
.pofilenames
]
273 for goaldir
in goaldirs
:
274 expandedfiles
= self
.browsefiles(dirfilter
=goaldir
, includedirs
=includedirs
, includefiles
=True)
275 expandgoalfiles
.extend([expandfile
for expandfile
in expandedfiles
if expandfile
.endswith(poext
)])
276 expandgoaldirs
.extend([expanddir
+ pathsep
for expanddir
in expandedfiles
if not expanddir
.endswith(poext
)])
277 goaldirs
= unique(goaldirs
+ expandgoaldirs
)
278 goalfiles
= unique(goalfiles
+ expandgoalfiles
)
280 if not dirfilter
.endswith(pathsep
) and not dirfilter
.endswith(poext
):
282 goalfiles
= [goalfile
for goalfile
in goalfiles
if goalfile
.startswith(dirfilter
)]
283 goaldirs
= [goaldir
for goaldir
in goaldirs
if goaldir
.startswith(dirfilter
)]
284 if maxdepth
is not None:
286 partialdirs
= [goalfile
for goalfile
in goalfiles
if goalfile
.count(pathsep
) > maxdepth
]
287 partialdirs
+= [goalfile
for goalfile
in goaldirs
if goalfile
.count(pathsep
) > maxdepth
]
288 makepartial
= lambda goalfile
: pathsep
.join(goalfile
.split(pathsep
)[:maxdepth
+1])+pathsep
289 partialdirs
= [makepartial(goalfile
) for goalfile
in partialdirs
]
290 goalfiles
= [goalfile
for goalfile
in goalfiles
if goalfile
.count(pathsep
) <= maxdepth
]
291 goaldirs
= [goaldir
for goaldir
in goaldirs
if goaldir
.count(pathsep
) <= maxdepth
+1]
293 goaldirs
+= partialdirs
295 return unique(goalfiles
+ goaldirs
)
297 return unique(goalfiles
)
300 def getnogoalfiles(self
, dirfilter
=None, maxdepth
=None, includedirs
=True, expanddirs
=False):
301 """Returns the files that are not in a goal. This works with getgoalfiles
302 and therefre the API resembles that closely"""
303 all
= self
.browsefiles(dirfilter
=dirfilter
, maxdepth
=maxdepth
, includedirs
=includedirs
)
304 pathsep
= os
.path
.sep
305 for testgoalname
in self
.getgoals():
306 goalfiles
= self
.getgoalfiles(testgoalname
, dirfilter
=dirfilter
, maxdepth
=maxdepth
, includedirs
=includedirs
, expanddirs
=expanddirs
, includepartial
=False)
307 for goalfile
in goalfiles
:
308 if goalfile
.endswith(pathsep
):
309 goalfile
= goalfile
[:-len(pathsep
)]
313 def getancestry(self
, filename
):
314 """returns parent directories of the file"""
316 parts
= filename
.split(os
.path
.sep
)
317 for i
in range(1, len(parts
)):
318 ancestor
= os
.path
.join(*parts
[:i
]) + os
.path
.sep
319 ancestry
.append(ancestor
)
322 def getfilegoals(self
, filename
):
323 """gets the goals the given file is part of"""
324 goals
= self
.getgoals()
326 ancestry
= self
.getancestry(filename
)
327 for goalname
, goalnode
in goals
.iteritems():
328 goalfiles
= getattr(goalnode
, "files", "")
329 goalfiles
= [goalfile
.strip() for goalfile
in goalfiles
.split(",") if goalfile
.strip()]
330 if filename
in goalfiles
:
331 filegoals
.append(goalname
)
333 for ancestor
in ancestry
:
334 if ancestor
in goalfiles
:
335 filegoals
.append(goalname
)
339 def setfilegoals(self
, session
, goalnames
, filename
):
340 """sets the given file to belong to the given goals exactly"""
341 filegoals
= self
.getfilegoals(filename
)
342 for othergoalname
in filegoals
:
343 if othergoalname
not in goalnames
:
344 self
.removefilefromgoal(session
, othergoalname
, filename
)
345 for goalname
in goalnames
:
346 goalfiles
= self
.getgoalfiles(goalname
)
347 if filename
not in goalfiles
:
348 goalfiles
.append(filename
)
349 self
.setgoalfiles(session
, goalname
, goalfiles
)
351 def removefilefromgoal(self
, session
, goalname
, filename
):
352 """removes the given file from the goal"""
353 goalfiles
= self
.getgoalfiles(goalname
)
354 if filename
in goalfiles
:
355 goalfiles
.remove(filename
)
356 self
.setgoalfiles(session
, goalname
, goalfiles
)
358 unique
= lambda filelist
: dict.fromkeys(filelist
).keys()
359 ancestry
= self
.getancestry(filename
)
360 for ancestor
in ancestry
:
361 if ancestor
in goalfiles
:
362 filedepth
= filename
.count(os
.path
.sep
)
363 ancestordirs
= self
.getgoalfiles(goalname
, ancestor
, maxdepth
=filedepth
+1, includedirs
=True, expanddirs
=True)
364 ancestordirs
= [ancestorfile
for ancestorfile
in ancestordirs
if ancestorfile
.endswith(os
.path
.sep
)]
365 if filename
.endswith(os
.path
.sep
):
366 ancestorfiles
= self
.getgoalfiles(goalname
, ancestor
, maxdepth
=filedepth
-1, expanddirs
=True)
368 ancestorfiles
= self
.getgoalfiles(goalname
, ancestor
, maxdepth
=filedepth
, expanddirs
=True)
369 ancestorfiles
= unique(ancestordirs
+ ancestorfiles
)
370 if not filename
in ancestorfiles
:
371 raise KeyError("expected to find file %s in ancestor %s files %r" % (filename
, ancestor
, ancestorfiles
))
372 ancestorfiles
.remove(filename
)
373 ancestorfiles
.remove(ancestor
)
374 goalfiles
.remove(ancestor
)
375 goalfiles
.extend(ancestorfiles
)
376 self
.setgoalfiles(session
, goalname
, goalfiles
)
379 def setgoalfiles(self
, session
, goalname
, goalfiles
):
380 """sets the goalfiles for the given goalname"""
381 if "admin" not in self
.getrights(session
):
382 raise RightsError(session
.localize("You do not have rights to alter goals here"))
383 if isinstance(goalfiles
, list):
384 goalfiles
= [goalfile
.strip() for goalfile
in goalfiles
if goalfile
.strip()]
386 goalfiles
= ", ".join(goalfiles
)
387 if not hasattr(self
.prefs
, "goals"):
388 self
.prefs
.goals
= prefs
.PrefNode(self
.prefs
, "goals")
389 goals
= self
.getgoals()
390 goalname
= goalname
.encode("utf-8")
391 if not goalname
in goals
:
392 # TODO: check that its a valid goalname (alphanumeric etc)
393 self
.prefs
.goals
.__setattr
__(goalname
, prefs
.PrefNode(self
.prefs
.goals
, goalname
))
394 goalnode
= self
.prefs
.goals
.__getattr
__(goalname
)
395 goalnode
.files
= goalfiles
398 def getgoalusers(self
, goalname
):
399 """gets the users for the given goal"""
400 goals
= self
.getgoals()
401 for testgoalname
, goalnode
in goals
.iteritems():
402 if goalname
!= testgoalname
: continue
403 goalusers
= getattr(goalnode
, "users", "")
404 goalusers
= [goaluser
.strip() for goaluser
in goalusers
.split(",") if goaluser
.strip()]
408 def getusergoals(self
, username
):
409 """gets the goals the given user is part of"""
410 goals
= getattr(self
.prefs
, "goals", {})
412 for goalname
, goalnode
in goals
.iteritems():
413 goalusers
= getattr(goalnode
, "users", "")
414 goalusers
= [goaluser
.strip() for goaluser
in goalusers
.split(",") if goaluser
.strip()]
415 if username
in goalusers
:
416 usergoals
.append(goalname
)
420 def addusertogoal(self
, session
, goalname
, username
, exclusive
=False):
421 """adds the given user to the goal"""
423 usergoals
= self
.getusergoals(username
)
424 for othergoalname
in usergoals
:
425 if othergoalname
!= goalname
:
426 self
.removeuserfromgoal(session
, othergoalname
, username
)
427 goalusers
= self
.getgoalusers(goalname
)
428 if username
not in goalusers
:
429 goalusers
.append(username
)
430 self
.setgoalusers(session
, goalname
, goalusers
)
432 def removeuserfromgoal(self
, session
, goalname
, username
):
433 """removes the given user from the goal"""
434 goalusers
= self
.getgoalusers(goalname
)
435 if username
in goalusers
:
436 goalusers
.remove(username
)
437 self
.setgoalusers(session
, goalname
, goalusers
)
439 def setgoalusers(self
, session
, goalname
, goalusers
):
440 """sets the goalusers for the given goalname"""
441 if isinstance(goalname
, unicode):
442 goalname
= goalname
.encode('utf-8')
443 if "admin" not in self
.getrights(session
):
444 raise RightsError(session
.localize("You do not have rights to alter goals here"))
445 if isinstance(goalusers
, list):
446 goalusers
= [goaluser
.strip() for goaluser
in goalusers
if goaluser
.strip()]
447 goalusers
= ", ".join(goalusers
)
448 if not hasattr(self
.prefs
, "goals"):
449 self
.prefs
.goals
= prefs
.PrefNode(self
.prefs
, "goals")
450 if not hasattr(self
.prefs
.goals
, goalname
):
451 self
.prefs
.goals
.__setattr
__(goalname
, prefs
.PrefNode(self
.prefs
.goals
, goalname
))
452 goalnode
= self
.prefs
.goals
.__getattr
__(goalname
)
453 goalnode
.users
= goalusers
456 def scanpofiles(self
):
457 """sets the list of pofilenames by scanning the project directory"""
458 self
.pofilenames
= self
.potree
.getpofiles(self
.languagecode
, self
.projectcode
, poext
=self
.fileext
)
459 filename_set
= set(self
.pofilenames
)
460 pootlefile_set
= set(self
.pofiles
.keys())
461 # add any files that we don't have yet
462 for filename
in filename_set
.difference(pootlefile_set
):
463 self
.pofiles
[filename
] = pootlefile
.pootlefile(self
, filename
)
464 # remove any files that have been deleted since initialization
465 for filename
in pootlefile_set
.difference(filename_set
):
466 del self
.pofiles
[filename
]
468 def getuploadpath(self
, dirname
, localfilename
):
469 """gets the path of a translation file being uploaded securely, creating directories as neccessary"""
470 if os
.path
.isabs(dirname
) or dirname
.startswith("."):
471 raise ValueError("invalid/insecure file path: %s" % dirname
)
472 if os
.path
.basename(localfilename
) != localfilename
or localfilename
.startswith("."):
473 raise ValueError("invalid/insecure file name: %s" % localfilename
)
474 if self
.filestyle
== "gnu":
475 if not self
.potree
.languagematch(self
.languagecode
, localfilename
[:-len("."+self
.fileext
)]):
476 raise ValueError("invalid GNU-style file name %s: must match '%s.%s' or '%s[_-][A-Z]{2,3}.%s'" % (localfilename
, self
.fileext
, self
.languagecode
, self
.languagecode
, self
.fileext
))
477 dircheck
= self
.podir
478 for part
in dirname
.split(os
.sep
):
479 dircheck
= os
.path
.join(dircheck
, part
)
480 if dircheck
and not os
.path
.isdir(dircheck
):
482 return os
.path
.join(self
.podir
, dirname
, localfilename
)
484 def uploadfile(self
, session
, dirname
, filename
, contents
, overwrite
=False):
485 """uploads an individual file"""
486 pathname
= self
.getuploadpath(dirname
, filename
)
487 for extention
in ["xliff", "xlf", "xlff"]:
488 if filename
.endswith(extention
):
489 pofilename
= filename
[:-len(os
.extsep
+extention
)] + os
.extsep
+ self
.fileext
490 popathname
= self
.getuploadpath(dirname
, pofilename
)
493 pofilename
= filename
494 popathname
= pathname
496 rights
= self
.getrights(session
)
498 if os
.path
.exists(popathname
) and not overwrite
:
499 origpofile
= self
.getpofile(os
.path
.join(dirname
, pofilename
))
500 newfileclass
= factory
.getclass(pathname
)
501 infile
= cStringIO
.StringIO(contents
)
502 newfile
= newfileclass
.parsefile(infile
)
503 if "admin" in rights
:
504 origpofile
.mergefile(newfile
, session
.username
)
505 elif "translate" in rights
:
506 origpofile
.mergefile(newfile
, session
.username
, allownewstrings
=False)
507 elif "suggest" in rights
:
508 origpofile
.mergefile(newfile
, session
.username
, suggestions
=True)
510 raise RightsError(session
.localize("You do not have rights to upload files here"))
512 if overwrite
and not ("admin" in rights
or "overwrite" in rights
):
513 raise RightsError(session
.localize("You do not have rights to overwrite files here"))
514 elif not os
.path
.exists(popathname
) and not ("admin" in rights
or "overwrite" in rights
):
515 raise RightsError(session
.localize("You do not have rights to upload new files here"))
516 outfile
= open(popathname
, "wb")
517 outfile
.write(contents
)
521 def updatepofile(self
, session
, dirname
, pofilename
):
522 """updates an individual PO file from version control"""
523 if "admin" not in self
.getrights(session
):
524 raise RightsError(session
.localize("You do not have rights to update files here"))
525 pathname
= self
.getuploadpath(dirname
, pofilename
)
526 # read from version control
527 if os
.path
.exists(pathname
):
528 popath
= os
.path
.join(dirname
, pofilename
)
529 currentpofile
= self
.getpofile(popath
)
530 # reading BASE version of file
531 origcontents
= versioncontrol
.getcleanfile(pathname
, "BASE")
532 origpofile
= pootlefile
.pootlefile(self
, popath
)
533 originfile
= cStringIO
.StringIO(origcontents
)
534 origpofile
.parse(originfile
)
535 # matching current file with BASE version
536 matches
= origpofile
.matchitems(currentpofile
, uselocations
=False)
537 # TODO: add some locking here...
538 # reading new version of file
539 versioncontrol
.updatefile(pathname
)
540 newpofile
= pootlefile
.pootlefile(self
, popath
)
541 newpofile
.pofreshen()
542 if not hasattr(newpofile
, "sourceindex"):
543 newpofile
.makeindex()
545 # sorting through old matches
546 for origpo
, localpo
in matches
:
547 # we need to find the corresponding newpo to see what to merge
551 # if it wasn't in the original, then use the addition for searching
554 origmsgstr
= origpo
.target
555 localmsgstr
= localpo
.target
556 if origmsgstr
== localmsgstr
:
560 # First try to find a match on location
561 for location
in origpo
.getlocations():
562 if location
in newpofile
.locationindex
:
563 newpo
= newpofile
.locationindex
[location
]
564 if newpo
is not None and newpo
.source
== localpo
.source
:
566 newmatches
.append((newpo
, localpo
))
569 source
= origpo
.source
570 if source
in newpofile
.sourceindex
:
571 newpo
= newpofile
.sourceindex
[source
]
572 newmatches
.append((newpo
, localpo
))
574 newmatches
.append((None, localpo
))
575 # finding new matches
576 for newpo
, localpo
in newmatches
:
578 # TODO: include localpo as obsolete
582 newpofile
.mergeitem(newpo
, localpo
, "versionmerge")
584 newpofile
.savepofile()
585 self
.pofiles
[pofilename
] = newpofile
586 # recalculate everything
587 newpofile
.readpofile()
589 versioncontrol
.updatefile(pathname
)
592 def commitpofile(self
, session
, dirname
, pofilename
):
593 """commits an individual PO file to version control"""
594 if "commit" not in self
.getrights(session
):
595 raise RightsError(session
.localize("You do not have rights to commit files here"))
596 pathname
= self
.getuploadpath(dirname
, pofilename
)
597 stats
= self
.getquickstats([os
.path
.join(dirname
, pofilename
)])
598 statsstring
= "%d of %d messages translated (%d fuzzy)." % \
599 (stats
["translated"], stats
["total"], stats
["fuzzy"])
600 versioncontrol
.commitfile(pathname
, message
="Commit from %s by user %s. %s" %
601 (session
.server
.instance
.title
, session
.username
, statsstring
))
603 def converttemplates(self
, session
):
604 """creates PO files from the templates"""
605 projectdir
= os
.path
.join(self
.potree
.podirectory
, self
.projectcode
)
606 if not os
.path
.exists(projectdir
):
608 templatesdir
= os
.path
.join(projectdir
, "templates")
609 if not os
.path
.exists(templatesdir
):
610 templatesdir
= os
.path
.join(projectdir
, "pot")
611 if not os
.path
.exists(templatesdir
):
612 templatesdir
= projectdir
613 if self
.potree
.isgnustyle(self
.projectcode
):
614 self
.filestyle
= "gnu"
616 self
.filestyle
= "std"
617 templates
= self
.potree
.gettemplates(self
.projectcode
)
618 if self
.filestyle
== "gnu":
619 self
.podir
= projectdir
621 raise NotImplementedError("Cannot create GNU-style translation project without templates")
623 self
.podir
= os
.path
.join(projectdir
, self
.languagecode
)
624 if not os
.path
.exists(self
.podir
):
626 for potfilename
in templates
:
627 inputfile
= open(os
.path
.join(templatesdir
, potfilename
), "rb")
628 outputfile
= cStringIO
.StringIO()
629 if self
.filestyle
== "gnu":
630 pofilename
= self
.languagecode
+ os
.extsep
+ "po"
632 pofilename
= potfilename
[:-len(os
.extsep
+"pot")] + os
.extsep
+ "po"
633 pofilename
= os
.path
.basename(pofilename
)
634 origpofilename
= os
.path
.join(self
.podir
, pofilename
)
635 if os
.path
.exists(origpofilename
):
636 origpofile
= open(origpofilename
)
639 pot2po
.convertpot(inputfile
, outputfile
, origpofile
)
640 outfile
= open(origpofilename
, "wb")
641 outfile
.write(outputfile
.getvalue())
645 def filtererrorhandler(self
, functionname
, str1
, str2
, e
):
646 print "error in filter %s: %r, %r, %s" % (functionname
, str1
, str2
, e
)
649 def getarchive(self
, pofilenames
):
650 """returns an archive of the given filenames"""
652 # using zip command line is fast
653 from tempfile
import mkstemp
654 # The temporary file below is opened and immediately closed for security reasons
655 fd
, tempzipfile
= mkstemp(prefix
='pootle', suffix
='.zip')
657 os
.system("cd %s ; zip -r - %s > %s" % (self
.podir
, " ".join(pofilenames
), tempzipfile
))
658 filedata
= open(tempzipfile
, "r").read()
662 if os
.path
.exists(tempzipfile
):
663 os
.remove(tempzipfile
)
665 # but if it doesn't work, we can do it from python
667 archivecontents
= cStringIO
.StringIO()
668 archive
= zipfile
.ZipFile(archivecontents
, 'w', zipfile
.ZIP_DEFLATED
)
669 for pofilename
in pofilenames
:
670 pofile
= self
.getpofile(pofilename
)
671 archive
.write(pofile
.filename
, pofilename
)
673 return archivecontents
.getvalue()
675 def uploadarchive(self
, session
, dirname
, archivecontents
):
676 """uploads the files inside the archive"""
679 # from tempfile import mktemp
680 # tempzipfile = mkstemp()
681 # using zip command line is fast
682 # os.system("cd %s ; zip -r - %s > %s" % (self.podir, " ".join(pofilenames), tempzipfile))
683 # return open(tempzipfile, "r").read()
685 # if os.path.exists(tempzipfile):
686 # os.remove(tempzipfile)
688 # but if it doesn't work, we can do it from python
690 archivefile
= cStringIO
.StringIO(archivecontents
)
691 archive
= zipfile
.ZipFile(archivefile
, 'r')
692 # TODO: find a better way to return errors...
693 for filename
in archive
.namelist():
694 if not filename
.endswith(os
.extsep
+ self
.fileext
):
695 print "error adding %s: not a %s file" % (filename
, os
.extsep
+ self
.fileext
)
697 contents
= archive
.read(filename
)
698 subdirname
, pofilename
= os
.path
.dirname(filename
), os
.path
.basename(filename
)
700 # TODO: use zipfile info to set the time and date of the file
701 self
.uploadfile(session
, os
.path
.join(dirname
, subdirname
), pofilename
, contents
)
702 except ValueError, e
:
703 print "error adding %s" % filename
, e
707 def ootemplate(self
):
708 """Tests whether this project has an OpenOffice.org template SDF file in
709 the templates directory."""
710 projectdir
= os
.path
.join(self
.potree
.podirectory
, self
.projectcode
)
711 templatefilename
= os
.path
.join(projectdir
, "templates", "en-US.sdf")
712 if os
.path
.exists(templatefilename
):
713 return templatefilename
718 """Returns an OpenOffice.org gsi file"""
719 #TODO: implement caching
720 templateoo
= self
.ootemplate()
721 if templateoo
is None:
723 outputoo
= os
.path
.join(self
.podir
, self
.languagecode
+ ".sdf")
724 inputdir
= os
.path
.join(self
.potree
.podirectory
, self
.projectcode
, self
.languagecode
)
725 po2oo
.main(["-i%s"%inputdir
, "-t%s"%templateoo
, "-o%s"%outputoo
, "-l%s"%self
.languagecode
, "--progress=none"])
726 return file(os
.path
.join(self
.podir
, self
.languagecode
+ ".sdf"), "r").read()
728 def browsefiles(self
, dirfilter
=None, depth
=None, maxdepth
=None, includedirs
=False, includefiles
=True):
729 """gets a list of pofilenames, optionally filtering with the parent directory"""
730 if dirfilter
is None:
731 pofilenames
= self
.pofilenames
733 if not dirfilter
.endswith(os
.path
.sep
) and not dirfilter
.endswith(os
.extsep
+ self
.fileext
):
734 dirfilter
+= os
.path
.sep
735 pofilenames
= [pofilename
for pofilename
in self
.pofilenames
if pofilename
.startswith(dirfilter
)]
738 for pofilename
in pofilenames
:
739 dirname
= os
.path
.dirname(pofilename
)
742 podirs
[dirname
] = True
744 dirname
= os
.path
.dirname(dirname
)
746 podirs
[dirname
] = True
747 podirs
= podirs
.keys()
752 if maxdepth
is not None:
753 pofilenames
= [pofilename
for pofilename
in pofilenames
if pofilename
.count(os
.path
.sep
) <= maxdepth
]
754 podirs
= [podir
for podir
in podirs
if podir
.count(os
.path
.sep
) <= maxdepth
]
755 if depth
is not None:
756 pofilenames
= [pofilename
for pofilename
in pofilenames
if pofilename
.count(os
.path
.sep
) == depth
]
757 podirs
= [podir
for podir
in podirs
if podir
.count(os
.path
.sep
) == depth
]
758 return pofilenames
+ podirs
760 def iterpofilenames(self
, lastpofilename
=None, includelast
=False):
761 """iterates through the pofilenames starting after the given pofilename"""
762 if not lastpofilename
:
765 index
= self
.pofilenames
.index(lastpofilename
)
768 while index
< len(self
.pofilenames
):
769 yield self
.pofilenames
[index
]
772 def getindexer(self
):
773 """get an indexing object for this project
775 Since we do not want to keep the indexing databases open for the lifetime of
776 the TranslationProject (it is cached!), it may NOT be part of the Project object,
777 but should be used via a short living local variable.
779 indexdir
= os
.path
.join(self
.podir
, self
.index_directory
)
780 index
= indexer
.get_indexer(indexdir
)
781 index
.set_field_analyzers({
782 "pofilename": index
.ANALYZER_EXACT
,
783 "itemno": index
.ANALYZER_EXACT
,
784 "pomtime": index
.ANALYZER_EXACT
})
788 """initializes the search index"""
789 if not indexer
.HAVE_INDEXER
:
791 pofilenames
= self
.pofiles
.keys()
793 for pofilename
in pofilenames
:
794 self
.updateindex(pofilename
, optimize
=False)
796 def updateindex(self
, pofilename
, items
=None, optimize
=True):
797 """updates the index with the contents of pofilename (limit to items if given)
799 There are three reasons for calling this function:
800 (A) creating a new instance of "TranslationProject" (see "initindex")
801 -> check if the index is up-to-date / rebuild the index if necessary
802 (B) translating a unit via the web interface
803 -> (re)index only the specified unit(s)
805 The argument "items" should be None for (A).
808 1) This function should get called, when the po file changes externally.
809 The function "pofreshen" in pootlefile.py would be the natural place
810 for this. But this causes circular calls between the current (r7514)
811 statistics code and "updateindex" leading to indexing database lock
813 WARNING: You have to stop the pootle server before manually changing
814 po files, if you want to keep the index database in sync.
816 @param pofilename: absolute filename of the po file
817 @type pofilename: str
818 @param items: list of unit numbers within the po file OR None (=rebuild all)
820 @param optimize: should the indexing database be optimized afterwards
823 if not indexer
.HAVE_INDEXER
:
825 index
= self
.getindexer()
826 pofile
= self
.pofiles
[pofilename
]
827 # check if the pomtime in the index == the latest pomtime
829 pomtime
= statistics
.getmodtime(pofile
.filename
)
830 pofilenamequery
= index
.make_query([("pofilename", pofilename
)], True)
831 pomtimequery
= index
.make_query([("pomtime", str(pomtime
))], True)
832 gooditemsquery
= index
.make_query([pofilenamequery
, pomtimequery
], True)
833 gooditemsnum
= index
.get_query_result(gooditemsquery
).get_matches_count()
834 # if there is at least one up-to-date indexing item, then the po file
835 # was not changed externally -> no need to update the database
836 if (gooditemsnum
> 0) and (not items
):
840 # Update only specific items - usually single translation via the web
841 # interface. All other items should still be up-to-date (even with an
843 print "updating", self
.languagecode
, "index for", pofilename
, "items", items
844 # delete the relevant items from the database
845 itemsquery
= index
.make_query([("itemno", str(itemno
)) for itemno
in items
], False)
846 index
.delete_doc([pofilenamequery
, itemsquery
])
849 # The po file is not indexed - or it was changed externally (see
850 # "pofreshen" in pootlefile.py).
851 print "updating", self
.projectcode
, self
.languagecode
, "index for", pofilename
852 # delete all items of this file
853 index
.delete_doc({"pofilename": pofilename
})
856 # rebuild the whole index
857 items
= range(pofile
.statistics
.getitemslen())
860 unit
= pofile
.getitem(itemno
)
861 doc
= {"pofilename": pofilename
, "pomtime": str(pomtime
), "itemno": str(itemno
)}
863 orig
= "\n".join(unit
.source
.strings
)
864 trans
= "\n".join(unit
.target
.strings
)
869 doc
["msgstr"] = trans
872 index
.begin_transaction()
874 for add_item
in addlist
:
875 index
.index_document(add_item
)
877 index
.commit_transaction()
878 index
.flush(optimize
=optimize
)
879 except (base
.ParseError
, IOError, OSError):
880 index
.delete_doc({"pofilename": pofilename
})
881 print "Not indexing %s, since it is corrupt" % (pofilename
,)
883 def matchessearch(self
, pofilename
, search
):
884 """returns whether any items in the pofilename match the search (based on collected stats etc)"""
885 # wrong file location in a "dirfilter" search?
886 if search
.dirfilter
is not None and not pofilename
.startswith(search
.dirfilter
):
888 # search.assignedto == [None] means assigned to nobody
889 if search
.assignedto
or search
.assignedaction
:
890 if search
.assignedto
== [None]:
891 assigns
= self
.pofiles
[pofilename
].getassigns().getunassigned(search
.assignedaction
)
893 assigns
= self
.pofiles
[pofilename
].getassigns().getassigns()
894 if search
.assignedto
is not None:
895 if search
.assignedto
not in assigns
:
897 assigns
= assigns
[search
.assignedto
]
899 assigns
= reduce(lambda x
, y
: x
+y
, [userassigns
.keys() for userassigns
in assigns
.values()], [])
900 if search
.assignedaction
is not None:
901 if search
.assignedaction
not in assigns
:
903 if search
.matchnames
:
904 postats
= self
.getpostats(pofilename
)
906 for name
in search
.matchnames
:
907 if postats
.get(name
):
913 def indexsearch(self
, search
, returnfields
):
914 """returns the results from searching the index with the given search"""
915 if not indexer
.HAVE_INDEXER
:
917 index
= self
.getindexer()
919 if search
.searchtext
:
920 textquery
= index
.make_query([("msgid", search
.searchtext
), ("msgstr", search
.searchtext
)], False)
921 searchparts
.append(textquery
)
923 pofilenames
= self
.browsefiles(dirfilter
=search
.dirfilter
)
924 filequery
= index
.make_query([("pofilename", pofilename
) for pofilename
in pofilenames
], False)
925 searchparts
.append(filequery
)
926 # TODO: add other search items
927 limitedquery
= index
.make_query(searchparts
, True)
928 return index
.search(limitedquery
, returnfields
)
930 def searchpofilenames(self
, lastpofilename
, search
, includelast
=False):
931 """find the next pofilename that has items matching the given search"""
932 if lastpofilename
and not lastpofilename
in self
.pofiles
:
933 # accessing will autoload this file...
934 self
.pofiles
[lastpofilename
]
935 if indexer
.HAVE_INDEXER
and search
.searchtext
:
936 # TODO: move this up a level, use index to manage whole search, so we don't do this twice
937 hits
= self
.indexsearch(search
, "pofilename")
938 # there will be only result for the field "pofilename" - so we just
940 searchpofilenames
= dict.fromkeys([hit
["pofilename"][0] for hit
in hits
])
942 searchpofilenames
= None
943 for pofilename
in self
.iterpofilenames(lastpofilename
, includelast
):
944 if searchpofilenames
is not None:
945 if pofilename
not in searchpofilenames
:
947 if self
.matchessearch(pofilename
, search
):
950 def searchpoitems(self
, pofilename
, item
, search
):
951 """finds the next item matching the given search"""
952 if search
.searchtext
:
953 grepfilter
= pogrep
.GrepFilter(search
.searchtext
, None, ignorecase
=True)
954 for pofilename
in self
.searchpofilenames(pofilename
, search
, includelast
=True):
955 pofile
= self
.getpofile(pofilename
)
956 if indexer
.HAVE_INDEXER
and (search
.searchtext
or search
.matchnames
):
957 filesearch
= search
.copy()
958 filesearch
.dirfilter
= pofilename
959 hits
= self
.indexsearch(filesearch
, "itemno")
960 # there will be only result for the field "itemno" - so we just
962 items
= [int(doc
["itemno"][0]) for doc
in hits
]
963 items
= [searchitem
for searchitem
in items
if searchitem
> item
]
965 notextsearch
= search
.copy()
966 notextsearch
.searchtext
= None
967 matchitems
= list(pofile
.iteritems(notextsearch
, item
))
969 items
= pofile
.iteritems(search
, item
)
972 if items
!= matchitems
:
973 if item
not in matchitems
:
975 # TODO: move this to iteritems
976 if search
.searchtext
:
977 unit
= pofile
.getitem(item
)
978 if grepfilter
.filterunit(unit
):
979 yield pofilename
, item
981 yield pofilename
, item
984 def reassignpoitems(self
, session
, search
, assignto
, action
):
985 """reassign all the items matching the search to the assignto user(s) evenly, with the given action"""
986 # remove all assignments for the given action
987 self
.unassignpoitems(session
, search
, None, action
)
988 assigncount
= self
.assignpoitems(session
, search
, assignto
, action
)
991 def assignpoitems(self
, session
, search
, assignto
, action
):
992 """assign all the items matching the search to the assignto user(s) evenly, with the given action"""
993 if not "assign" in self
.getrights(session
):
994 raise RightsError(session
.localize("You do not have rights to alter assignments here"))
995 if search
.searchtext
:
996 grepfilter
= pogrep
.GrepFilter(search
.searchtext
, None, ignorecase
=True)
997 if not isinstance(assignto
, list):
998 assignto
= [assignto
]
999 usercount
= len(assignto
)
1003 pofilenames
= [pofilename
for pofilename
in self
.searchpofilenames(None, search
, includelast
=True)]
1004 wordcounts
= [(pofilename
, self
.getpofile(pofilename
).statistics
.getquickstats()['totalsourcewords']) for pofilename
in pofilenames
]
1005 totalwordcount
= sum([wordcount
for pofilename
, wordcount
in wordcounts
])
1007 wordsperuser
= totalwordcount
/ usercount
1008 print "assigning", totalwordcount
, "words to", usercount
, "user(s)", wordsperuser
, "words per user"
1011 for pofilename
, wordcount
in wordcounts
:
1012 pofile
= self
.getpofile(pofilename
)
1013 sourcewordcount
= pofile
.statistics
.getunitstats()['sourcewordcount']
1014 for item
in pofile
.iteritems(search
, None):
1015 # TODO: move this to iteritems
1016 if search
.searchtext
:
1018 unit
= pofile
.getitem(item
)
1019 if grepfilter
.filterunit(unit
):
1023 itemwordcount
= sourcewordcount
[item
]
1024 #itemwordcount = statsdb.wordcount(str(pofile.getitem(item).source))
1025 if userwords
+ itemwordcount
> wordsperuser
:
1026 usernum
= min(usernum
+1, len(assignto
)-1)
1028 userwords
+= itemwordcount
1029 pofile
.getassigns().assignto(item
, assignto
[usernum
], action
)
1033 def unassignpoitems(self
, session
, search
, assignedto
, action
=None):
1034 """unassigns all the items matching the search to the assignedto user"""
1035 if not "assign" in self
.getrights(session
):
1036 raise RightsError(session
.localize("You do not have rights to alter assignments here"))
1037 if search
.searchtext
:
1038 grepfilter
= pogrep
.GrepFilter(search
.searchtext
, None, ignorecase
=True)
1040 for pofilename
in self
.searchpofilenames(None, search
, includelast
=True):
1041 pofile
= self
.getpofile(pofilename
)
1042 for item
in pofile
.iteritems(search
, None):
1043 # TODO: move this to iteritems
1044 if search
.searchtext
:
1045 unit
= pofile
.getitem(item
)
1046 if grepfilter
.filterunit(unit
):
1047 pofile
.getassigns().unassign(item
, assignedto
, action
)
1050 pofile
.getassigns().unassign(item
, assignedto
, action
)
1054 def updatequickstats(self
, pofilename
, translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
, save
=True):
1055 """updates the quick stats on the given file"""
1056 self
.quickstats
[pofilename
] = (translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
)
1058 self
.savequickstats()
1060 def savequickstats(self
):
1061 """saves the quickstats"""
1062 self
.quickstatsfilename
= os
.path
.join(self
.podir
, "pootle-%s-%s.stats" % (self
.projectcode
, self
.languagecode
))
1063 quickstatsfile
= open(self
.quickstatsfilename
, "w")
1064 sortedquickstats
= self
.quickstats
.items()
1065 sortedquickstats
.sort()
1066 for pofilename
, (translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
) in sortedquickstats
:
1067 quickstatsfile
.write("%s, %d, %d, %d, %d, %d, %d\n" % \
1068 (pofilename
, translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
))
1069 quickstatsfile
.close()
1071 def readquickstats(self
):
1072 """reads the quickstats from disk"""
1073 self
.quickstats
= {}
1074 self
.quickstatsfilename
= os
.path
.join(self
.podir
, "pootle-%s-%s.stats" % (self
.projectcode
, self
.languagecode
))
1075 if os
.path
.exists(self
.quickstatsfilename
):
1076 quickstatsfile
= open(self
.quickstatsfilename
, "r")
1077 for line
in quickstatsfile
:
1078 items
= line
.split(",")
1080 #Must be an old format style without the fuzzy stats
1081 self
.quickstats
= self
.getquickstats()
1082 self
.savequickstats()
1085 pofilename
, translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
= items
1086 self
.quickstats
[pofilename
] = tuple([int(a
.strip()) for a
in \
1087 translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
])
1089 def getquickstats(self
, pofilenames
=None):
1090 """Gets translated and total stats and wordcounts without doing calculations returning dictionary."""
1091 if pofilenames
is None:
1092 pofilenames
= self
.pofilenames
1093 alltranslatedwords
, alltranslated
, allfuzzywords
, allfuzzy
, alltotalwords
, alltotal
= 0, 0, 0, 0, 0, 0
1095 for pofilename
in pofilenames
:
1096 if pofilename
not in self
.quickstats
:
1097 slowfiles
.append(pofilename
)
1099 translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
= self
.quickstats
[pofilename
]
1100 alltranslatedwords
+= translatedwords
1101 alltranslated
+= translated
1102 allfuzzywords
+= fuzzywords
1104 alltotalwords
+= totalwords
1106 for pofilename
in slowfiles
:
1107 self
.pofiles
[pofilename
].statistics
.updatequickstats(save
=False)
1108 translatedwords
, translated
, fuzzywords
, fuzzy
, totalwords
, total
= self
.quickstats
[pofilename
]
1109 alltranslatedwords
+= translatedwords
1110 alltranslated
+= translated
1111 allfuzzywords
+= fuzzywords
1113 alltotalwords
+= totalwords
1116 self
.savequickstats()
1117 return {"translatedsourcewords": alltranslatedwords
, "translated": alltranslated
,
1118 "fuzzysourcewords": allfuzzywords
, "fuzzy": allfuzzy
,
1119 "totalsourcewords": alltotalwords
, "total": alltotal
}
1121 def combinestats(self
, pofilenames
=None):
1122 """combines translation statistics for the given po files (or all if None given)"""
1123 if pofilenames
is None:
1124 pofilenames
= self
.pofilenames
1125 pofilenames
= [pofilename
for pofilename
in pofilenames
1126 if pofilename
!= None and not os
.path
.isdir(pofilename
)]
1127 total_stats
= self
.combine_totals(pofilenames
)
1128 total_stats
['units'] = self
.combine_unit_stats(pofilenames
)
1129 total_stats
['assign'] = self
.combineassignstats(pofilenames
)
1132 def combine_totals(self
, pofilenames
):
1134 for pofilename
in pofilenames
:
1135 pototals
= self
.getpototals(pofilename
)
1136 for name
, items
in pototals
.iteritems():
1137 totalstats
[name
] = totalstats
.get(name
, 0) + pototals
[name
]
1140 def combine_unit_stats(self
, pofilenames
):
1142 for pofilename
in pofilenames
:
1143 postats
= self
.getpostats(pofilename
)
1144 for name
, items
in postats
.iteritems():
1145 unit_stats
.setdefault(name
, []).extend([(pofilename
, item
) for item
in items
])
1148 def combineassignstats(self
, pofilenames
=None, action
=None):
1149 """combines assign statistics for the given po files (or all if None given)"""
1151 for pofilename
in pofilenames
:
1152 assignstats
= self
.getassignstats(pofilename
, action
)
1153 for name
, items
in assignstats
.iteritems():
1154 assign_stats
.setdefault(name
, []).extend([(pofilename
, item
) for item
in items
])
1157 def countwords(self
, stats
):
1158 """counts the number of words in the items represented by the stats list"""
1160 for pofilename
, item
in stats
:
1161 pofile
= self
.pofiles
[pofilename
]
1162 if 0 <= item
< len(pofile
.statistics
.getunitstats()['sourcewordcount']):
1163 wordcount
+= pofile
.statistics
.getunitstats()['sourcewordcount'][item
]
1164 print "projects::countwords()"
1167 def getpomtime(self
):
1168 """returns the modification time of the last modified file in the project"""
1169 return max([pofile
.pomtime
for pofile
in self
.pofiles
.values()])
1170 pomtime
= property(getpomtime
)
1172 def track(self
, pofilename
, item
, message
):
1173 """sends a track message to the pofile"""
1174 self
.pofiles
[pofilename
].track(item
, message
)
1176 def gettracks(self
, pofilenames
=None):
1177 """calculates translation statistics for the given po files (or all if None given)"""
1179 if pofilenames
is None:
1180 pofilenames
= self
.pofilenames
1181 for pofilename
in pofilenames
:
1182 if not pofilename
or os
.path
.isdir(pofilename
):
1184 tracker
= self
.pofiles
[pofilename
].tracker
1185 items
= tracker
.keys()
1188 alltracks
.append("%s item %d: %s" % (pofilename
, item
, tracker
[item
]))
1191 def getpostats(self
, pofilename
):
1192 """calculates translation statistics for the given po file"""
1193 return self
.pofiles
[pofilename
].statistics
.getstats()
1195 def getpototals(self
, pofilename
):
1196 """calculates translation statistics for the given po file"""
1197 return self
.pofiles
[pofilename
].statistics
.getquickstats()
1199 def getassignstats(self
, pofilename
, action
=None):
1200 """calculates translation statistics for the given po file (can filter by action if given)"""
1201 polen
= self
.getpototals(pofilename
).get("total", 0)
1202 # Temporary code to avoid traceback. Was:
1203 # polen = len(self.getpostats(pofilename)["total"])
1204 assigns
= self
.pofiles
[pofilename
].getassigns().getassigns()
1206 for username
, userassigns
in assigns
.iteritems():
1208 for assignaction
, items
in userassigns
.iteritems():
1209 if action
is None or assignaction
== action
:
1210 allitems
+= [item
for item
in items
if 0 <= item
< polen
and item
not in allitems
]
1212 assignstats
[username
] = allitems
1215 def getpofile(self
, pofilename
, freshen
=True):
1216 """parses the file into a pofile object and stores in self.pofiles"""
1217 pofile
= self
.pofiles
[pofilename
]
1222 def getpofilelen(self
, pofilename
):
1223 """returns number of items in the given pofilename"""
1224 pofile
= self
.getpofile(pofilename
)
1225 return pofile
.statistics
.getitemslen()
1227 def getitems(self
, pofilename
, itemstart
, itemstop
):
1228 """returns a set of items from the pofile, converted to original and translation strings"""
1229 pofile
= self
.getpofile(pofilename
)
1230 units
= [pofile
.units
[index
] for index
in pofile
.statistics
.getstats()["total"][max(itemstart
,0):itemstop
]]
1233 def updatetranslation(self
, pofilename
, item
, newvalues
, session
):
1234 """updates a translation with a new value..."""
1235 if "translate" not in self
.getrights(session
):
1236 raise RightsError(session
.localize("You do not have rights to change translations here"))
1237 pofile
= self
.pofiles
[pofilename
]
1238 pofile
.track(item
, "edited by %s" % session
.username
)
1239 languageprefs
= getattr(session
.instance
.languages
, self
.languagecode
, None)
1240 pofile
.updateunit(item
, newvalues
, session
.prefs
, languageprefs
)
1241 self
.updateindex(pofilename
, [item
])
1243 def suggesttranslation(self
, pofilename
, item
, trans
, session
):
1244 """stores a new suggestion for a translation..."""
1245 if "suggest" not in self
.getrights(session
):
1246 raise RightsError(session
.localize("You do not have rights to suggest changes here"))
1247 pofile
= self
.getpofile(pofilename
)
1248 pofile
.track(item
, "suggestion made by %s" % session
.username
)
1249 pofile
.addsuggestion(item
, trans
, session
.username
)
1251 def getsuggestions(self
, pofile
, item
):
1252 """find all the suggestions submitted for the given (pofile or pofilename) and item"""
1253 if isinstance(pofile
, (str, unicode)):
1255 pofile
= self
.getpofile(pofilename
)
1256 suggestpos
= pofile
.getsuggestions(item
)
1259 def acceptsuggestion(self
, pofile
, item
, suggitem
, newtrans
, session
):
1260 """accepts the suggestion into the main pofile"""
1261 if not "review" in self
.getrights(session
):
1262 raise RightsError(session
.localize("You do not have rights to review suggestions here"))
1263 if isinstance(pofile
, (str, unicode)):
1265 pofile
= self
.getpofile(pofilename
)
1266 pofile
.track(item
, "suggestion by %s accepted by %s" % (self
.getsuggester(pofile
, item
, suggitem
), session
.username
))
1267 pofile
.deletesuggestion(item
, suggitem
)
1268 self
.updatetranslation(pofilename
, item
, {"target": newtrans
, "fuzzy": False}, session
)
1270 def getsuggester(self
, pofile
, item
, suggitem
):
1271 """returns who suggested the given item's suggitem if recorded, else None"""
1272 if isinstance(pofile
, (str, unicode)):
1274 pofile
= self
.getpofile(pofilename
)
1275 return pofile
.getsuggester(item
, suggitem
)
1277 def rejectsuggestion(self
, pofile
, item
, suggitem
, newtrans
, session
):
1278 """rejects the suggestion and removes it from the pending file"""
1279 if not "review" in self
.getrights(session
):
1280 raise RightsError(session
.localize("You do not have rights to review suggestions here"))
1281 if isinstance(pofile
, (str, unicode)):
1283 pofile
= self
.getpofile(pofilename
)
1284 pofile
.track(item
, "suggestion by %s rejected by %s" % (self
.getsuggester(pofile
, item
, suggitem
), session
.username
))
1285 pofile
.deletesuggestion(item
, suggitem
)
1287 def gettmsuggestions(self
, pofile
, item
):
1288 """find all the TM suggestions for the given (pofile or pofilename) and item"""
1289 if isinstance(pofile
, (str, unicode)):
1291 pofile
= self
.getpofile(pofilename
)
1292 tmsuggestpos
= pofile
.gettmsuggestions(item
)
1295 def isterminologyproject(self
):
1296 """returns whether this project is the main terminology project for a
1297 language. Currently it is indicated by the project code 'terminology'"""
1298 return self
.projectcode
== "terminology"
1300 def gettermbase(self
):
1301 """returns this project's terminology store"""
1302 if self
.isterminologyproject():
1303 if len(self
.pofiles
) > 0:
1304 for termfile
in self
.pofiles
.values():
1305 termfile
.pofreshen()
1308 termfilename
= "pootle-terminology."+self
.fileext
1309 if termfilename
in self
.pofiles
:
1310 termfile
= self
.getpofile(termfilename
, freshen
=True)
1314 def gettermmatcher(self
):
1315 """returns the terminology matcher"""
1316 termbase
= self
.gettermbase()
1318 newmtime
= termbase
.pomtime
1319 if newmtime
!= self
.termmatchermtime
:
1320 self
.termmatchermtime
= newmtime
1321 if self
.isterminologyproject():
1322 self
.termmatcher
= match
.terminologymatcher(self
.pofiles
.values())
1324 self
.termmatcher
= match
.terminologymatcher(termbase
)
1325 elif not self
.isterminologyproject() and self
.potree
.hasproject(self
.languagecode
, "terminology"):
1326 termproject
= self
.potree
.getproject(self
.languagecode
, "terminology")
1327 self
.termmatcher
= termproject
.gettermmatcher()
1328 self
.termmatchermtime
= termproject
.termmatchermtime
1330 self
.termmatcher
= None
1331 self
.termmatchermtime
= None
1332 return self
.termmatcher
1334 def getterminology(self
, session
, pofile
, item
):
1335 """find all the terminology for the given (pofile or pofilename) and item"""
1337 termmatcher
= self
.gettermmatcher()
1340 if isinstance(pofile
, (str, unicode)):
1342 pofile
= self
.getpofile(pofilename
)
1343 return termmatcher
.matches(pofile
.getitem(item
).source
)
1344 except Exception, e
:
1345 session
.server
.errorhandler
.logerror(traceback
.format_exc())
1348 def savepofile(self
, pofilename
):
1349 """saves changes to disk"""
1350 pofile
= self
.getpofile(pofilename
)
1353 def getoutput(self
, pofilename
):
1354 """returns pofile source"""
1355 pofile
= self
.getpofile(pofilename
)
1356 return pofile
.getoutput()
1358 def convert(self
, pofilename
, destformat
):
1359 """converts the pofile to the given format, returning (etag_if_filepath, filepath_or_contents)"""
1360 pofile
= self
.getpofile(pofilename
, freshen
=False)
1361 destfilename
= pofile
.filename
[:-len(self
.fileext
)] + destformat
1362 destmtime
= statistics
.getmodtime(destfilename
)
1363 pomtime
= statistics
.getmodtime(pofile
.filename
)
1364 if pomtime
and destmtime
== pomtime
:
1366 return pomtime
, destfilename
1367 except Exception, e
:
1368 print "error reading cached converted file %s: %s" % (destfilename
, e
)
1370 converters
= {"csv": po2csv
.po2csv
, "xlf": po2xliff
.po2xliff
, "po": xliff2po
.xliff2po
, "ts": po2ts
.po2ts
, "mo": pocompile
.POCompile
}
1371 converterclass
= converters
.get(destformat
, None)
1372 if converterclass
is None:
1373 raise ValueError("No converter available for %s" % destfilename
)
1374 contents
= converterclass().convertstore(pofile
)
1375 if not isinstance(contents
, basestring
):
1376 contents
= str(contents
)
1378 destfile
= open(destfilename
, "w")
1379 destfile
.write(contents
)
1381 currenttime
, modtime
= time
.time(), pofile
.pomtime
1382 os
.utime(destfilename
, (currenttime
, modtime
))
1383 return modtime
, destfilename
1384 except Exception, e
:
1385 print "error caching converted file %s: %s" % (destfilename
, e
)
1386 return False, contents
1388 def gettext(self
, message
):
1389 """uses the project as a live translator for the given message"""
1390 for pofilename
, pofile
in self
.pofiles
.iteritems():
1391 if pofile
.pomtime
!= statistics
.getmodtime(pofile
.filename
):
1394 elif not hasattr(pofile
, "sourceindex"):
1396 unit
= pofile
.sourceindex
.get(message
, None)
1397 if not unit
or not unit
.istranslated():
1400 if tmsg
is not None:
1404 def ugettext(self
, message
):
1405 """gets the translation of the message by searching through all the pofiles (unicode version)"""
1406 for pofilename
, pofile
in self
.pofiles
.iteritems():
1408 if pofile
.pomtime
!= statistics
.getmodtime(pofile
.filename
):
1411 elif not hasattr(pofile
, "sourceindex"):
1413 unit
= pofile
.sourceindex
.get(message
, None)
1414 if not unit
or not unit
.istranslated():
1417 if tmsg
is not None:
1418 if isinstance(tmsg
, unicode):
1421 return unicode(tmsg
, pofile
.encoding
)
1422 except Exception, e
:
1423 print "error reading translation from pofile %s: %s" % (pofilename
, e
)
1424 return unicode(message
)
1426 def ungettext(self
, singular
, plural
, n
):
1427 """gets the plural translation of the message by searching through all the pofiles (unicode version)"""
1428 for pofilename
, pofile
in self
.pofiles
.iteritems():
1430 if pofile
.pomtime
!= statistics
.getmodtime(pofile
.filename
):
1433 elif not hasattr(pofile
, "sourceindex"):
1435 nplural
, pluralequation
= pofile
.getheaderplural()
1437 pluralfn
= gettext
.c2py(pluralequation
)
1438 unit
= pofile
.sourceindex
.get(singular
, None)
1439 if not unit
or not unit
.istranslated():
1441 tmsg
= unit
.target
.strings
[pluralfn(n
)]
1442 if tmsg
is not None:
1443 if isinstance(tmsg
, unicode):
1446 return unicode(tmsg
, pofile
.encoding
)
1447 except Exception, e
:
1448 print "error reading translation from pofile %s: %s" % (pofilename
, e
)
1450 return unicode(singular
)
1452 return unicode(plural
)
1454 def hascreatemofiles(self
, projectcode
):
1455 """returns whether the project has createmofile set"""
1456 return self
.potree
.getprojectcreatemofiles(projectcode
) == 1
1458 class DummyProject(TranslationProject
):
1459 """a project that is just being used for handling pootlefiles"""
1460 def __init__(self
, podir
, checker
=None, projectcode
=None, languagecode
=None):
1461 """initializes the project with the given podir"""
1464 self
.checker
= checks
.TeeChecker()
1466 self
.checker
= checker
1467 self
.projectcode
= projectcode
1468 self
.languagecode
= languagecode
1469 self
.readquickstats()
1471 def scanpofiles(self
):
1472 """A null operation if potree is not present"""
1475 def readquickstats(self
):
1476 """dummy statistics are empty"""
1477 self
.quickstats
= {}
1479 def savequickstats(self
):
1480 """saves quickstats if possible"""
1483 class DummyStatsProject(DummyProject
):
1484 """a project that is just being used for refresh of statistics"""
1485 def __init__(self
, podir
, checker
, projectcode
=None, languagecode
=None):
1486 """initializes the project with the given podir"""
1487 DummyProject
.__init
__(self
, podir
, checker
, projectcode
, languagecode
)
1489 def readquickstats(self
):
1490 """reads statistics from whatever files are available"""
1491 self
.quickstats
= {}
1492 if self
.projectcode
is not None and self
.languagecode
is not None:
1493 TranslationProject
.readquickstats(self
)
1495 def savequickstats(self
):
1496 """saves quickstats if possible"""
1497 if self
.projectcode
is not None and self
.languagecode
is not None:
1498 TranslationProject
.savequickstats(self
)
1500 class TemplatesProject(TranslationProject
):
1501 """Manages Template files (.pot files) for a project"""
1503 def __init__(self
, projectcode
, potree
):
1504 super(TemplatesProject
, self
).__init
__("templates", projectcode
, potree
, create
=False)
1506 def getrights(self
, session
=None, username
=None, usedefaults
=True):
1507 """gets the rights for the given user (name or session, or not-logged-in if username is None)"""
1508 # internal admin sessions have all rights
1509 # We don't send the usedefaults parameter through, because we don't want users of this method to
1510 # change the default behaviour in a template project. Yes, I know: ignorance and deceit.
1511 rights
= super(TemplatesProject
, self
).getrights(session
=session
, username
=username
)
1512 if rights
is not None:
1513 rights
= [right
for right
in rights
if right
not in ["translate", "suggest", "pocompile"]]