Package translate :: Package storage :: Module base
[hide private]
[frames] | no frames]

Source Code for Module translate.storage.base

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  # 
  4  # Copyright 2006-2009 Zuza Software Foundation 
  5  # 
  6  # This file is part of the Translate Toolkit. 
  7  # 
  8  # This program is free software; you can redistribute it and/or modify 
  9  # it under the terms of the GNU General Public License as published by 
 10  # the Free Software Foundation; either version 2 of the License, or 
 11  # (at your option) any later version. 
 12  # 
 13  # This program is distributed in the hope that it will be useful, 
 14  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  # GNU General Public License for more details. 
 17  # 
 18  # You should have received a copy of the GNU General Public License 
 19  # along with this program; if not, see <http://www.gnu.org/licenses/>. 
 20   
 21  """Base classes for storage interfaces. 
 22   
 23  @organization: Zuza Software Foundation 
 24  @copyright: 2006-2009 Zuza Software Foundation 
 25  @license: U{GPL <http://www.fsf.org/licensing/licenses/gpl.html>} 
 26  """ 
 27   
 28  try: 
 29      import cPickle as pickle 
 30  except: 
 31      import pickle 
 32  from exceptions import NotImplementedError 
 33  import translate.i18n 
 34  from translate.storage.placeables import StringElem, general, parse as rich_parse 
 35  from translate.misc.typecheck import accepts, Self, IsOneOf 
 36  from translate.misc.multistring import multistring 
 37   
38 -def force_override(method, baseclass):
39 """Forces derived classes to override method.""" 40 41 if type(method.im_self) == type(baseclass): 42 # then this is a classmethod and im_self is the actual class 43 actualclass = method.im_self 44 else: 45 actualclass = method.im_class 46 if actualclass != baseclass: 47 raise NotImplementedError( 48 "%s does not reimplement %s as required by %s" % \ 49 (actualclass.__name__, method.__name__, baseclass.__name__) 50 )
51 52
53 -class ParseError(Exception):
54 - def __init__(self, inner_exc):
55 self.inner_exc = inner_exc
56
57 - def __str__(self):
58 return repr(self.inner_exc)
59 60
61 -class TranslationUnit(object):
62 """Base class for translation units. 63 64 Our concept of a I{translation unit} is influenced heavily by XLIFF: 65 U{http://www.oasis-open.org/committees/xliff/documents/xliff-specification.htm} 66 67 As such most of the method- and variable names borrows from XLIFF terminology. 68 69 A translation unit consists of the following: 70 - A I{source} string. This is the original translatable text. 71 - A I{target} string. This is the translation of the I{source}. 72 - Zero or more I{notes} on the unit. Notes would typically be some 73 comments from a translator on the unit, or some comments originating from 74 the source code. 75 - Zero or more I{locations}. Locations indicate where in the original 76 source code this unit came from. 77 - Zero or more I{errors}. Some tools (eg. L{pofilter <filters.pofilter>}) can run checks on 78 translations and produce error messages. 79 80 @group Source: *source* 81 @group Target: *target* 82 @group Notes: *note* 83 @group Locations: *location* 84 @group Errors: *error* 85 """ 86 87 rich_parsers = [] 88 """A list of functions to use for parsing a string into a rich string tree.""" 89
90 - def __init__(self, source):
91 """Constructs a TranslationUnit containing the given source string.""" 92 self.notes = "" 93 self._store = None 94 self.source = source 95 self._target = None 96 self._rich_source = None 97 self._rich_target = None
98
99 - def __eq__(self, other):
100 """Compares two TranslationUnits. 101 102 @type other: L{TranslationUnit} 103 @param other: Another L{TranslationUnit} 104 @rtype: Boolean 105 @return: Returns True if the supplied TranslationUnit equals this unit. 106 """ 107 return self.source == other.source and self.target == other.target
108
109 - def rich_to_multistring(cls, elem_list):
110 """Convert a "rich" string tree to a C{multistring}: 111 112 >>> from translate.storage.placeables.interfaces import X 113 >>> rich = [StringElem(['foo', X(id='xxx', sub=[' ']), 'bar'])] 114 >>> TranslationUnit.rich_to_multistring(rich) 115 multistring(u'foo bar') 116 """ 117 return multistring([unicode(elem) for elem in elem_list])
118 rich_to_multistring = classmethod(rich_to_multistring) 119
120 - def multistring_to_rich(cls, mulstring):
121 """Convert a multistring to a list of "rich" string trees: 122 123 >>> target = multistring([u'foo', u'bar', u'baz']) 124 >>> TranslationUnit.multistring_to_rich(target) 125 [<StringElem([<StringElem([u'foo'])>])>, 126 <StringElem([<StringElem([u'bar'])>])>, 127 <StringElem([<StringElem([u'baz'])>])>] 128 """ 129 if isinstance(mulstring, multistring): 130 return [rich_parse(s, cls.rich_parsers) for s in mulstring.strings] 131 return [rich_parse(mulstring, cls.rich_parsers)]
132
133 - def setsource(self, source):
134 """Sets the source string to the given value.""" 135 self._rich_source = None 136 self._source = source
137 source = property(lambda self: self._source, setsource) 138
139 - def settarget(self, target):
140 """Sets the target string to the given value.""" 141 self._rich_target = None 142 self._target = target
143 target = property(lambda self: self._target, settarget) 144
145 - def _get_rich_source(self):
146 if self._rich_source is None: 147 self._rich_source = self.multistring_to_rich(self.source) 148 return self._rich_source
149 - def _set_rich_source(self, value):
150 if not hasattr(value, '__iter__'): 151 raise ValueError('value must be iterable') 152 if len(value) < 1: 153 raise ValueError('value must have at least one element.') 154 if not isinstance(value[0], StringElem): 155 raise ValueError('value[0] must be of type StringElem.') 156 self._rich_source = list(value) 157 self.source = self.rich_to_multistring(value)
158 rich_source = property(_get_rich_source, _set_rich_source) 159 """ @see: rich_to_multistring 160 @see: multistring_to_rich""" 161
162 - def _get_rich_target(self):
163 if self._rich_target is None: 164 self._rich_target = self.multistring_to_rich(self.target) 165 return self._rich_target
166 - def _set_rich_target(self, value):
167 if not hasattr(value, '__iter__'): 168 raise ValueError('value must be iterable') 169 if len(value) < 1: 170 raise ValueError('value must have at least one element.') 171 if not isinstance(value[0], StringElem): 172 raise ValueError('value[0] must be of type StringElem.') 173 self._rich_target = list(value) 174 self.target = self.rich_to_multistring(value)
175 rich_target = property(_get_rich_target, _set_rich_target) 176 """ @see: rich_to_multistring 177 @see: multistring_to_rich""" 178
179 - def gettargetlen(self):
180 """Returns the length of the target string. 181 182 @note: Plural forms might be combined. 183 @rtype: Integer 184 """ 185 length = len(self.target or "") 186 strings = getattr(self.target, "strings", []) 187 if strings: 188 length += sum([len(pluralform) for pluralform in strings[1:]]) 189 return length
190
191 - def getid(self):
192 """A unique identifier for this unit. 193 194 @rtype: string 195 @return: an identifier for this unit that is unique in the store 196 197 Derived classes should override this in a way that guarantees a unique 198 identifier for each unit in the store. 199 """ 200 return self.source
201
202 - def getlocations(self):
203 """A list of source code locations. 204 205 @note: Shouldn't be implemented if the format doesn't support it. 206 @rtype: List 207 """ 208 return []
209
210 - def addlocation(self, location):
211 """Add one location to the list of locations. 212 213 @note: Shouldn't be implemented if the format doesn't support it. 214 """ 215 pass
216
217 - def addlocations(self, location):
218 """Add a location or a list of locations. 219 220 @note: Most classes shouldn't need to implement this, 221 but should rather implement L{addlocation()}. 222 @warning: This method might be removed in future. 223 """ 224 if isinstance(location, list): 225 for item in location: 226 self.addlocation(item) 227 else: 228 self.addlocation(location)
229
230 - def getcontext(self):
231 """Get the message context.""" 232 return ""
233
234 - def getnotes(self, origin=None):
235 """Returns all notes about this unit. 236 237 It will probably be freeform text or something reasonable that can be 238 synthesised by the format. 239 It should not include location comments (see L{getlocations()}). 240 """ 241 return getattr(self, "notes", "")
242
243 - def addnote(self, text, origin=None):
244 """Adds a note (comment). 245 246 @type text: string 247 @param text: Usually just a sentence or two. 248 @type origin: string 249 @param origin: Specifies who/where the comment comes from. 250 Origin can be one of the following text strings: 251 - 'translator' 252 - 'developer', 'programmer', 'source code' (synonyms) 253 """ 254 if getattr(self, "notes", None): 255 self.notes += '\n'+text 256 else: 257 self.notes = text
258
259 - def removenotes(self):
260 """Remove all the translator's notes.""" 261 self.notes = u''
262
263 - def adderror(self, errorname, errortext):
264 """Adds an error message to this unit. 265 266 @type errorname: string 267 @param errorname: A single word to id the error. 268 @type errortext: string 269 @param errortext: The text describing the error. 270 """ 271 pass
272
273 - def geterrors(self):
274 """Get all error messages. 275 276 @rtype: Dictionary 277 """ 278 return {}
279
280 - def markreviewneeded(self, needsreview=True, explanation=None):
281 """Marks the unit to indicate whether it needs review. 282 283 @keyword needsreview: Defaults to True. 284 @keyword explanation: Adds an optional explanation as a note. 285 """ 286 pass
287
288 - def istranslated(self):
289 """Indicates whether this unit is translated. 290 291 This should be used rather than deducing it from .target, 292 to ensure that other classes can implement more functionality 293 (as XLIFF does). 294 """ 295 return bool(self.target) and not self.isfuzzy()
296
297 - def istranslatable(self):
298 """Indicates whether this unit can be translated. 299 300 This should be used to distinguish real units for translation from 301 header, obsolete, binary or other blank units. 302 """ 303 return True
304
305 - def isfuzzy(self):
306 """Indicates whether this unit is fuzzy.""" 307 return False
308
309 - def markfuzzy(self, value=True):
310 """Marks the unit as fuzzy or not.""" 311 pass
312
313 - def isobsolete(self):
314 """indicate whether a unit is obsolete""" 315 return False
316
317 - def makeobsolete(self):
318 """Make a unit obsolete""" 319 pass
320
321 - def isheader(self):
322 """Indicates whether this unit is a header.""" 323 return False
324
325 - def isreview(self):
326 """Indicates whether this unit needs review.""" 327 return False
328
329 - def isblank(self):
330 """Used to see if this unit has no source or target string. 331 332 @note: This is probably used more to find translatable units, 333 and we might want to move in that direction rather and get rid of this. 334 """ 335 return not (self.source or self.target)
336
337 - def hasplural(self):
338 """Tells whether or not this specific unit has plural strings.""" 339 #TODO: Reconsider 340 return False
341
342 - def getsourcelanguage(self):
343 return getattr(self._store, "sourcelanguage", "en")
344
345 - def gettargetlanguage(self):
346 return getattr(self._store, "targetlanguage", None)
347
348 - def merge(self, otherunit, overwrite=False, comments=True):
349 """Do basic format agnostic merging.""" 350 if not self.target or overwrite: 351 self.rich_target = otherunit.rich_target
352
353 - def unit_iter(self):
354 """Iterator that only returns this unit.""" 355 yield self
356
357 - def getunits(self):
358 """This unit in a list.""" 359 return [self]
360
361 - def buildfromunit(cls, unit):
362 """Build a native unit from a foreign unit, preserving as much 363 information as possible.""" 364 if type(unit) == cls and hasattr(unit, "copy") and callable(unit.copy): 365 return unit.copy() 366 newunit = cls(unit.source) 367 newunit.target = unit.target 368 newunit.markfuzzy(unit.isfuzzy()) 369 locations = unit.getlocations() 370 if locations: 371 newunit.addlocations(locations) 372 notes = unit.getnotes() 373 if notes: 374 newunit.addnote(notes) 375 return newunit
376 buildfromunit = classmethod(buildfromunit) 377 378 xid = property(lambda self: None, lambda self, value: None) 379 rid = property(lambda self: None, lambda self, value: None)
380 381
382 -class TranslationStore(object):
383 """Base class for stores for multiple translation units of type UnitClass.""" 384 385 UnitClass = TranslationUnit 386 """The class of units that will be instantiated and used by this class""" 387 Name = "Base translation store" 388 """The human usable name of this store type""" 389 Mimetypes = None 390 """A list of MIME types associated with this store type""" 391 Extensions = None 392 """A list of file extentions associated with this store type""" 393 _binary = False 394 """Indicates whether a file should be accessed as a binary file.""" 395 suggestions_in_format = False 396 """Indicates if format can store suggestions and alternative translation for a unit""" 397
398 - def __init__(self, unitclass=None):
399 """Constructs a blank TranslationStore.""" 400 self.units = [] 401 self.sourcelanguage = None 402 self.targetlanguage = None 403 if unitclass: 404 self.UnitClass = unitclass 405 super(TranslationStore, self).__init__()
406
407 - def getsourcelanguage(self):
408 """Gets the source language for this store""" 409 return self.sourcelanguage
410
411 - def setsourcelanguage(self, sourcelanguage):
412 """Sets the source language for this store""" 413 self.sourcelanguage = sourcelanguage
414
415 - def gettargetlanguage(self):
416 """Gets the target language for this store""" 417 return self.targetlanguage
418
419 - def settargetlanguage(self, targetlanguage):
420 """Sets the target language for this store""" 421 self.targetlanguage = targetlanguage
422
423 - def unit_iter(self):
424 """Iterator over all the units in this store.""" 425 for unit in self.units: 426 yield unit
427
428 - def getunits(self):
429 """Return a list of all units in this store.""" 430 return [unit for unit in self.unit_iter()]
431
432 - def addunit(self, unit):
433 """Appends the given unit to the object's list of units. 434 435 This method should always be used rather than trying to modify the 436 list manually. 437 438 @type unit: L{TranslationUnit} 439 @param unit: The unit that will be added. 440 """ 441 unit._store = self 442 self.units.append(unit)
443
444 - def addsourceunit(self, source):
445 """Adds and returns a new unit with the given source string. 446 447 @rtype: L{TranslationUnit} 448 """ 449 unit = self.UnitClass(source) 450 self.addunit(unit) 451 return unit
452
453 - def findid(self, id):
454 """find unit with matching id by checking id_index""" 455 self.require_index() 456 return self.id_index.get(id, None)
457
458 - def findunit(self, source):
459 """Finds the unit with the given source string. 460 461 @rtype: L{TranslationUnit} or None 462 """ 463 if len(getattr(self, "sourceindex", [])): 464 if source in self.sourceindex: 465 return self.sourceindex[source][0] 466 else: 467 for unit in self.units: 468 if unit.source == source: 469 return unit 470 return None
471 472
473 - def findunits(self, source):
474 """Finds the units with the given source string. 475 476 @rtype: L{TranslationUnit} or None 477 """ 478 if len(getattr(self, "sourceindex", [])): 479 if source in self.sourceindex: 480 return self.sourceindex[source] 481 else: 482 #FIXME: maybe we should generate index here instead since 483 #we'll scan all units anyway 484 result = [] 485 for unit in self.units: 486 if unit.source == source: 487 result.append(unit) 488 return result 489 return None
490
491 - def translate(self, source):
492 """Returns the translated string for a given source string. 493 494 @rtype: String or None 495 """ 496 unit = self.findunit(source) 497 if unit and unit.target: 498 return unit.target 499 else: 500 return None
501
502 - def remove_unit_from_index(self, unit):
503 """Remove a unit from source and locaton indexes""" 504 def remove_unit(source): 505 if source in self.sourceindex: 506 try: 507 self.sourceindex[source].remove(unit) 508 if len(self.sourceindex[source]) == 0: 509 del(self.sourceindex[source]) 510 except ValueError: 511 pass
512 513 if unit.hasplural(): 514 for source in unit.source.strings: 515 remove_unit(source) 516 else: 517 remove_unit(unit.source) 518 519 for location in unit.getlocations(): 520 if location in self.locationindex and self.locationindex[location] is not None \ 521 and self.locationindex[location] == unit: 522 del(self.locationindex[location])
523 524
525 - def add_unit_to_index(self, unit):
526 """Add a unit to source and location idexes""" 527 self.id_index[unit.getid()] = unit 528 529 def insert_unit(source): 530 if not source in self.sourceindex: 531 self.sourceindex[source] = [unit] 532 else: 533 self.sourceindex[source].append(unit)
534 535 if unit.hasplural(): 536 for source in unit.source.strings: 537 insert_unit(source) 538 else: 539 insert_unit(unit.source) 540 541 for location in unit.getlocations(): 542 if location in self.locationindex: 543 # if sources aren't unique, don't use them 544 #FIXME: maybe better store a list of units like sourceindex 545 self.locationindex[location] = None 546 else: 547 self.locationindex[location] = unit 548
549 - def makeindex(self):
550 """Indexes the items in this store. At least .sourceindex should be usefull.""" 551 self.locationindex = {} 552 self.sourceindex = {} 553 self.id_index = {} 554 for index, unit in enumerate(self.units): 555 unit.index = index 556 if unit.istranslatable(): 557 self.add_unit_to_index(unit)
558
559 - def require_index(self):
560 """make sure source index exists""" 561 if not hasattr(self, "sourceindex"): 562 self.makeindex()
563
564 - def getids(self):
565 """return a list of unit ids""" 566 self.require_index() 567 return self.id_index.keys()
568
569 - def __getstate__(self):
570 odict = self.__dict__.copy() 571 odict['fileobj'] = None 572 return odict
573
574 - def __setstate__(self, dict):
575 self.__dict__.update(dict) 576 if getattr(self, "filename", False): 577 self.fileobj = open(self.filename)
578
579 - def __str__(self):
580 """Converts to a string representation that can be parsed back using L{parsestring()}.""" 581 # We can't pickle fileobj if it is there, so let's hide it for a while. 582 fileobj = getattr(self, "fileobj", None) 583 self.fileobj = None 584 dump = pickle.dumps(self) 585 self.fileobj = fileobj 586 return dump
587
588 - def isempty(self):
589 """Returns True if the object doesn't contain any translation units.""" 590 if len(self.units) == 0: 591 return True 592 for unit in self.units: 593 if unit.istranslatable(): 594 return False 595 return True
596
597 - def _assignname(self):
598 """Tries to work out what the name of the filesystem file is and 599 assigns it to .filename.""" 600 fileobj = getattr(self, "fileobj", None) 601 if fileobj: 602 filename = getattr(fileobj, "name", getattr(fileobj, "filename", None)) 603 if filename: 604 self.filename = filename
605
606 - def parsestring(cls, storestring):
607 """Converts the string representation back to an object.""" 608 newstore = cls() 609 if storestring: 610 newstore.parse(storestring) 611 return newstore
612 parsestring = classmethod(parsestring) 613
614 - def parse(self, data):
615 """parser to process the given source string""" 616 self.units = pickle.loads(data).units
617
618 - def savefile(self, storefile):
619 """Writes the string representation to the given file (or filename).""" 620 if isinstance(storefile, basestring): 621 mode = 'w' 622 if self._binary: 623 mode = 'wb' 624 storefile = open(storefile, mode) 625 self.fileobj = storefile 626 self._assignname() 627 storestring = str(self) 628 storefile.write(storestring) 629 storefile.close()
630
631 - def save(self):
632 """Save to the file that data was originally read from, if available.""" 633 fileobj = getattr(self, "fileobj", None) 634 mode = 'w' 635 if self._binary: 636 mode = 'wb' 637 if not fileobj: 638 filename = getattr(self, "filename", None) 639 if filename: 640 fileobj = file(filename, mode) 641 else: 642 fileobj.close() 643 filename = getattr(fileobj, "name", getattr(fileobj, "filename", None)) 644 if not filename: 645 raise ValueError("No file or filename to save to") 646 fileobj = fileobj.__class__(filename, mode) 647 self.savefile(fileobj)
648
649 - def parsefile(cls, storefile):
650 """Reads the given file (or opens the given filename) and parses back to an object.""" 651 mode = 'r' 652 if cls._binary: 653 mode = 'rb' 654 if isinstance(storefile, basestring): 655 storefile = open(storefile, mode) 656 mode = getattr(storefile, "mode", mode) 657 #For some reason GzipFile returns 1, so we have to test for that here 658 if mode == 1 or "r" in mode: 659 storestring = storefile.read() 660 storefile.close() 661 else: 662 storestring = "" 663 newstore = cls.parsestring(storestring) 664 newstore.fileobj = storefile 665 newstore._assignname() 666 return newstore
667 parsefile = classmethod(parsefile) 668