1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects.
26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library.
31 """
32
33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon
35 from translate.misc import quote
36 from translate.lang import data
37 from ctypes import *
38 import ctypes.util
39 try:
40 import cStringIO as StringIO
41 except ImportError:
42 import StringIO
43 import os
44 import pypo
45 import re
46 import sys
47 import tempfile
48
49 lsep = " "
50 """Seperator for #: entries"""
51
52 STRING = c_char_p
53
54
57
58
59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
61
62
63
67
69 _fields_ = [
70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
74 ]
75
76
77 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text
79 if severity >= 1:
80 raise ValueError(message_text)
81
82 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2):
83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2
84 if severity >= 1:
85 raise ValueError(message_text1)
86
87
88
89
90 gpo = None
91
92
93 names = ['gettextpo', 'libgettextpo']
94 for name in names:
95 lib_location = ctypes.util.find_library(name)
96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo:
99 break
100 else:
101
102
103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e:
106 raise ImportError("gettext PO library not found")
107
108
109
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int
113
114
115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING]
118
119
120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
124
125
126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_is_format.argtypes = [c_int, STRING]
133 gpo.po_message_set_format.argtypes = [c_int, STRING, c_int]
134 gpo.po_message_msgctxt.restype = STRING
135 gpo.po_message_msgid.restype = STRING
136 gpo.po_message_msgid_plural.restype = STRING
137 gpo.po_message_msgstr.restype = STRING
138 gpo.po_message_msgstr_plural.restype = STRING
139
140
141 gpo.po_message_set_comments.argtypes = [c_int, STRING]
142 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
143 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
144 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
145
146
147 xerror_handler = po_xerror_handler()
148 xerror_handler.xerror = xerror_prototype(xerror_cb)
149 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
150
153
156
159
162
164 """Returns the libgettextpo version
165
166 @rtype: three-value tuple
167 @return: libgettextpo version in the following format::
168 (major version, minor version, subminor version)
169 """
170 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
171 major = libversion.value >> 16
172 minor = libversion.value >> 8
173 subminor = libversion.value - (major << 16) - (minor << 8)
174 return major, minor, subminor
175
176
177 -class pounit(pocommon.pounit):
178 - def __init__(self, source=None, encoding='utf-8', gpo_message=None):
179 self._rich_source = None
180 self._rich_target = None
181 self._encoding = encoding
182 if not gpo_message:
183 self._gpo_message = gpo.po_message_create()
184 if source or source == "":
185 self.source = source
186 self.target = ""
187 elif gpo_message:
188 self._gpo_message = gpo_message
189
191 if isinstance(msgid_plural, list):
192 msgid_plural = "".join(msgid_plural)
193 gpo.po_message_set_msgid_plural(self._gpo_message, msgid_plural)
194 msgid_plural = property(None, setmsgid_plural)
195
197 def remove_msgid_comments(text):
198 if not text:
199 return text
200 if text.startswith("_:"):
201 remainder = re.search(r"_: .*\n(.*)", text)
202 if remainder:
203 return remainder.group(1)
204 else:
205 return u""
206 else:
207 return text
208 singular = remove_msgid_comments(gpo.po_message_msgid(self._gpo_message).decode(self._encoding))
209 if singular:
210 if self.hasplural():
211 multi = multistring(singular, self._encoding)
212 pluralform = gpo.po_message_msgid_plural(self._gpo_message).decode(self._encoding)
213 multi.strings.append(pluralform)
214 return multi
215 else:
216 return singular
217 else:
218 return u""
219
232
233 source = property(getsource, setsource)
234
236 if self.hasplural():
237 plurals = []
238 nplural = 0
239 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
240 while plural:
241 plurals.append(plural.decode(self._encoding))
242 nplural += 1
243 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
244 if plurals:
245 multi = multistring(plurals, encoding=self._encoding)
246 else:
247 multi = multistring(u"")
248 else:
249 multi = (gpo.po_message_msgstr(self._gpo_message) or "").decode(self._encoding)
250 return multi
251
253
254 if self.hasplural():
255 if isinstance(target, multistring):
256 target = target.strings
257 elif isinstance(target, basestring):
258 target = [target]
259
260 elif isinstance(target, (dict, list)):
261 if len(target) == 1:
262 target = target[0]
263 else:
264 raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target))
265
266
267
268
269
270 if isinstance(target, (dict, list)):
271 i = 0
272 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
273 while message is not None:
274 gpo.po_message_set_msgstr_plural(self._gpo_message, i, None)
275 i += 1
276 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
277
278 if isinstance(target, list):
279 for i in range(len(target)):
280 targetstring = target[i]
281 if isinstance(targetstring, unicode):
282 targetstring = targetstring.encode(self._encoding)
283 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
284
285 elif isinstance(target, dict):
286 for i, targetstring in enumerate(target.itervalues()):
287 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
288
289 else:
290 if isinstance(target, unicode):
291 target = target.encode(self._encoding)
292 if target is None:
293 gpo.po_message_set_msgstr(self._gpo_message, "")
294 else:
295 gpo.po_message_set_msgstr(self._gpo_message, target)
296 target = property(gettarget, settarget)
297
299 """The unique identifier for this unit according to the convensions in
300 .mo files."""
301 id = (gpo.po_message_msgid(self._gpo_message) or "").decode(self._encoding)
302
303
304
305
306
307
308
309 context = gpo.po_message_msgctxt(self._gpo_message)
310 if context:
311 id = u"%s\04%s" % (context.decode(self._encoding), id)
312 return id
313
315 if origin == None:
316 comments = gpo.po_message_comments(self._gpo_message) + \
317 gpo.po_message_extracted_comments(self._gpo_message)
318 elif origin == "translator":
319 comments = gpo.po_message_comments(self._gpo_message)
320 elif origin in ["programmer", "developer", "source code"]:
321 comments = gpo.po_message_extracted_comments(self._gpo_message)
322 else:
323 raise ValueError("Comment type not valid")
324
325 if comments and get_libgettextpo_version() < (0, 17, 0):
326 comments = "\n".join([line.strip() for line in comments.split("\n")])
327
328 return comments[:-1].decode(self._encoding)
329
330 - def addnote(self, text, origin=None, position="append"):
331
332 if not (text and text.strip()):
333 return
334 text = data.forceunicode(text)
335 oldnotes = self.getnotes(origin)
336 newnotes = None
337 if oldnotes:
338 if position == "append":
339 newnotes = oldnotes + "\n" + text
340 elif position == "merge":
341 if oldnotes != text:
342 oldnoteslist = oldnotes.split("\n")
343 for newline in text.split("\n"):
344 newline = newline.rstrip()
345
346 if newline not in oldnotes or len(newline) < 5:
347 oldnoteslist.append(newline)
348 newnotes = "\n".join(oldnoteslist)
349 else:
350 newnotes = text + '\n' + oldnotes
351 else:
352 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
353
354 if newnotes:
355 newlines = []
356 needs_space = get_libgettextpo_version() < (0, 17, 0)
357 for line in newnotes.split("\n"):
358 if line and needs_space:
359 newlines.append(" " + line)
360 else:
361 newlines.append(line)
362 newnotes = "\n".join(newlines).encode(self._encoding)
363 if origin in ["programmer", "developer", "source code"]:
364 gpo.po_message_set_extracted_comments(self._gpo_message, newnotes)
365 else:
366 gpo.po_message_set_comments(self._gpo_message, newnotes)
367
369 gpo.po_message_set_comments(self._gpo_message, "")
370
372 newpo = self.__class__()
373 newpo._gpo_message = self._gpo_message
374 return newpo
375
376 - def merge(self, otherpo, overwrite=False, comments=True, authoritative=False):
410
412
413
414 return self.getid() == "" and len(self.target) > 0
415
418
421
424
431
433 return gpo.po_message_is_fuzzy(self._gpo_message)
434
436 gpo.po_message_set_fuzzy(self._gpo_message, present)
437
439 return gpo.po_message_is_obsolete(self._gpo_message)
440
442
443
444 gpo.po_message_set_obsolete(self._gpo_message, True)
445
447 gpo.po_message_set_obsolete(self._gpo_message, False)
448
450 return gpo.po_message_msgid_plural(self._gpo_message) is not None
451
463
467 msgidcomment = property(_extract_msgidcomments, setmsgidcomment)
468
473
475 locations = []
476 i = 0
477 location = gpo.po_message_filepos(self._gpo_message, i)
478 while location:
479 locname = gpo.po_filepos_file(location)
480 locline = gpo.po_filepos_start_line(location)
481 if locline == -1:
482 locstring = locname
483 else:
484 locstring = locname + ":" + str(locline)
485 locations.append(locstring)
486 i += 1
487 location = gpo.po_message_filepos(self._gpo_message, i)
488 return locations
489
491 for loc in location.split():
492 parts = loc.split(":")
493 file = parts[0]
494 if len(parts) == 2:
495 line = int(parts[1] or "0")
496 else:
497 line = -1
498 gpo.po_message_add_filepos(self._gpo_message, file, line)
499
500 - def getcontext(self):
501 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
502 if msgctxt:
503 return msgctxt.decode(self._encoding)
504 else:
505 msgidcomment = self._extract_msgidcomments()
506 return msgidcomment
507
542 buildfromunit = classmethod(buildfromunit)
543
544 -class pofile(pocommon.pofile):
545 UnitClass = pounit
547 self.UnitClass = unitclass
548 pocommon.pofile.__init__(self, unitclass=unitclass)
549 self._gpo_memory_file = None
550 self._gpo_message_iterator = None
551 self._encoding = encodingToUse(encoding)
552 if inputfile is not None:
553 self.parse(inputfile)
554 else:
555 self._gpo_memory_file = gpo.po_file_create()
556 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
557
558 - def addunit(self, unit, new=True):
559 if new:
560 gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message)
561 super(pofile, self).addunit(unit)
562
564 """make sure each msgid is unique ; merge comments etc from duplicates into original"""
565
566
567 id_dict = {}
568 uniqueunits = []
569
570
571 markedpos = []
572 def addcomment(thepo):
573 thepo.msgidcomment = " ".join(thepo.getlocations())
574 markedpos.append(thepo)
575 for thepo in self.units:
576 id = thepo.getid()
577 if thepo.isheader() and not thepo.getlocations():
578
579 uniqueunits.append(thepo)
580 elif id in id_dict:
581 if duplicatestyle == "merge":
582 if id:
583 id_dict[id].merge(thepo)
584 else:
585 addcomment(thepo)
586 uniqueunits.append(thepo)
587 elif duplicatestyle == "msgctxt":
588 origpo = id_dict[id]
589 if origpo not in markedpos:
590 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations()))
591 markedpos.append(thepo)
592 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
593 uniqueunits.append(thepo)
594 else:
595 if not id:
596 if duplicatestyle == "merge":
597 addcomment(thepo)
598 else:
599 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
600 id_dict[id] = thepo
601 uniqueunits.append(thepo)
602 new_gpo_memory_file = gpo.po_file_create()
603 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
604 for unit in uniqueunits:
605 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
606 gpo.po_message_iterator_free(self._gpo_message_iterator)
607 self._gpo_message_iterator = new_gpo_message_iterator
608 self._gpo_memory_file = new_gpo_memory_file
609 self.units = uniqueunits
610
612 def obsolete_workaround():
613
614
615
616 for unit in self.units:
617 if unit.isobsolete():
618 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
619 location = gpo.po_message_filepos(unit._gpo_message, 0)
620 while location:
621 gpo.po_message_remove_filepos(unit._gpo_message, 0)
622 location = gpo.po_message_filepos(unit._gpo_message, 0)
623 outputstring = ""
624 if self._gpo_memory_file:
625 obsolete_workaround()
626 f, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
627 os.close(f)
628 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, fname, xerror_handler)
629 f = open(fname)
630 outputstring = f.read()
631 f.close()
632 os.remove(fname)
633 return outputstring
634
636 """Returns True if the object doesn't contain any translation units."""
637 if len(self.units) == 0:
638 return True
639
640 if self.units[0].isheader():
641 units = self.units[1:]
642 else:
643 units = self.units
644
645 for unit in units:
646 if not unit.isblank() and not unit.isobsolete():
647 return False
648 return True
649
651 if hasattr(input, 'name'):
652 self.filename = input.name
653 elif not getattr(self, 'filename', ''):
654 self.filename = ''
655
656 if hasattr(input, "read"):
657 posrc = input.read()
658 input.close()
659 input = posrc
660
661 needtmpfile = not os.path.isfile(input)
662 if needtmpfile:
663
664 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
665 os.write(fd, input)
666 input = fname
667 os.close(fd)
668
669 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
670 if self._gpo_memory_file is None:
671 print >> sys.stderr, "Error:"
672
673 if needtmpfile:
674 os.remove(input)
675
676
677 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
678 if self._header:
679 charset = gpo.po_header_field(self._header, "Content-Type")
680 if charset:
681 charset = re.search("charset=([^\\s]+)", charset).group(1)
682 self._encoding = encodingToUse(charset)
683 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
684 newmessage = gpo.po_next_message(self._gpo_message_iterator)
685 while newmessage:
686 newunit = pounit(gpo_message=newmessage, encoding=self._encoding)
687 self.addunit(newunit, new=False)
688 newmessage = gpo.po_next_message(self._gpo_message_iterator)
689 self._free_iterator()
690
692
693
694 return
695 self._free_iterator()
696 if self._gpo_memory_file is not None:
697 gpo.po_file_free(self._gpo_memory_file)
698 self._gpo_memory_file = None
699
701
702
703 return
704 if self._gpo_message_iterator is not None:
705 gpo.po_message_iterator_free(self._gpo_message_iterator)
706 self._gpo_message_iterator = None
707