1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """module for accessing mozilla xpi packages"""
23
24 from __future__ import generators
25 import zipfile
26 import os.path
27 import StringIO
28 import re
29
30 from translate import __version__
31
32
33
34 from translate.misc import zipfileext
35 ZipFileBase = zipfileext.ZipFileExt
36
37
38 from translate.misc import wStringIO
39
40
46
47 NamedStringInput = wStringIO.StringIO
48 NamedStringOutput = wStringIO.StringIO
49
50
52
53 def cp(a, b):
54 l = min(len(a), len(b))
55 for n in range(l):
56 if a[n] != b[n]:
57 return a[:n]
58 return a[:l]
59 if itemlist:
60 return reduce(cp, itemlist)
61 else:
62 return ''
63
64
66
67 def changed(*args, **kwargs):
68 self.changed = True
69 method(*args, **kwargs)
70 return changed
71
72
74 """catches output if there has been, before closing"""
75
85
87 """wrap the underlying close method, to pass the value to onclose before it goes"""
88 if self.changed:
89 value = self.getvalue()
90 self.onclose(value)
91 NamedStringInput.close(self)
92
94 """zip files call flush, not close, on file-like objects"""
95 value = self.getvalue()
96 self.onclose(value)
97 NamedStringInput.flush(self)
98
100 """use this method to force the closing of the stream if it isn't closed yet"""
101 if not self.closed:
102 self.close()
103
104
106 """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
107
113
115 """remember to call the given method before closing"""
116 if hasattr(self, "pendingsaves"):
117 if not pendingsave in self.pendingsaves:
118 self.pendingsaves.append(pendingsave)
119 else:
120 self.pendingsaves = [pendingsave]
121
123 """close the stream, remembering to call any addcatcher methods first"""
124 if hasattr(self, "pendingsaves"):
125 for pendingsave in self.pendingsaves:
126 pendingsave()
127
128 if ZipFileCatcher is None:
129 self.oldclose()
130 else:
131 super(ZipFileCatcher, self).close()
132
134 """writes the string into the archive, overwriting the file if it exists..."""
135 if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
136 filename = zinfo_or_arcname.filename
137 else:
138 filename = zinfo_or_arcname
139 if filename in self.NameToInfo:
140 self.delete(filename)
141 self.writestr(zinfo_or_arcname, bytes)
142 self.writeendrec()
143
144
146
148 """sets up the xpi file"""
149 self.includenonloc = kwargs.get("includenonloc", True)
150 if "includenonloc" in kwargs:
151 del kwargs["includenonloc"]
152 if "compression" not in kwargs:
153 kwargs["compression"] = zipfile.ZIP_DEFLATED
154 self.locale = kwargs.pop("locale", None)
155 self.region = kwargs.pop("region", None)
156 super(XpiFile, self).__init__(*args, **kwargs)
157 self.jarfiles = {}
158 self.findlangreg()
159 self.jarprefixes = self.findjarprefixes()
160 self.reverseprefixes = dict([
161 (prefix, jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix])
162 self.reverseprefixes["package/"] = None
163
165 """iterate through the jar files in the xpi as ZipFile objects"""
166 for filename in self.namelist():
167 if filename.lower().endswith('.jar'):
168 if filename not in self.jarfiles:
169 jarstream = self.openinputstream(None, filename)
170 jarfile = ZipFileCatcher(jarstream, mode=self.mode)
171 self.jarfiles[filename] = jarfile
172 else:
173 jarfile = self.jarfiles[filename]
174 yield filename, jarfile
175
177 """returns whether the given file is needed for localization (basically .dtd and .properties)"""
178 base, ext = os.path.splitext(filename)
179 return ext in (os.extsep + "dtd", os.extsep + "properties")
180
182 """finds the common prefix of all the files stored in the jar files"""
183 dirstructure = {}
184 locale = self.locale
185 region = self.region
186 localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$")
187 regionmatch = re.compile("^[a-zA-Z]{2,3}$")
188
189 osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$")
190 for jarfilename, jarfile in self.iterjars():
191 jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1)
192 if localematch.match(jarname) and not osmatch.match(jarname):
193 if locale is None:
194 locale = jarname
195 elif locale != jarname:
196 locale = 0
197 elif regionmatch.match(jarname):
198 if region is None:
199 region = jarname
200 elif region != jarname:
201 region = 0
202 for filename in jarfile.namelist():
203 if filename.endswith('/'):
204 continue
205 if not self.islocfile(filename) and not self.includenonloc:
206 continue
207 parts = filename.split('/')[:-1]
208 treepoint = dirstructure
209 for partnum in range(len(parts)):
210 part = parts[partnum]
211 if part in treepoint:
212 treepoint = treepoint[part]
213 else:
214 treepoint[part] = {}
215 treepoint = treepoint[part]
216 localeentries = {}
217 if 'locale' in dirstructure:
218 for dirname in dirstructure['locale']:
219 localeentries[dirname] = 1
220 if localematch.match(dirname) and not osmatch.match(dirname):
221 if locale is None:
222 locale = dirname
223 elif locale != dirname:
224 print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0"
225 locale = 0
226 elif regionmatch.match(dirname):
227 if region is None:
228 region = dirname
229 elif region != dirname:
230 region = 0
231 if locale and locale in localeentries:
232 del localeentries[locale]
233 if region and region in localeentries:
234 del localeentries[region]
235 if locale and not region:
236 if "-" in locale:
237 region = locale.split("-", 1)[1]
238 else:
239 region = ""
240 self.setlangreg(locale, region)
241
243 """set the locale and region of this xpi"""
244 if locale == 0 or locale is None:
245 raise ValueError("unable to determine locale")
246 self.locale = locale
247 self.region = region
248 self.dirmap = {}
249 if self.locale is not None:
250 self.dirmap[('locale', self.locale)] = ('lang-reg',)
251 if self.region:
252 self.dirmap[('locale', self.region)] = ('reg',)
253
255 """checks the uniqueness of the jar files contents"""
256 uniquenames = {}
257 jarprefixes = {}
258 for jarfilename, jarfile in self.iterjars():
259 jarprefixes[jarfilename] = ""
260 for filename in jarfile.namelist():
261 if filename.endswith('/'):
262 continue
263 if filename in uniquenames:
264 jarprefixes[jarfilename] = True
265 jarprefixes[uniquenames[filename]] = True
266 else:
267 uniquenames[filename] = jarfilename
268 for jarfilename, hasconflicts in jarprefixes.items():
269 if hasconflicts:
270 shortjarfilename = os.path.split(jarfilename)[1]
271 shortjarfilename = os.path.splitext(shortjarfilename)[0]
272 jarprefixes[jarfilename] = shortjarfilename + '/'
273
274 commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix])
275 if commonjarprefix:
276 for jarfilename, prefix in jarprefixes.items():
277 if prefix:
278 jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1)
279 return jarprefixes
280
282 """converts a zipfile filepath to an os-style filepath"""
283 return os.path.join(*zippath.split('/'))
284
286 """converts an os-style filepath to a zipfile filepath"""
287 return '/'.join(ospath.split(os.sep))
288
290 """uses a map to simplify the directory structure"""
291 parts = tuple(filename.split('/'))
292 possiblematch = None
293 for prefix, mapto in self.dirmap.iteritems():
294 if parts[:len(prefix)] == prefix:
295 if possiblematch is None or len(possiblematch[0]) < len(prefix):
296 possiblematch = prefix, mapto
297 if possiblematch is not None:
298 prefix, mapto = possiblematch
299 mapped = mapto + parts[len(prefix):]
300 return '/'.join(mapped)
301 return filename
302
304 """uses a map to rename files that occur straight in the xpi"""
305 if filename.startswith('bin/chrome/') and filename.endswith(".manifest"):
306 return 'bin/chrome/lang-reg.manifest'
307 return filename
308
310 """unmaps the filename..."""
311 possiblematch = None
312 parts = tuple(filename.split('/'))
313 for prefix, mapto in self.dirmap.iteritems():
314 if parts[:len(mapto)] == mapto:
315 if possiblematch is None or len(possiblematch[0]) < len(mapto):
316 possiblematch = (mapto, prefix)
317 if possiblematch is None:
318 return filename
319 mapto, prefix = possiblematch
320 reversemapped = prefix + parts[len(mapto):]
321 return '/'.join(reversemapped)
322
324 """uses a map to rename files that occur straight in the xpi"""
325 if filename == 'bin/chrome/lang-reg.manifest':
326 if self.locale:
327 return '/'.join(('bin', 'chrome', self.locale + '.manifest'))
328 else:
329 for otherfilename in self.namelist():
330 if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"):
331 return otherfilename
332 return filename
333
335 """converts a filename from within a jarfile to an os-style filepath"""
336 if jarfilename:
337 jarprefix = self.jarprefixes[jarfilename]
338 return self.ziptoospath(jarprefix + self.mapfilename(filename))
339 else:
340 return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
341
343 """converts an extracted os-style filepath to a jarfilename and filename"""
344 zipparts = ospath.split(os.sep)
345 prefix = zipparts[0] + '/'
346 if prefix in self.reverseprefixes:
347 jarfilename = self.reverseprefixes[prefix]
348 filename = self.reversemapfile('/'.join(zipparts[1:]))
349 if jarfilename is None:
350 filename = self.reversemapxpifilename(filename)
351 return jarfilename, filename
352 else:
353 filename = self.ostozippath(ospath)
354 if filename in self.namelist():
355 return None, filename
356 filename = self.reversemapfile('/'.join(zipparts))
357 possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix]
358 for jarfilename in possiblejarfilenames:
359 jarfile = self.jarfiles[jarfilename]
360 if filename in jarfile.namelist():
361 return jarfilename, filename
362 raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
363
365 """checks whether the given file exists inside the xpi"""
366 if jarfilename is None:
367 return filename in self.namelist()
368 else:
369 jarfile = self.jarfiles[jarfilename]
370 return filename in jarfile.namelist()
371
373 """checks whether the given file exists inside the xpi"""
374 jarfilename, filename = self.ostojarpath(ospath)
375 if jarfilename is None:
376 return filename in self.namelist()
377 else:
378 jarfile = self.jarfiles[jarfilename]
379 return filename in jarfile.namelist()
380
389 inputstream = CatchPotentialOutput(contents, onclose)
390 self.addcatcher(inputstream.slam)
391 else:
392 jarfile = self.jarfiles[jarfilename]
393 contents = jarfile.read(filename)
394 inputstream = NamedStringInput(contents)
395 inputstream.name = self.jartoospath(jarfilename, filename)
396 if hasattr(self.fp, 'name'):
397 inputstream.name = "%s:%s" % (self.fp.name, inputstream.name)
398 return inputstream
399
401 """opens a file for writing (possibly inside a jarfile as a StringIO"""
402 if jarfilename is None:
403
404 def onclose(contents):
405 self.overwritestr(filename, contents)
406 else:
407 if jarfilename in self.jarfiles:
408 jarfile = self.jarfiles[jarfilename]
409 else:
410 jarstream = self.openoutputstream(None, jarfilename)
411 jarfile = ZipFileCatcher(jarstream, "w")
412 self.jarfiles[jarfilename] = jarfile
413 self.addcatcher(jarstream.slam)
414
415 def onclose(contents):
416 jarfile.overwritestr(filename, contents)
417 outputstream = wStringIO.CatchStringOutput(onclose)
418 outputstream.name = "%s %s" % (jarfilename, filename)
419 if jarfilename is None:
420 self.addcatcher(outputstream.slam)
421 else:
422 jarfile.addcatcher(outputstream.slam)
423 return outputstream
424
426 """Close the file, and for mode "w" and "a" write the ending records."""
427 for jarfile in self.jarfiles.itervalues():
428 jarfile.close()
429 super(XpiFile, self).close()
430
432 """test the xpi zipfile and all enclosed jar files..."""
433 for jarfile in self.jarfiles.itervalues():
434 jarfile.testzip()
435 super(XpiFile, self).testzip()
436
437 - def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
438 """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream"""
439 jarfile = self.jarfiles[origjarfilename]
440 origlang = self.locale[:self.locale.find("-")]
441 if newregion:
442 newlocale = "%s-%s" % (newlang, newregion)
443 else:
444 newlocale = newlang
445 for filename in jarfile.namelist():
446 filenameparts = filename.split("/")
447 for i in range(len(filenameparts)):
448 part = filenameparts[i]
449 if part == origlang:
450 filenameparts[i] = newlang
451 elif part == self.locale:
452 filenameparts[i] = newlocale
453 elif part == self.region:
454 filenameparts[i] = newregion
455 newfilename = '/'.join(filenameparts)
456 fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename)
457 fileinputstream = self.openinputstream(origjarfilename, filename)
458 fileoutputstream.write(fileinputstream.read())
459 fileinputstream.close()
460 fileoutputstream.close()
461
462 - def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
463 """Create a new .xpi file with the same contents as this one..."""
464 other = XpiFile(newfilename, "w", locale=newlang, region=newregion)
465 origlang = self.locale[:self.locale.find("-")]
466
467 if newlang is None:
468 newlang = origlang
469 if newregion is None:
470 newregion = self.region
471 if newregion:
472 newlocale = "%s-%s" % (newlang, newregion)
473 else:
474 newlocale = newlang
475 for filename in self.namelist():
476 filenameparts = filename.split('/')
477 basename = filenameparts[-1]
478 if basename.startswith(self.locale):
479 newbasename = basename.replace(self.locale, newlocale)
480 elif basename.startswith(origlang):
481 newbasename = basename.replace(origlang, newlang)
482 elif basename.startswith(self.region):
483 newbasename = basename.replace(self.region, newregion)
484 else:
485 newbasename = basename
486 if newbasename != basename:
487 filenameparts[-1] = newbasename
488 renamefilename = "/".join(filenameparts)
489 print "cloning", filename, "and renaming to", renamefilename
490 else:
491 print "cloning", filename
492 renamefilename = filename
493 if filename.lower().endswith(".jar"):
494 self.restructurejar(filename, renamefilename, other, newlang, newregion)
495 else:
496 inputstream = self.openinputstream(None, filename)
497 outputstream = other.openoutputstream(None, renamefilename)
498 outputstream.write(inputstream.read())
499 inputstream.close()
500 outputstream.close()
501 other.close()
502 if newmode is None:
503 newmode = self.mode
504 if newmode == "w":
505 newmode = "a"
506 other = XpiFile(newfilename, newmode)
507 other.setlangreg(newlocale, newregion)
508 return other
509
511 """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary"""
512 if includenonjars:
513 for filename in self.namelist():
514 if filename.endswith('/') and not includedirs:
515 continue
516 if not self.islocfile(filename) and not self.includenonloc:
517 continue
518 if not filename.lower().endswith(".jar"):
519 yield self.jartoospath(None, filename)
520 for jarfilename, jarfile in self.iterjars():
521 for filename in jarfile.namelist():
522 if filename.endswith('/'):
523 if not includedirs:
524 continue
525 if not self.islocfile(filename) and not self.includenonloc:
526 continue
527 yield self.jartoospath(jarfilename, filename)
528
529
531 """iterates through all the files. this is the method use by the converters"""
532 for inputpath in self.iterextractnames(includenonjars=True):
533 yield inputpath
534
536 """returns whether the given pathname exists in the archive"""
537 try:
538 jarfilename, filename = self.ostojarpath(fullpath)
539 except IndexError:
540 return False
541 return self.jarfileexists(jarfilename, filename)
542
547
549 """opens an output file given the full pathname"""
550 try:
551 jarfilename, filename = self.ostojarpath(fullpath)
552 except IndexError:
553 return None
554 return self.openoutputstream(jarfilename, filename)
555
556
557 if __name__ == '__main__':
558 import optparse
559 optparser = optparse.OptionParser(version="%prog " + __version__.sver)
560 optparser.usage = "%prog [-l|-x] [options] file.xpi"
561 optparser.add_option("-l", "--list", help="list files", \
562 action="store_true", dest="listfiles", default=False)
563 optparser.add_option("-p", "--prefix", help="show common prefix", \
564 action="store_true", dest="showprefix", default=False)
565 optparser.add_option("-x", "--extract", help="extract files", \
566 action="store_true", dest="extractfiles", default=False)
567 optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \
568 default=".", metavar="EXTRACTDIR")
569 (options, args) = optparser.parse_args()
570 if len(args) < 1:
571 optparser.error("need at least one argument")
572 xpifile = XpiFile(args[0])
573 if options.showprefix:
574 for prefix, mapto in xpifile.dirmap.iteritems():
575 print "/".join(prefix), "->", "/".join(mapto)
576 if options.listfiles:
577 for name in xpifile.iterextractnames(includenonjars=True, includedirs=True):
578 print name
579 if options.extractfiles:
580 if options.extractdir and not os.path.isdir(options.extractdir):
581 os.mkdir(options.extractdir)
582 for name in xpifile.iterextractnames(includenonjars=True, includedirs=False):
583 abspath = os.path.join(options.extractdir, name)
584
585 currentpath = options.extractdir
586 subparts = os.path.dirname(name).split(os.sep)
587 for part in subparts:
588 currentpath = os.path.join(currentpath, part)
589 if not os.path.isdir(currentpath):
590 os.mkdir(currentpath)
591 outputstream = open(abspath, 'w')
592 jarfilename, filename = xpifile.ostojarpath(name)
593 inputstream = xpifile.openinputstream(jarfilename, filename)
594 outputstream.write(inputstream.read())
595 outputstream.close()
596