third_party/transifex-client/txclib/project.py

   1 # -*- coding: utf-8 -*-
   2 import base64
   3 import copy
   4 import getpass
   5 import os
   6 import re
   7 import fnmatch
   8 import urllib2
   9 import datetime, time
  10 import ConfigParser
  11
  12 from txclib.web import *
  13 from txclib.utils import *
  14 from txclib.urls import API_URLS
  15 from txclib.config import OrderedRawConfigParser, Flipdict
  16 from txclib.log import logger
  17 from txclib.http_utils import http_response
  18 from txclib.processors import visit_hostname
  19
  20
  21 class ProjectNotInit(Exception):
  22     pass
  23
  24
  25 class Project(object):
  26     """
  27     Represents an association between the local and remote project instances.
  28     """
  29
  30     def __init__(self, path_to_tx=None, init=True):
  31         """
  32         Initialize the Project attributes.
  33         """
  34         if init:
  35             self._init(path_to_tx)
  36
  37     def _init(self, path_to_tx=None):
  38         instructions = "Run 'tx init' to initialize your project first!"
  39         try:
  40             self.root = self._get_tx_dir_path(path_to_tx)
  41             self.config_file = self._get_config_file_path(self.root)
  42             self.config = self._read_config_file(self.config_file)
  43             self.txrc_file = self._get_transifex_file()
  44             self.txrc = self._get_transifex_config(self.txrc_file)
  45         except ProjectNotInit, e:
  46             logger.error('\n'.join([unicode(e), instructions]))
  47             raise
  48
  49     def _get_config_file_path(self, root_path):
  50         """Check the .tx/config file exists."""
  51         config_file = os.path.join(root_path, ".tx", "config")
  52         logger.debug("Config file is %s" % config_file)
  53         if not os.path.exists(config_file):
  54             msg = "Cannot find the config file (.tx/config)!"
  55             raise ProjectNotInit(msg)
  56         return config_file
  57
  58     def _get_tx_dir_path(self, path_to_tx):
  59         """Check the .tx directory exists."""
  60         root_path = path_to_tx or find_dot_tx()
  61         logger.debug("Path to tx is %s." % root_path)
  62         if not root_path:
  63             msg = "Cannot find any .tx directory!"
  64             raise ProjectNotInit(msg)
  65         return root_path
  66
  67     def _read_config_file(self, config_file):
  68         """Parse the config file and return its contents."""
  69         config = OrderedRawConfigParser()
  70         try:
  71             config.read(config_file)
  72         except Exception, err:
  73             msg = "Cannot open/parse .tx/config file: %s" % err
  74             raise ProjectNotInit(msg)
  75         return config
  76
  77     def _get_transifex_config(self, txrc_file):
  78         """Read the configuration from the .transifexrc file."""
  79         txrc = OrderedRawConfigParser()
  80         try:
  81             txrc.read(txrc_file)
  82         except Exception, e:
  83             msg = "Cannot read global configuration file: %s" % e
  84             raise ProjectNotInit(msg)
  85         self._migrate_txrc_file(txrc)
  86         return txrc
  87
  88     def _migrate_txrc_file(self, txrc):
  89         """Migrate the txrc file, if needed."""
  90         for section in txrc.sections():
  91             orig_hostname = txrc.get(section, 'hostname')
  92             hostname = visit_hostname(orig_hostname)
  93             if hostname != orig_hostname:
  94                 msg = "Hostname %s should be changed to %s."
  95                 logger.info(msg % (orig_hostname, hostname))
  96                 if (sys.stdin.isatty() and sys.stdout.isatty() and
  97                     confirm('Change it now? ', default=True)):
  98                     txrc.set(section, 'hostname', hostname)
  99                     msg = 'Hostname changed'
 100                     logger.info(msg)
 101                 else:
 102                     hostname = orig_hostname
 103             self._save_txrc_file(txrc)
 104         return txrc
 105
 106     def _get_transifex_file(self, directory=None):
 107         """Fetch the path of the .transifexrc file.
 108
 109         It is in the home directory ofthe user by default.
 110         """
 111         if directory is None:
 112             directory = os.path.expanduser('~')
 113         txrc_file = os.path.join(directory, ".transifexrc")
 114         logger.debug(".transifexrc file is at %s" % directory)
 115         if not os.path.exists(txrc_file):
 116             msg = "No authentication data found."
 117             logger.info(msg)
 118             mask = os.umask(077)
 119             open(txrc_file, 'w').close()
 120             os.umask(mask)
 121         return txrc_file
 122
 123     def validate_config(self):
 124         """
 125         To ensure the json structure is correctly formed.
 126         """
 127         pass
 128
 129     def getset_host_credentials(self, host, user=None, password=None):
 130         """
 131         Read .transifexrc and report user,pass for a specific host else ask the
 132         user for input.
 133         """
 134         try:
 135             username = self.txrc.get(host, 'username')
 136             passwd = self.txrc.get(host, 'password')
 137         except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
 138             logger.info("No entry found for host %s. Creating..." % host)
 139             username = user or raw_input("Please enter your transifex username: ")
 140             while (not username):
 141                 username = raw_input("Please enter your transifex username: ")
 142             passwd = password
 143             while (not passwd):
 144                 passwd = getpass.getpass()
 145
 146             logger.info("Updating %s file..." % self.txrc_file)
 147             self.txrc.add_section(host)
 148             self.txrc.set(host, 'username', username)
 149             self.txrc.set(host, 'password', passwd)
 150             self.txrc.set(host, 'token', '')
 151             self.txrc.set(host, 'hostname', host)
 152
 153         return username, passwd
 154
 155     def set_remote_resource(self, resource, source_lang, i18n_type, host,
 156             file_filter="translations<sep>%(proj)s.%(res)s<sep><lang>.%(extension)s"):
 157         """
 158         Method to handle the add/conf of a remote resource.
 159         """
 160         if not self.config.has_section(resource):
 161             self.config.add_section(resource)
 162
 163         p_slug, r_slug = resource.split('.')
 164         file_filter = file_filter.replace("<sep>", r"%s" % os.path.sep)
 165         self.url_info = {
 166             'host': host,
 167             'project': p_slug,
 168             'resource': r_slug
 169         }
 170         extension = self._extension_for(i18n_type)[1:]
 171
 172         self.config.set(resource, 'source_lang', source_lang)
 173         self.config.set(
 174             resource, 'file_filter',
 175             file_filter % {'proj': p_slug, 'res': r_slug, 'extension': extension}
 176         )
 177         if host != self.config.get('main', 'host'):
 178             self.config.set(resource, 'host', host)
 179
 180     def get_resource_host(self, resource):
 181         """
 182         Returns the host that the resource is configured to use. If there is no
 183         such option we return the default one
 184         """
 185         if self.config.has_option(resource, 'host'):
 186             return self.config.get(resource, 'host')
 187         return self.config.get('main', 'host')
 188
 189     def get_resource_lang_mapping(self, resource):
 190         """
 191         Get language mappings for a specific resource.
 192         """
 193         lang_map = Flipdict()
 194         try:
 195             args = self.config.get("main", "lang_map")
 196             for arg in args.replace(' ', '').split(','):
 197                 k,v = arg.split(":")
 198                 lang_map.update({k:v})
 199         except ConfigParser.NoOptionError:
 200             pass
 201         except (ValueError, KeyError):
 202             raise Exception("Your lang map configuration is not correct.")
 203
 204         if self.config.has_section(resource):
 205             res_lang_map = Flipdict()
 206             try:
 207                 args = self.config.get(resource, "lang_map")
 208                 for arg in args.replace(' ', '').split(','):
 209                     k,v = arg.split(":")
 210                     res_lang_map.update({k:v})
 211             except ConfigParser.NoOptionError:
 212                 pass
 213             except (ValueError, KeyError):
 214                 raise Exception("Your lang map configuration is not correct.")
 215
 216         # merge the lang maps and return result
 217         lang_map.update(res_lang_map)
 218
 219         return lang_map
 220
 221
 222     def get_resource_files(self, resource):
 223         """
 224         Get a dict for all files assigned to a resource. First we calculate the
 225         files matching the file expression and then we apply all translation
 226         excpetions. The resulting dict will be in this format:
 227
 228         { 'en': 'path/foo/en/bar.po', 'de': 'path/foo/de/bar.po', 'es': 'path/exceptions/es.po'}
 229
 230         NOTE: All paths are relative to the root of the project
 231         """
 232         tr_files = {}
 233         if self.config.has_section(resource):
 234             try:
 235                 file_filter = self.config.get(resource, "file_filter")
 236             except ConfigParser.NoOptionError:
 237                 file_filter = "$^"
 238             source_lang = self.config.get(resource, "source_lang")
 239             source_file = self.get_resource_option(resource, 'source_file') or None
 240             expr_re = regex_from_filefilter(file_filter, self.root)
 241             expr_rec = re.compile(expr_re)
 242             for root, dirs, files in os.walk(self.root):
 243                 for f in files:
 244                     f_path = os.path.abspath(os.path.join(root, f))
 245                     match = expr_rec.match(f_path)
 246                     if match:
 247                         lang = match.group(1)
 248                         if lang != source_lang:
 249                             f_path = relpath(f_path, self.root)
 250                             if f_path != source_file:
 251                                 tr_files.update({lang: f_path})
 252
 253             for (name, value) in self.config.items(resource):
 254                 if name.startswith("trans."):
 255                     lang = name.split('.')[1]
 256                     # delete language which has same file
 257                     if value in tr_files.values():
 258                         keys = []
 259                         for k, v in tr_files.iteritems():
 260                             if v == value:
 261                                 keys.append(k)
 262                         if len(keys) == 1:
 263                             del tr_files[keys[0]]
 264                         else:
 265                             raise Exception("Your configuration seems wrong."\
 266                                 " You have multiple languages pointing to"\
 267                                 " the same file.")
 268                     # Add language with correct file
 269                     tr_files.update({lang:value})
 270
 271             return tr_files
 272
 273         return None
 274
 275     def get_resource_option(self, resource, option):
 276         """
 277         Return the requested option for a specific resource
 278
 279         If there is no such option, we return None
 280         """
 281
 282         if self.config.has_section(resource):
 283             if self.config.has_option(resource, option):
 284                 return self.config.get(resource, option)
 285         return None
 286
 287     def get_resource_list(self, project=None):
 288         """
 289         Parse config file and return tuples with the following format
 290
 291         [ (project_slug, resource_slug), (..., ...)]
 292         """
 293
 294         resource_list= []
 295         for r in self.config.sections():
 296             if r == 'main':
 297                 continue
 298             p_slug, r_slug = r.split('.', 1)
 299             if project and p_slug != project:
 300                 continue
 301             resource_list.append(r)
 302
 303         return resource_list
 304
 305     def save(self):
 306         """
 307         Store the config dictionary in the .tx/config file of the project.
 308         """
 309         self._save_tx_config()
 310         self._save_txrc_file()
 311
 312     def _save_tx_config(self, config=None):
 313         """Save the local config file."""
 314         if config is None:
 315             config = self.config
 316         fh = open(self.config_file,"w")
 317         config.write(fh)
 318         fh.close()
 319
 320     def _save_txrc_file(self, txrc=None):
 321         """Save the .transifexrc file."""
 322         if txrc is None:
 323             txrc = self.txrc
 324         mask = os.umask(077)
 325         fh = open(self.txrc_file, 'w')
 326         txrc.write(fh)
 327         fh.close()
 328         os.umask(mask)
 329
 330     def get_full_path(self, relpath):
 331         if relpath[0] == "/":
 332             return relpath
 333         else:
 334             return os.path.join(self.root, relpath)
 335
 336     def pull(self, languages=[], resources=[], overwrite=True, fetchall=False,
 337         fetchsource=False, force=False, skip=False, minimum_perc=0, mode=None):
 338         """Pull all translations file from transifex server."""
 339         self.minimum_perc = minimum_perc
 340         resource_list = self.get_chosen_resources(resources)
 341
 342         if mode == 'reviewed':
 343             url = 'pull_reviewed_file'
 344         elif mode == 'translator':
 345             url = 'pull_translator_file'
 346         elif mode == 'developer':
 347             url = 'pull_developer_file'
 348         else:
 349             url = 'pull_file'
 350
 351         for resource in resource_list:
 352             logger.debug("Handling resource %s" % resource)
 353             self.resource = resource
 354             project_slug, resource_slug = resource.split('.')
 355             files = self.get_resource_files(resource)
 356             slang = self.get_resource_option(resource, 'source_lang')
 357             sfile = self.get_resource_option(resource, 'source_file')
 358             lang_map = self.get_resource_lang_mapping(resource)
 359             host = self.get_resource_host(resource)
 360             logger.debug("Language mapping is: %s" % lang_map)
 361             if mode is None:
 362                 mode = self._get_option(resource, 'mode')
 363             self.url_info = {
 364                 'host': host,
 365                 'project': project_slug,
 366                 'resource': resource_slug
 367             }
 368             logger.debug("URL data are: %s" % self.url_info)
 369
 370             stats = self._get_stats_for_resource()
 371
 372
 373             try:
 374                 file_filter = self.config.get(resource, 'file_filter')
 375             except ConfigParser.NoOptionError:
 376                 file_filter = None
 377
 378             # Pull source file
 379             pull_languages = set([])
 380             new_translations = set([])
 381
 382             if fetchall:
 383                 new_translations = self._new_translations_to_add(
 384                     files, slang, lang_map, stats, force
 385                 )
 386                 if new_translations:
 387                     msg = "New translations found for the following languages: %s"
 388                     logger.info(msg % ', '.join(new_translations))
 389
 390             existing, new = self._languages_to_pull(
 391                 languages, files, lang_map, stats, force
 392             )
 393             pull_languages |= existing
 394             new_translations |= new
 395             logger.debug("Adding to new translations: %s" % new)
 396
 397             if fetchsource:
 398                 if sfile and slang not in pull_languages:
 399                     pull_languages.add(slang)
 400                 elif slang not in new_translations:
 401                     new_translations.add(slang)
 402
 403             if pull_languages:
 404                 logger.debug("Pulling languages for: %s" % pull_languages)
 405                 msg = "Pulling translations for resource %s (source: %s)"
 406                 logger.info(msg % (resource, sfile))
 407
 408             for lang in pull_languages:
 409                 local_lang = lang
 410                 if lang in lang_map.values():
 411                     remote_lang = lang_map.flip[lang]
 412                 else:
 413                     remote_lang = lang
 414                 if languages and lang not in pull_languages:
 415                     logger.debug("Skipping language %s" % lang)
 416                     continue
 417                 if lang != slang:
 418                     local_file = files.get(lang, None) or files[lang_map[lang]]
 419                 else:
 420                     local_file = sfile
 421                 logger.debug("Using file %s" % local_file)
 422
 423                 kwargs = {
 424                     'lang': remote_lang,
 425                     'stats': stats,
 426                     'local_file': local_file,
 427                     'force': force,
 428                     'mode': mode,
 429                 }
 430                 if not self._should_update_translation(**kwargs):
 431                     msg = "Skipping '%s' translation (file: %s)."
 432                     logger.info(
 433                         msg % (color_text(remote_lang, "RED"), local_file)
 434                     )
 435                     continue
 436
 437                 if not overwrite:
 438                     local_file = ("%s.new" % local_file)
 439                 logger.warning(
 440                     " -> %s: %s" % (color_text(remote_lang, "RED"), local_file)
 441                 )
 442                 try:
 443                     r = self.do_url_request(url, language=remote_lang)
 444                 except Exception,e:
 445                     if not skip:
 446                         raise e
 447                     else:
 448                         logger.error(e)
 449                         continue
 450                 base_dir = os.path.split(local_file)[0]
 451                 mkdir_p(base_dir)
 452                 fd = open(local_file, 'wb')
 453                 fd.write(r)
 454                 fd.close()
 455
 456             if new_translations:
 457                 msg = "Pulling new translations for resource %s (source: %s)"
 458                 logger.info(msg % (resource, sfile))
 459                 for lang in new_translations:
 460                     if lang in lang_map.keys():
 461                         local_lang = lang_map[lang]
 462                     else:
 463                         local_lang = lang
 464                     remote_lang = lang
 465                     if file_filter:
 466                         local_file = relpath(os.path.join(self.root,
 467                             file_filter.replace('<lang>', local_lang)), os.curdir)
 468                     else:
 469                         trans_dir = os.path.join(self.root, ".tx", resource)
 470                         if not os.path.exists(trans_dir):
 471                             os.mkdir(trans_dir)
 472                         local_file = relpath(os.path.join(trans_dir, '%s_translation' %
 473                             local_lang, os.curdir))
 474
 475                     if lang != slang:
 476                         satisfies_min = self._satisfies_min_translated(
 477                             stats[remote_lang], mode
 478                         )
 479                         if not satisfies_min:
 480                             msg = "Skipping language %s due to used options."
 481                             logger.info(msg % lang)
 482                             continue
 483                     logger.warning(
 484                         " -> %s: %s" % (color_text(remote_lang, "RED"), local_file)
 485                     )
 486                     r = self.do_url_request(url, language=remote_lang)
 487
 488                     base_dir = os.path.split(local_file)[0]
 489                     mkdir_p(base_dir)
 490                     fd = open(local_file, 'wb')
 491                     fd.write(r)
 492                     fd.close()
 493
 494     def push(self, source=False, translations=False, force=False, resources=[], languages=[],
 495         skip=False, no_interactive=False):
 496         """
 497         Push all the resources
 498         """
 499         resource_list = self.get_chosen_resources(resources)
 500         self.skip = skip
 501         self.force = force
 502         for resource in resource_list:
 503             push_languages = []
 504             project_slug, resource_slug = resource.split('.')
 505             files = self.get_resource_files(resource)
 506             slang = self.get_resource_option(resource, 'source_lang')
 507             sfile = self.get_resource_option(resource, 'source_file')
 508             lang_map = self.get_resource_lang_mapping(resource)
 509             host = self.get_resource_host(resource)
 510             logger.debug("Language mapping is: %s" % lang_map)
 511             logger.debug("Using host %s" % host)
 512             self.url_info = {
 513                 'host': host,
 514                 'project': project_slug,
 515                 'resource': resource_slug
 516             }
 517
 518             logger.info("Pushing translations for resource %s:" % resource)
 519
 520             stats = self._get_stats_for_resource()
 521
 522             if force and not no_interactive:
 523                 answer = raw_input("Warning: By using --force, the uploaded"
 524                     " files will overwrite remote translations, even if they"
 525                     " are newer than your uploaded files.\nAre you sure you"
 526                     " want to continue? [y/N] ")
 527
 528                 if not answer in ["", 'Y', 'y', "yes", 'YES']:
 529                     return
 530
 531             if source:
 532                 if sfile == None:
 533                     logger.error("You don't seem to have a proper source file"
 534                         " mapping for resource %s. Try without the --source"
 535                         " option or set a source file first and then try again." %
 536                         resource)
 537                     continue
 538                 # Push source file
 539                 try:
 540                     logger.warning("Pushing source file (%s)" % sfile)
 541                     if not self._resource_exists(stats):
 542                         logger.info("Resource does not exist.  Creating...")
 543                         fileinfo = "%s;%s" % (resource_slug, slang)
 544                         filename = self.get_full_path(sfile)
 545                         self._create_resource(resource, project_slug, fileinfo, filename)
 546                     self.do_url_request(
 547                         'push_source', multipart=True, method="PUT",
 548                         files=[(
 549                                 "%s;%s" % (resource_slug, slang)
 550                                 , self.get_full_path(sfile)
 551                         )],
 552                     )
 553                 except Exception, e:
 554                     if not skip:
 555                         raise
 556                     else:
 557                         logger.error(e)
 558             else:
 559                 try:
 560                     self.do_url_request('resource_details')
 561                 except Exception, e:
 562                     code = getattr(e, 'code', None)
 563                     if code == 404:
 564                         msg = "Resource %s doesn't exist on the server."
 565                         logger.error(msg % resource)
 566                         continue
 567
 568             if translations:
 569                 # Check if given language codes exist
 570                 if not languages:
 571                     push_languages = files.keys()
 572                 else:
 573                     push_languages = []
 574                     f_langs = files.keys()
 575                     for l in languages:
 576                         if l in lang_map.keys():
 577                             l = lang_map[l]
 578                         push_languages.append(l)
 579                         if l not in f_langs:
 580                             msg = "Warning: No mapping found for language code '%s'."
 581                             logger.error(msg % color_text(l,"RED"))
 582                 logger.debug("Languages to push are %s" % push_languages)
 583
 584                 # Push translation files one by one
 585                 for lang in push_languages:
 586                     local_lang = lang
 587                     if lang in lang_map.values():
 588                         remote_lang = lang_map.flip[lang]
 589                     else:
 590                         remote_lang = lang
 591
 592                     local_file = files[local_lang]
 593
 594                     kwargs = {
 595                         'lang': remote_lang,
 596                         'stats': stats,
 597                         'local_file': local_file,
 598                         'force': force,
 599                     }
 600                     if not self._should_push_translation(**kwargs):
 601                         msg = "Skipping '%s' translation (file: %s)."
 602                         logger.info(msg % (color_text(lang, "RED"), local_file))
 603                         continue
 604
 605                     msg = "Pushing '%s' translations (file: %s)"
 606                     logger.warning(
 607                          msg % (color_text(remote_lang, "RED"), local_file)
 608                     )
 609                     try:
 610                         self.do_url_request(
 611                             'push_translation', multipart=True, method='PUT',
 612                             files=[(
 613                                     "%s;%s" % (resource_slug, remote_lang),
 614                                     self.get_full_path(local_file)
 615                             )], language=remote_lang
 616                         )
 617                         logger.debug("Translation %s pushed." % remote_lang)
 618                     except Exception, e:
 619                         if not skip:
 620                             raise e
 621                         else:
 622                             logger.error(e)
 623
 624     def delete(self, resources=[], languages=[], skip=False, force=False):
 625         """Delete translations."""
 626         resource_list = self.get_chosen_resources(resources)
 627         self.skip = skip
 628         self.force = force
 629
 630         if not languages:
 631             delete_func = self._delete_resource
 632         else:
 633             delete_func = self._delete_translations
 634
 635         for resource in resource_list:
 636             project_slug, resource_slug = resource.split('.')
 637             host = self.get_resource_host(resource)
 638             self.url_info = {
 639                 'host': host,
 640                 'project': project_slug,
 641                 'resource': resource_slug
 642             }
 643             logger.debug("URL data are: %s" % self.url_info)
 644             project_details = parse_json(
 645                 self.do_url_request('project_details', project=self)
 646             )
 647             teams = project_details['teams']
 648             stats = self._get_stats_for_resource()
 649             delete_func(project_details, resource, stats, languages)
 650
 651     def _delete_resource(self, project_details, resource, stats, *args):
 652         """Delete a resource from Transifex."""
 653         project_slug, resource_slug = resource.split('.')
 654         project_resource_slugs = [
 655             r['slug'] for r in project_details['resources']
 656         ]
 657         logger.info("Deleting resource %s:" % resource)
 658         if resource_slug not in project_resource_slugs:
 659             if not self.skip:
 660                 msg = "Skipping: %s : Resource does not exist."
 661                 logger.info(msg % resource)
 662             return
 663         if not self.force:
 664             slang = self.get_resource_option(resource, 'source_lang')
 665             for language in stats:
 666                 if language == slang:
 667                     continue
 668                 if int(stats[language]['translated_entities']) > 0:
 669                     msg = (
 670                         "Skipping: %s : Unable to delete resource because it "
 671                         "has a not empty %s translation.\nPlease use -f or "
 672                         "--force option to delete this resource."
 673                     )
 674                     logger.info(msg % (resource, language))
 675                     return
 676         try:
 677             self.do_url_request('delete_resource', method="DELETE")
 678             self.config.remove_section(resource)
 679             self.save()
 680             msg = "Deleted resource %s of project %s."
 681             logger.info(msg % (resource_slug, project_slug))
 682         except Exception, e:
 683             msg = "Unable to delete resource %s of project %s."
 684             logger.error(msg % (resource_slug, project_slug))
 685             if not self.skip:
 686                 raise
 687
 688     def _delete_translations(self, project_details, resource, stats, languages):
 689         """Delete the specified translations for the specified resource."""
 690         logger.info("Deleting translations from resource %s:" % resource)
 691         for language in languages:
 692             self._delete_translation(project_details, resource, stats, language)
 693
 694     def _delete_translation(self, project_details, resource, stats, language):
 695         """Delete a specific translation from the specified resource."""
 696         project_slug, resource_slug = resource.split('.')
 697         if language not in stats:
 698             if not self.skip:
 699                 msg = "Skipping %s: Translation does not exist."
 700                 logger.warning(msg % (language))
 701             return
 702         if not self.force:
 703             teams = project_details['teams']
 704             if language in teams:
 705                 msg = (
 706                     "Skipping %s: Unable to delete translation because it is "
 707                     "associated with a team.\nPlease use -f or --force option "
 708                     "to delete this translation."
 709                 )
 710                 logger.warning(msg % language)
 711                 return
 712             if int(stats[language]['translated_entities']) > 0:
 713                 msg = (
 714                     "Skipping %s: Unable to delete translation because it "
 715                     "is not empty.\nPlease use -f or --force option to delete "
 716                     "this translation."
 717                 )
 718                 logger.warning(msg % language)
 719                 return
 720         try:
 721             self.do_url_request(
 722                 'delete_translation', language=language, method="DELETE"
 723             )
 724             msg = "Deleted language %s from resource %s of project %s."
 725             logger.info(msg % (language, resource_slug, project_slug))
 726         except Exception, e:
 727             msg = "Unable to delete translation %s"
 728             logger.error(msg % language)
 729             if not self.skip:
 730                 raise
 731
 732     def do_url_request(self, api_call, multipart=False, data=None,
 733                        files=[], encoding=None, method="GET", **kwargs):
 734         """
 735         Issues a url request.
 736         """
 737         # Read the credentials from the config file (.transifexrc)
 738         host = self.url_info['host']
 739         try:
 740             username = self.txrc.get(host, 'username')
 741             passwd = self.txrc.get(host, 'password')
 742             token = self.txrc.get(host, 'token')
 743             hostname = self.txrc.get(host, 'hostname')
 744         except ConfigParser.NoSectionError:
 745             raise Exception("No user credentials found for host %s. Edit"
 746                 " ~/.transifexrc and add the appropriate info in there." %
 747                 host)
 748
 749         # Create the Url
 750         kwargs['hostname'] = hostname
 751         kwargs.update(self.url_info)
 752         url = (API_URLS[api_call] % kwargs).encode('UTF-8')
 753         logger.debug(url)
 754
 755         opener = None
 756         headers = None
 757         req = None
 758
 759         if multipart:
 760             opener = urllib2.build_opener(MultipartPostHandler)
 761             for info,filename in files:
 762                 data = { "resource" : info.split(';')[0],
 763                          "language" : info.split(';')[1],
 764                          "uploaded_file" :  open(filename,'rb') }
 765
 766             urllib2.install_opener(opener)
 767             req = RequestWithMethod(url=url, data=data, method=method)
 768         else:
 769             req = RequestWithMethod(url=url, data=data, method=method)
 770             if encoding:
 771                 req.add_header("Content-Type",encoding)
 772
 773         base64string = base64.encodestring('%s:%s' % (username, passwd))[:-1]
 774         authheader = "Basic %s" % base64string
 775         req.add_header("Authorization", authheader)
 776         req.add_header("Accept-Encoding", "gzip,deflate")
 777         req.add_header("User-Agent", user_agent_identifier())
 778
 779         try:
 780             response = urllib2.urlopen(req, timeout=300)
 781             return http_response(response)
 782         except urllib2.HTTPError, e:
 783             if e.code in [401, 403, 404]:
 784                 raise e
 785             elif 200 <= e.code < 300:
 786                 return None
 787             else:
 788                 # For other requests, we should print the message as well
 789                 raise Exception("Remote server replied: %s" % e.read())
 790         except urllib2.URLError, e:
 791             error = e.args[0]
 792             raise Exception("Remote server replied: %s" % error[1])
 793
 794
 795     def _should_update_translation(self, lang, stats, local_file, force=False,
 796                                    mode=None):
 797         """Whether a translation should be udpated from Transifex.
 798
 799         We use the following criteria for that:
 800         - If user requested to force the download.
 801         - If language exists in Transifex.
 802         - If the local file is older than the Transifex's file.
 803         - If the user requested a x% completion.
 804
 805         Args:
 806             lang: The language code to check.
 807             stats: The (global) statistics object.
 808             local_file: The local translation file.
 809             force: A boolean flag.
 810             mode: The mode for the translation.
 811         Returns:
 812             True or False.
 813         """
 814         return self._should_download(lang, stats, local_file, force)
 815
 816     def _should_add_translation(self, lang, stats, force=False, mode=None):
 817         """Whether a translation should be added from Transifex.
 818
 819         We use the following criteria for that:
 820         - If user requested to force the download.
 821         - If language exists in Transifex.
 822         - If the user requested a x% completion.
 823
 824         Args:
 825             lang: The language code to check.
 826             stats: The (global) statistics object.
 827             force: A boolean flag.
 828             mode: The mode for the translation.
 829         Returns:
 830             True or False.
 831         """
 832         return self._should_download(lang, stats, None, force)
 833
 834     def _should_download(self, lang, stats, local_file=None, force=False,
 835                          mode=None):
 836         """Return whether a translation should be downloaded.
 837
 838         If local_file is None, skip the timestamps check (the file does
 839         not exist locally).
 840         """
 841         try:
 842             lang_stats = stats[lang]
 843         except KeyError, e:
 844             logger.debug("No lang %s in statistics" % lang)
 845             return False
 846
 847         satisfies_min = self._satisfies_min_translated(lang_stats, mode)
 848         if not satisfies_min:
 849             return False
 850
 851         if force:
 852             logger.debug("Downloading translation due to -f")
 853             return True
 854
 855         if local_file is not None:
 856             remote_update = self._extract_updated(lang_stats)
 857             if not self._remote_is_newer(remote_update, local_file):
 858                 logger.debug("Local is newer than remote for lang %s" % lang)
 859                 return False
 860         return True
 861
 862     def _should_push_translation(self, lang, stats, local_file, force=False):
 863         """Return whether a local translation file should be
 864         pushed to Trasnifex.
 865
 866         We use the following criteria for that:
 867         - If user requested to force the upload.
 868         - If language exists in Transifex.
 869         - If local file is younger than the remote file.
 870
 871         Args:
 872             lang: The language code to check.
 873             stats: The (global) statistics object.
 874             local_file: The local translation file.
 875             force: A boolean flag.
 876         Returns:
 877             True or False.
 878         """
 879         if force:
 880             logger.debug("Push translation due to -f.")
 881             return True
 882         try:
 883             lang_stats = stats[lang]
 884         except KeyError, e:
 885             logger.debug("Language %s does not exist in Transifex." % lang)
 886             return True
 887         if local_file is not None:
 888             remote_update = self._extract_updated(lang_stats)
 889             if self._remote_is_newer(remote_update, local_file):
 890                 msg  = "Remote translation is newer than local file for lang %s"
 891                 logger.debug(msg % lang)
 892                 return False
 893         return True
 894
 895     def _generate_timestamp(self, update_datetime):
 896         """Generate a UNIX timestamp from the argument.
 897
 898         Args:
 899             update_datetime: The datetime in the format used by Transifex.
 900         Returns:
 901             A float, representing the timestamp that corresponds to the
 902             argument.
 903         """
 904         time_format = "%Y-%m-%d %H:%M:%S"
 905         return time.mktime(
 906             datetime.datetime(
 907                 *time.strptime(update_datetime, time_format)[0:5]
 908             ).utctimetuple()
 909         )
 910
 911     def _get_time_of_local_file(self, path):
 912         """Get the modified time of the path_.
 913
 914         Args:
 915             path: The path we want the mtime for.
 916         Returns:
 917             The time as a timestamp or None, if the file does not exist
 918         """
 919         if not os.path.exists(path):
 920             return None
 921         return time.mktime(time.gmtime(os.path.getmtime(path)))
 922
 923     def _satisfies_min_translated(self, stats, mode=None):
 924         """Check whether a translation fulfills the filter used for
 925         minimum translated percentage.
 926
 927         Args:
 928             perc: The current translation percentage.
 929         Returns:
 930             True or False
 931         """
 932         cur = self._extract_completed(stats, mode)
 933         option_name = 'minimum_perc'
 934         if self.minimum_perc is not None:
 935             minimum_percent = self.minimum_perc
 936         else:
 937             global_minimum = int(
 938                 self.get_resource_option('main', option_name) or 0
 939             )
 940             resource_minimum = int(
 941                 self.get_resource_option(
 942                     self.resource, option_name
 943                 ) or global_minimum
 944             )
 945             minimum_percent = resource_minimum
 946         return cur >= minimum_percent
 947
 948     def _remote_is_newer(self, remote_updated, local_file):
 949         """Check whether the remote translation is newer that the local file.
 950
 951         Args:
 952             remote_updated: The date and time the translation was last
 953                 updated remotely.
 954             local_file: The local file.
 955         Returns:
 956             True or False.
 957         """
 958         if remote_updated is None:
 959             logger.debug("No remote time")
 960             return False
 961         remote_time = self._generate_timestamp(remote_updated)
 962         local_time = self._get_time_of_local_file(
 963             self.get_full_path(local_file)
 964         )
 965         logger.debug(
 966             "Remote time is %s and local %s" % (remote_time, local_time)
 967         )
 968         if local_time is not None and remote_time < local_time:
 969             return False
 970         return True
 971
 972     @classmethod
 973     def _extract_completed(cls, stats, mode=None):
 974         """Extract the information for the translated percentage from the stats.
 975
 976         Args:
 977             stats: The stats object for a language as returned by Transifex.
 978             mode: The mode of translations requested.
 979         Returns:
 980             The percentage of translation as integer.
 981         """
 982         if mode == 'reviewed':
 983             key = 'reviewed_percentage'
 984         else:
 985             key = 'completed'
 986         try:
 987             return int(stats[key][:-1])
 988         except KeyError, e:
 989             return 0
 990
 991     @classmethod
 992     def _extract_updated(cls, stats):
 993         """Extract the  information for the last update of a translation.
 994
 995         Args:
 996             stats: The stats object for a language as returned by Transifex.
 997         Returns:
 998             The last update field.
 999         """
1000         try:
1001             return stats['last_update']
1002         except KeyError, e:
1003             return None
1004
1005     def _new_translations_to_add(self, files, slang, lang_map,
1006                                  stats, force=False):
1007         """Return a list of translations which are new to the
1008         local installation.
1009         """
1010         new_translations = []
1011         timestamp = time.time()
1012         langs = stats.keys()
1013         logger.debug("Available languages are: %s" % langs)
1014
1015         for lang in langs:
1016             lang_exists = lang in files.keys()
1017             lang_is_source = lang == slang
1018             mapped_lang_exists = (
1019                 lang in lang_map and lang_map[lang] in files.keys()
1020             )
1021             if lang_exists or lang_is_source or mapped_lang_exists:
1022                 continue
1023             if self._should_add_translation(lang, stats, force):
1024                 new_translations.append(lang)
1025         return set(new_translations)
1026
1027     def _get_stats_for_resource(self):
1028         """Get the statistics information for a resource."""
1029         try:
1030             r = self.do_url_request('resource_stats')
1031             logger.debug("Statistics response is %s" % r)
1032             stats = parse_json(r)
1033         except urllib2.HTTPError, e:
1034             logger.debug("Resource not found: %s" % e)
1035             stats = {}
1036         except Exception,e:
1037             logger.debug("Network error: %s" % e)
1038             raise
1039         return stats
1040
1041     def get_chosen_resources(self, resources):
1042         """Get the resources the user selected.
1043
1044         Support wildcards in the resources specified by the user.
1045
1046         Args:
1047             resources: A list of resources as specified in command-line or
1048                 an empty list.
1049         Returns:
1050             A list of resources.
1051         """
1052         configured_resources = self.get_resource_list()
1053         if not resources:
1054             return configured_resources
1055
1056         selected_resources = []
1057         for resource in resources:
1058             found = False
1059             for full_name in configured_resources:
1060                 if fnmatch.fnmatch(full_name, resource):
1061                     selected_resources.append(full_name)
1062                     found = True
1063             if not found:
1064                 msg = "Specified resource '%s' does not exist."
1065                 raise Exception(msg % resource)
1066         logger.debug("Operating on resources: %s" % selected_resources)
1067         return selected_resources
1068
1069     def _languages_to_pull(self, languages, files, lang_map, stats, force):
1070         """Get a set of langauges to pull.
1071
1072         Args:
1073             languages: A list of languages the user selected in cmd.
1074             files: A dictionary of current local translation files.
1075         Returns:
1076             A tuple of a set of existing languages and new translations.
1077         """
1078         if not languages:
1079             pull_languages = set([])
1080             pull_languages |= set(files.keys())
1081             mapped_files = []
1082             for lang in pull_languages:
1083                 if lang in lang_map.flip:
1084                     mapped_files.append(lang_map.flip[lang])
1085             pull_languages -= set(lang_map.flip.keys())
1086             pull_languages |= set(mapped_files)
1087             return (pull_languages, set([]))
1088         else:
1089             pull_languages = []
1090             new_translations = []
1091             f_langs = files.keys()
1092             for l in languages:
1093                 if l not in f_langs and not (l in lang_map and lang_map[l] in f_langs):
1094                     if self._should_add_translation(l, stats, force):
1095                         new_translations.append(l)
1096                 else:
1097                     if l in lang_map.keys():
1098                         l = lang_map[l]
1099                     pull_languages.append(l)
1100             return (set(pull_languages), set(new_translations))
1101
1102     def _extension_for(self, i18n_type):
1103         """Return the extension used for the specified type."""
1104         try:
1105             res = parse_json(self.do_url_request('formats'))
1106             return res[i18n_type]['file-extensions'].split(',')[0]
1107         except Exception,e:
1108             logger.error(e)
1109             return ''
1110
1111     def _resource_exists(self, stats):
1112         """Check if resource exists.
1113
1114         Args:
1115             stats: The statistics dict as returned by Tx.
1116         Returns:
1117             True, if the resource exists in the server.
1118         """
1119         return bool(stats)
1120
1121     def _create_resource(self, resource, pslug, fileinfo, filename, **kwargs):
1122         """Create a resource.
1123
1124         Args:
1125             resource: The full resource name.
1126             pslug: The slug of the project.
1127             fileinfo: The information of the resource.
1128             filename: The name of the file.
1129         Raises:
1130             URLError, in case of a problem.
1131         """
1132         multipart = True
1133         method = "POST"
1134         api_call = 'create_resource'
1135
1136         host = self.url_info['host']
1137         try:
1138             username = self.txrc.get(host, 'username')
1139             passwd = self.txrc.get(host, 'password')
1140             token = self.txrc.get(host, 'token')
1141             hostname = self.txrc.get(host, 'hostname')
1142         except ConfigParser.NoSectionError:
1143             raise Exception("No user credentials found for host %s. Edit"
1144                 " ~/.transifexrc and add the appropriate info in there." %
1145                 host)
1146
1147         # Create the Url
1148         kwargs['hostname'] = hostname
1149         kwargs.update(self.url_info)
1150         kwargs['project'] = pslug
1151         url = (API_URLS[api_call] % kwargs).encode('UTF-8')
1152
1153         opener = None
1154         headers = None
1155         req = None
1156
1157         i18n_type = self._get_option(resource, 'type')
1158         if i18n_type is None:
1159             logger.error(
1160                 "Please define the resource type in .tx/config (eg. type = PO)."
1161                 " More info: http://bit.ly/txcl-rt"
1162             )
1163
1164         opener = urllib2.build_opener(MultipartPostHandler)
1165         data = {
1166             "slug": fileinfo.split(';')[0],
1167             "name": fileinfo.split(';')[0],
1168             "uploaded_file":  open(filename,'rb'),
1169             "i18n_type": i18n_type
1170         }
1171         urllib2.install_opener(opener)
1172         req = RequestWithMethod(url=url, data=data, method=method)
1173
1174         base64string = base64.encodestring('%s:%s' % (username, passwd))[:-1]
1175         authheader = "Basic %s" % base64string
1176         req.add_header("Authorization", authheader)
1177
1178         try:
1179             fh = urllib2.urlopen(req)
1180         except urllib2.HTTPError, e:
1181             if e.code in [401, 403, 404]:
1182                 raise e
1183             else:
1184                 # For other requests, we should print the message as well
1185                 raise Exception("Remote server replied: %s" % e.read())
1186         except urllib2.URLError, e:
1187             error = e.args[0]
1188             raise Exception("Remote server replied: %s" % error[1])
1189
1190         raw = fh.read()
1191         fh.close()
1192         return raw
1193
1194     def _get_option(self, resource, option):
1195         """Get the value for the option in the config file.
1196
1197         If the option is not in the resource section, look for it in
1198         the project.
1199
1200         Args:
1201             resource: The resource name.
1202             option: The option the value of which we are interested in.
1203         Returns:
1204             The option value or None, if it does not exist.
1205         """
1206         value = self.get_resource_option(resource, option)
1207         if value is None:
1208             if self.config.has_option('main', option):
1209                 return self.config.get('main', option)
1210         return value
1211
1212     def set_i18n_type(self, resources, i18n_type):
1213         """Set the type for the specified resources."""
1214         self._set_resource_option(resources, key='type', value=i18n_type)
1215
1216     def set_min_perc(self, resources, perc):
1217         """Set the minimum percentage for the resources."""
1218         self._set_resource_option(resources, key='minimum_perc', value=perc)
1219
1220     def set_default_mode(self, resources, mode):
1221         """Set the default mode for the specified resources."""
1222         self._set_resource_option(resources, key='mode', value=mode)
1223
1224     def _set_resource_option(self, resources, key, value):
1225         """Set options in the config file.
1226
1227         If resources is empty. set the option globally.
1228         """
1229         if not resources:
1230             self.config.set('main', key, value)
1231             return
1232         for r in resources:
1233             self.config.set(r, key, value)