diff options
| author | Raphael Collet <rco@openerp.com> | 2016-09-02 15:58:39 +0200 |
|---|---|---|
| committer | Raphael Collet <rco@openerp.com> | 2016-09-02 17:28:12 +0200 |
| commit | 9e64f9f9514194d74e166b032add8511182f2677 (patch) | |
| tree | 5086710c04c6b923be18a6d1d97b3f0f3a2a9c4a /odoo/tools | |
| parent | 11812b0b9e82b2b1b9041d8250ce01e7b4a5f7ee (diff) | |
[REF] openerp: move `openerp` to `odoo`
Diffstat (limited to 'odoo/tools')
33 files changed, 9733 insertions, 0 deletions
diff --git a/odoo/tools/__init__.py b/odoo/tools/__init__.py new file mode 100644 index 00000000000..6e657a8ddd3 --- /dev/null +++ b/odoo/tools/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import copy +import win32 +import appdirs +from config import config +from misc import * +from convert import * +from translate import * +from graph import graph +from image import * +from amount_to_text import * +from amount_to_text_en import * +from pdf_utils import * +from yaml_import import * +from sql import * +from float_utils import * +from mail import * +from func import * +from debugger import * diff --git a/odoo/tools/amount_to_text.py b/odoo/tools/amount_to_text.py new file mode 100644 index 00000000000..8e89b48fbc4 --- /dev/null +++ b/odoo/tools/amount_to_text.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +#------------------------------------------------------------- +# French +#------------------------------------------------------------- + + +to_19_fr = ( u'zéro', 'un', 'deux', 'trois', 'quatre', 'cinq', 'six', + 'sept', 'huit', 'neuf', 'dix', 'onze', 'douze', 'treize', + 'quatorze', 'quinze', 'seize', 'dix-sept', 'dix-huit', 'dix-neuf' ) +tens_fr = ( 'vingt', 'trente', 'quarante', 'Cinquante', 'Soixante', 'Soixante-dix', 'Quatre-vingts', 'Quatre-vingt Dix') +denom_fr = ( '', + 'Mille', 'Millions', 'Milliards', 'Billions', 'Quadrillions', + 'Quintillion', 'Sextillion', 'Septillion', 'Octillion', 'Nonillion', + 'Décillion', 'Undecillion', 'Duodecillion', 'Tredecillion', 'Quattuordecillion', + 'Sexdecillion', 'Septendecillion', 'Octodecillion', 'Icosillion', 'Vigintillion' ) + +def _convert_nn_fr(val): + """ convert a value < 100 to French + """ + if val < 20: + return to_19_fr[val] + for (dcap, dval) in ((k, 20 + (10 * v)) for (v, k) in enumerate(tens_fr)): + if dval + 10 > val: + if val % 10: + return dcap + '-' + to_19_fr[val % 10] + return dcap + +def _convert_nnn_fr(val): + """ convert a value < 1000 to french + + special cased because it is the level that kicks + off the < 100 special case. The rest are more general. This also allows you to + get strings in the form of 'forty-five hundred' if called directly. + """ + word = '' + (mod, rem) = (val % 100, val // 100) + if rem > 0: + word = to_19_fr[rem] + ' Cent' + if mod > 0: + word += ' ' + if mod > 0: + word += _convert_nn_fr(mod) + return word + +def french_number(val): + if val < 100: + return _convert_nn_fr(val) + if val < 1000: + return _convert_nnn_fr(val) + for (didx, dval) in ((v - 1, 1000 ** v) for v in range(len(denom_fr))): + if dval > val: + mod = 1000 ** didx + l = val // mod + r = val - (l * mod) + ret = _convert_nnn_fr(l) + ' ' + denom_fr[didx] + if r > 0: + ret = ret + ', ' + french_number(r) + return ret + +def amount_to_text_fr(number, currency): + number = '%.2f' % number + units_name = currency + list = str(number).split('.') + start_word = french_number(abs(int(list[0]))) + end_word = french_number(int(list[1])) + cents_number = int(list[1]) + cents_name = (cents_number > 1) and ' Cents' or ' Cent' + final_result = start_word +' '+units_name+' '+ end_word +' '+cents_name + return final_result + +#------------------------------------------------------------- +# Dutch +#------------------------------------------------------------- + +to_19_nl = ( 'Nul', 'Een', 'Twee', 'Drie', 'Vier', 'Vijf', 'Zes', + 'Zeven', 'Acht', 'Negen', 'Tien', 'Elf', 'Twaalf', 'Dertien', + 'Veertien', 'Vijftien', 'Zestien', 'Zeventien', 'Achttien', 'Negentien' ) +tens_nl = ( 'Twintig', 'Dertig', 'Veertig', 'Vijftig', 'Zestig', 'Zeventig', 'Tachtig', 'Negentig') +denom_nl = ( '', + 'Duizend', 'Miljoen', 'Miljard', 'Triljoen', 'Quadriljoen', + 'Quintillion', 'Sextiljoen', 'Septillion', 'Octillion', 'Nonillion', + 'Decillion', 'Undecillion', 'Duodecillion', 'Tredecillion', 'Quattuordecillion', + 'Sexdecillion', 'Septendecillion', 'Octodecillion', 'Novemdecillion', 'Vigintillion' ) + +def _convert_nn_nl(val): + """ convert a value < 100 to Dutch + """ + if val < 20: + return to_19_nl[val] + for (dcap, dval) in ((k, 20 + (10 * v)) for (v, k) in enumerate(tens_nl)): + if dval + 10 > val: + if val % 10: + return dcap + '-' + to_19_nl[val % 10] + return dcap + +def _convert_nnn_nl(val): + """ convert a value < 1000 to Dutch + + special cased because it is the level that kicks + off the < 100 special case. The rest are more general. This also allows you to + get strings in the form of 'forty-five hundred' if called directly. + """ + word = '' + (mod, rem) = (val % 100, val // 100) + if rem > 0: + word = to_19_nl[rem] + ' Honderd' + if mod > 0: + word += ' ' + if mod > 0: + word += _convert_nn_nl(mod) + return word + +def dutch_number(val): + if val < 100: + return _convert_nn_nl(val) + if val < 1000: + return _convert_nnn_nl(val) + for (didx, dval) in ((v - 1, 1000 ** v) for v in range(len(denom_nl))): + if dval > val: + mod = 1000 ** didx + l = val // mod + r = val - (l * mod) + ret = _convert_nnn_nl(l) + ' ' + denom_nl[didx] + if r > 0: + ret = ret + ', ' + dutch_number(r) + return ret + +def amount_to_text_nl(number, currency): + number = '%.2f' % number + units_name = currency + list = str(number).split('.') + start_word = dutch_number(int(list[0])) + end_word = dutch_number(int(list[1])) + cents_number = int(list[1]) + cents_name = (cents_number > 1) and 'cent' or 'cent' + final_result = start_word +' '+units_name+' '+ end_word +' '+cents_name + return final_result + +#------------------------------------------------------------- +# Generic functions +#------------------------------------------------------------- + +_translate_funcs = {'fr' : amount_to_text_fr, 'nl' : amount_to_text_nl} + +def add_amount_to_text_function(lang, func): + _translate_funcs[lang] = func + +#TODO: we should use the country AND language (ex: septante VS soixante dix) +#TODO: we should use en by default, but the translation func is yet to be implemented +def amount_to_text(nbr, lang='fr', currency='euro'): + """ Converts an integer to its textual representation, using the language set in the context if any. + + Example:: + + 1654: mille six cent cinquante-quatre. + """ +# if nbr > 1000000: +##TODO: use logger +# print "WARNING: number too large '%d', can't translate it!" % (nbr,) +# return str(nbr) + + if not _translate_funcs.has_key(lang): +#TODO: use logger + print "WARNING: no translation function found for lang: '%s'" % (lang,) +#TODO: (default should be en) same as above + lang = 'fr' + return _translate_funcs[lang](abs(nbr), currency) + +if __name__=='__main__': + from sys import argv + + lang = 'nl' + if len(argv) < 2: + for i in range(1,200): + print i, ">>", amount_to_text(i, lang) + for i in range(200,999999,139): + print i, ">>", amount_to_text(i, lang) + else: + print amount_to_text(int(argv[1]), lang) diff --git a/odoo/tools/amount_to_text_en.py b/odoo/tools/amount_to_text_en.py new file mode 100644 index 00000000000..46657251701 --- /dev/null +++ b/odoo/tools/amount_to_text_en.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import logging +from translate import _ + +_logger = logging.getLogger(__name__) + +#------------------------------------------------------------- +#ENGLISH +#------------------------------------------------------------- + +to_19 = ( 'Zero', 'One', 'Two', 'Three', 'Four', 'Five', 'Six', + 'Seven', 'Eight', 'Nine', 'Ten', 'Eleven', 'Twelve', 'Thirteen', + 'Fourteen', 'Fifteen', 'Sixteen', 'Seventeen', 'Eighteen', 'Nineteen' ) +tens = ( 'Twenty', 'Thirty', 'Forty', 'Fifty', 'Sixty', 'Seventy', 'Eighty', 'Ninety') +denom = ( '', + 'Thousand', 'Million', 'Billion', 'Trillion', 'Quadrillion', + 'Quintillion', 'Sextillion', 'Septillion', 'Octillion', 'Nonillion', + 'Decillion', 'Undecillion', 'Duodecillion', 'Tredecillion', 'Quattuordecillion', + 'Sexdecillion', 'Septendecillion', 'Octodecillion', 'Novemdecillion', 'Vigintillion' ) + +def _convert_nn(val): + """convert a value < 100 to English. + """ + if val < 20: + return to_19[val] + for (dcap, dval) in ((k, 20 + (10 * v)) for (v, k) in enumerate(tens)): + if dval + 10 > val: + if val % 10: + return dcap + '-' + to_19[val % 10] + return dcap + +def _convert_nnn(val): + """ + convert a value < 1000 to english, special cased because it is the level that kicks + off the < 100 special case. The rest are more general. This also allows you to + get strings in the form of 'forty-five hundred' if called directly. + """ + word = '' + (mod, rem) = (val % 100, val // 100) + if rem > 0: + word = to_19[rem] + ' Hundred' + if mod > 0: + word += ' ' + if mod > 0: + word += _convert_nn(mod) + return word + +def english_number(val): + if val < 100: + return _convert_nn(val) + if val < 1000: + return _convert_nnn(val) + for (didx, dval) in ((v - 1, 1000 ** v) for v in range(len(denom))): + if dval > val: + mod = 1000 ** didx + l = val // mod + r = val - (l * mod) + ret = _convert_nnn(l) + ' ' + denom[didx] + if r > 0: + ret = ret + ', ' + english_number(r) + return ret + +def amount_to_text(number, currency): + number = '%.2f' % number + units_name = currency + list = str(number).split('.') + start_word = english_number(int(list[0])) + end_word = english_number(int(list[1])) + cents_number = int(list[1]) + cents_name = (cents_number > 1) and 'Cents' or 'Cent' + + return ' '.join(filter(None, [start_word, units_name, (start_word or units_name) and (end_word or cents_name) and 'and', end_word, cents_name])) + + +#------------------------------------------------------------- +# Generic functions +#------------------------------------------------------------- + +_translate_funcs = {'en' : amount_to_text} + +#TODO: we should use the country AND language (ex: septante VS soixante dix) +#TODO: we should use en by default, but the translation func is yet to be implemented +def amount_to_text(nbr, lang='en', currency='euro'): + """ Converts an integer to its textual representation, using the language set in the context if any. + + Example:: + + 1654: thousands six cent cinquante-quatre. + """ + import openerp.loglevels as loglevels +# if nbr > 10000000: +# _logger.warning(_("Number too large '%d', can not translate it")) +# return str(nbr) + + if not _translate_funcs.has_key(lang): + _logger.warning(_("no translation function found for lang: '%s'"), lang) + #TODO: (default should be en) same as above + lang = 'en' + return _translate_funcs[lang](abs(nbr), currency) + +if __name__=='__main__': + from sys import argv + + lang = 'nl' + if len(argv) < 2: + for i in range(1,200): + print i, ">>", int_to_text(i, lang) + for i in range(200,999999,139): + print i, ">>", int_to_text(i, lang) + else: + print int_to_text(int(argv[1]), lang) diff --git a/odoo/tools/appdirs.py b/odoo/tools/appdirs.py new file mode 100644 index 00000000000..bb46a9ee27e --- /dev/null +++ b/odoo/tools/appdirs.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2005-2010 ActiveState Software Inc. +# Copyright (c) 2013 Eddy Petrișor + +"""Utilities for determining application-specific dirs. + +See <http://github.com/ActiveState/appdirs> for details and usage. +""" +# Dev Notes: +# - MSDN on where to store app data files: +# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 +# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html +# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + +__version_info__ = (1, 3, 0) +__version__ = '.'.join(map(str, __version_info__)) + + +import sys +import os + +PY3 = sys.version_info[0] == 3 + +if PY3: + unicode = str + + + +def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user data directories are: + Mac OS X: ~/Library/Application Support/<AppName> + Unix: ~/.local/share/<AppName> # or in $XDG_DATA_HOME, if defined + Win XP (not roaming): C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName> + Win XP (roaming): C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName> + Win 7 (not roaming): C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName> + Win 7 (roaming): C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName> + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by deafult "~/.local/share/<AppName>". + """ + if sys.platform == "win32": + if appauthor is None: + appauthor = appname + const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" + path = os.path.normpath(_get_win_folder(const)) + if appname: + path = os.path.join(path, appauthor, appname) + elif sys.platform == 'darwin': + path = os.path.expanduser('~/Library/Application Support/') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of data dirs should be + returned. By default, the first item from XDG_DATA_DIRS is + returned, or '/usr/local/share/<AppName>', + if XDG_DATA_DIRS is not set + + Typical user data directories are: + Mac OS X: /Library/Application Support/<AppName> + Unix: /usr/local/share/<AppName> or /usr/share/<AppName> + Win XP: C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName> + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + Win 7: C:\ProgramData\<AppAuthor>\<AppName> # Hidden, but writeable on Win 7. + + For Unix, this is using the $XDG_DATA_DIRS[0] default. + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if sys.platform == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) + if appname: + path = os.path.join(path, appauthor, appname) + elif sys.platform == 'darwin': + path = os.path.expanduser('/Library/Application Support') + if appname: + path = os.path.join(path, appname) + else: + # XDG default for $XDG_DATA_DIRS + # only first, if multipath is False + path = os.getenv('XDG_DATA_DIRS', + os.pathsep.join(['/usr/local/share', '/usr/share'])) + pathlist = [ os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) ] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [ os.sep.join([x, appname]) for x in pathlist ] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + if appname and version: + path = os.path.join(path, version) + return path + + +def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific config dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx> + for a discussion of issues. + + Typical user data directories are: + Mac OS X: same as user_data_dir + Unix: ~/.config/<AppName> # or in $XDG_CONFIG_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by deafult "~/.local/share/<AppName>". + """ + if sys.platform in [ "win32", "darwin" ]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of config dirs should be + returned. By default, the first item from XDG_CONFIG_DIRS is + returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set + + Typical user data directories are: + Mac OS X: same as site_data_dir + Unix: /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in + $XDG_CONFIG_DIRS + Win *: same as site_data_dir + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + + For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if sys.platform in [ "win32", "darwin" ]: + path = site_data_dir(appname, appauthor) + if appname and version: + path = os.path.join(path, version) + else: + # XDG default for $XDG_CONFIG_DIRS + # only first, if multipath is False + path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') + pathlist = [ os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) ] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [ os.sep.join([x, appname]) for x in pathlist ] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + +def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific cache dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Cache" to the base app data dir for Windows. See + discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Caches/<AppName> + Unix: ~/.cache/<AppName> (XDG default) + Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache + Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache + + On Windows the only suggestion in the MSDN docs is that local settings go in + the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming + app data dir (the default returned by `user_data_dir` above). Apps typically + put cache data somewhere *under* the given dir here. Some examples: + ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache + ...\Acme\SuperApp\Cache\1.0 + OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. + This can be disabled with the `opinion=False` option. + """ + if sys.platform == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + if appname: + path = os.path.join(path, appauthor, appname) + if opinion: + path = os.path.join(path, "Cache") + elif sys.platform == 'darwin': + path = os.path.expanduser('~/Library/Caches') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + +def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific log dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be "<major>.<minor>". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Logs" to the base app data dir for Windows, and "log" to the + base cache dir for Unix. See discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Logs/<AppName> + Unix: ~/.cache/<AppName>/log # or under $XDG_CACHE_HOME if defined + Win XP: C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs + Vista: C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs + + On Windows the only suggestion in the MSDN docs is that local settings + go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in + examples of what some windows apps use for a logs dir.) + + OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` + value for Windows and appends "log" to the user cache dir for Unix. + This can be disabled with the `opinion=False` option. + """ + if sys.platform == "darwin": + path = os.path.join( + os.path.expanduser('~/Library/Logs'), + appname) + elif sys.platform == "win32": + path = user_data_dir(appname, appauthor, version); version=False + if opinion: + path = os.path.join(path, "Logs") + else: + path = user_cache_dir(appname, appauthor, version); version=False + if opinion: + path = os.path.join(path, "log") + if appname and version: + path = os.path.join(path, version) + return path + + +class AppDirs(object): + """Convenience wrapper for getting application dirs.""" + def __init__(self, appname, appauthor=None, version=None, + roaming=False, multipath=False): + self.appname = appname + self.appauthor = appauthor + self.version = version + self.roaming = roaming + self.multipath = multipath + @property + def user_data_dir(self): + return user_data_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + @property + def site_data_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + @property + def user_config_dir(self): + return user_config_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + @property + def site_config_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + @property + def user_cache_dir(self): + return user_cache_dir(self.appname, self.appauthor, + version=self.version) + @property + def user_log_dir(self): + return user_log_dir(self.appname, self.appauthor, + version=self.version) + + + + +#---- internal support stuff + +def _get_win_folder_from_registry(csidl_name): + """This is a fallback technique at best. I'm not sure if using the + registry for this guarantees us the correct answer for all CSIDL_* + names. + """ + import _winreg + + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + }[csidl_name] + + key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") + dir, type = _winreg.QueryValueEx(key, shell_folder_name) + return dir + +def _get_win_folder_with_pywin32(csidl_name): + from win32com.shell import shellcon, shell + dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) + # Try to make this a unicode path because SHGetFolderPath does + # not return unicode strings when there is unicode data in the + # path. + try: + dir = unicode(dir) + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + try: + import win32api + dir = win32api.GetShortPathName(dir) + except ImportError: + pass + except UnicodeError: + pass + return dir + +def _get_win_folder_with_ctypes(csidl_name): + import ctypes + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + }[csidl_name] + + buf = ctypes.create_unicode_buffer(1024) + ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if have highbit chars. See + # <http://bugs.activestate.com/show_bug.cgi?id=85099>. + has_high_char = False + for c in buf: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf2 = ctypes.create_unicode_buffer(1024) + if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + return buf.value + +if sys.platform == "win32": + try: + import win32com.shell + _get_win_folder = _get_win_folder_with_pywin32 + except ImportError: + try: + import ctypes + _get_win_folder = _get_win_folder_with_ctypes + except ImportError: + _get_win_folder = _get_win_folder_from_registry + + + +#---- self test code + +if __name__ == "__main__": + appname = "MyApp" + appauthor = "MyCompany" + + props = ("user_data_dir", "site_data_dir", + "user_config_dir", "site_config_dir", + "user_cache_dir", "user_log_dir") + + print("-- app dirs (with optional 'version')") + dirs = AppDirs(appname, appauthor, version="1.0") + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'version')") + dirs = AppDirs(appname, appauthor) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'appauthor')") + dirs = AppDirs(appname) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) diff --git a/odoo/tools/assertion_report.py b/odoo/tools/assertion_report.py new file mode 100644 index 00000000000..72930991527 --- /dev/null +++ b/odoo/tools/assertion_report.py @@ -0,0 +1,28 @@ + +class assertion_report(object): + """ + Simple pair of success and failures counts (used to record YAML and XML + `assert` tags as well as unittest tests outcome (in this case, not + individual `assert`)). + """ + def __init__(self): + self.successes = 0 + self.failures = 0 + + def record_success(self): + self.successes += 1 + + def record_failure(self): + self.failures += 1 + + def record_result(self, result): + if result is None: + pass + elif result is True: + self.record_success() + elif result is False: + self.record_failure() + + def __str__(self): + res = 'Assertions report: %s successes, %s failures' % (self.successes, self.failures) + return res diff --git a/odoo/tools/cache.py b/odoo/tools/cache.py new file mode 100644 index 00000000000..e71ab9dd121 --- /dev/null +++ b/odoo/tools/cache.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +# decorator makes wrappers that have the same API as their wrapped function; +# this is important for the openerp.api.guess() that relies on signatures +from collections import defaultdict +from decorator import decorator +from inspect import formatargspec, getargspec +import logging + +unsafe_eval = eval + +_logger = logging.getLogger(__name__) + + +class ormcache_counter(object): + """ Statistic counters for cache entries. """ + __slots__ = ['hit', 'miss', 'err'] + + def __init__(self): + self.hit = 0 + self.miss = 0 + self.err = 0 + + @property + def ratio(self): + return 100.0 * self.hit / (self.hit + self.miss or 1) + +# statistic counters dictionary, maps (dbname, modelname, method) to counter +STAT = defaultdict(ormcache_counter) + + +class ormcache(object): + """ LRU cache decorator for model methods. + The parameters are strings that represent expressions referring to the + signature of the decorated method, and are used to compute a cache key:: + + @ormcache('model_name', 'mode') + def _compute_domain(self, model_name, mode="read"): + ... + + For the sake of backward compatibility, the decorator supports the named + parameter `skiparg`:: + + @ormcache(skiparg=1) + def _compute_domain(self, model_name, mode="read"): + ... + """ + def __init__(self, *args, **kwargs): + self.args = args + self.skiparg = kwargs.get('skiparg') + + def __call__(self, method): + self.method = method + self.determine_key() + lookup = decorator(self.lookup, method) + lookup.clear_cache = self.clear + return lookup + + def determine_key(self): + """ Determine the function that computes a cache key from arguments. """ + if self.skiparg is None: + # build a string that represents function code and evaluate it + args = formatargspec(*getargspec(self.method))[1:-1] + if self.args: + code = "lambda %s: (%s,)" % (args, ", ".join(self.args)) + else: + code = "lambda %s: ()" % (args,) + self.key = unsafe_eval(code) + else: + # backward-compatible function that uses self.skiparg + self.key = lambda *args, **kwargs: args[self.skiparg:] + + def lru(self, model): + counter = STAT[(model.pool.db_name, model._name, self.method)] + return model.pool.cache, (model._name, self.method), counter + + def lookup(self, method, *args, **kwargs): + d, key0, counter = self.lru(args[0]) + key = key0 + self.key(*args, **kwargs) + try: + r = d[key] + counter.hit += 1 + return r + except KeyError: + counter.miss += 1 + value = d[key] = self.method(*args, **kwargs) + return value + except TypeError: + counter.err += 1 + return self.method(*args, **kwargs) + + def clear(self, model, *args): + """ Clear the registry cache """ + d, key0, _ = self.lru(model) + d.clear() + model.pool.cache_cleared = True + + +class ormcache_context(ormcache): + """ This LRU cache decorator is a variant of :class:`ormcache`, with an + extra parameter ``keys`` that defines a sequence of dictionary keys. Those + keys are looked up in the ``context`` parameter and combined to the cache + key made by :class:`ormcache`. + """ + def __init__(self, *args, **kwargs): + super(ormcache_context, self).__init__(*args, **kwargs) + self.keys = kwargs['keys'] + + def determine_key(self): + """ Determine the function that computes a cache key from arguments. """ + assert self.skiparg is None, "ormcache_context() no longer supports skiparg" + # build a string that represents function code and evaluate it + spec = getargspec(self.method) + args = formatargspec(*spec)[1:-1] + cont_expr = "(context or {})" if 'context' in spec.args else "self._context" + keys_expr = "tuple(map(%s.get, %r))" % (cont_expr, self.keys) + if self.args: + code = "lambda %s: (%s, %s)" % (args, ", ".join(self.args), keys_expr) + else: + code = "lambda %s: (%s,)" % (args, keys_expr) + self.key = unsafe_eval(code) + + +class ormcache_multi(ormcache): + """ This LRU cache decorator is a variant of :class:`ormcache`, with an + extra parameter ``multi`` that gives the name of a parameter. Upon call, the + corresponding argument is iterated on, and every value leads to a cache + entry under its own key. + """ + def __init__(self, *args, **kwargs): + super(ormcache_multi, self).__init__(*args, **kwargs) + self.multi = kwargs['multi'] + + def determine_key(self): + """ Determine the function that computes a cache key from arguments. """ + assert self.skiparg is None, "ormcache_multi() no longer supports skiparg" + assert isinstance(self.multi, basestring), "ormcache_multi() parameter multi must be an argument name" + + super(ormcache_multi, self).determine_key() + + # key_multi computes the extra element added to the key + spec = getargspec(self.method) + args = formatargspec(*spec)[1:-1] + code_multi = "lambda %s: %s" % (args, self.multi) + self.key_multi = unsafe_eval(code_multi) + + # self.multi_pos is the position of self.multi in args + self.multi_pos = spec.args.index(self.multi) + + def lookup(self, method, *args, **kwargs): + d, key0, counter = self.lru(args[0]) + base_key = key0 + self.key(*args, **kwargs) + ids = self.key_multi(*args, **kwargs) + result = {} + missed = [] + + # first take what is available in the cache + for i in ids: + key = base_key + (i,) + try: + result[i] = d[key] + counter.hit += 1 + except Exception: + counter.miss += 1 + missed.append(i) + + if missed: + # call the method for the ids that were not in the cache; note that + # thanks to decorator(), the multi argument will be bound and passed + # positionally in args. + args = list(args) + args[self.multi_pos] = missed + result.update(method(*args, **kwargs)) + + # store those new results back in the cache + for i in missed: + key = base_key + (i,) + d[key] = result[i] + + return result + + +class dummy_cache(object): + """ Cache decorator replacement to actually do no caching. """ + def __init__(self, *l, **kw): + pass + + def __call__(self, fn): + fn.clear_cache = self.clear + return fn + + def clear(self, *l, **kw): + pass + + +def log_ormcache_stats(sig=None, frame=None): + """ Log statistics of ormcache usage by database, model, and method. """ + from openerp.modules.registry import Registry + import threading + + me = threading.currentThread() + me_dbname = me.dbname + entries = defaultdict(int) + for dbname, reg in Registry.registries.iteritems(): + for key in reg.cache.iterkeys(): + entries[(dbname,) + key[:2]] += 1 + for key, count in sorted(entries.items()): + dbname, model_name, method = key + me.dbname = dbname + stat = STAT[key] + _logger.info("%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s", + count, stat.hit, stat.miss, stat.err, stat.ratio, model_name, method.__name__) + + me.dbname = me_dbname + + +def get_cache_key_counter(bound_method, *args, **kwargs): + """ Return the cache, key and stat counter for the given call. """ + model = bound_method.im_self + ormcache = bound_method.clear_cache.im_self + cache, key0, counter = ormcache.lru(model) + key = key0 + ormcache.key(model, *args, **kwargs) + return cache, key, counter + +# For backward compatibility +cache = ormcache diff --git a/odoo/tools/config.py b/odoo/tools/config.py new file mode 100644 index 00000000000..3526b1738de --- /dev/null +++ b/odoo/tools/config.py @@ -0,0 +1,588 @@ +#openerp.loggers.handlers. -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import ConfigParser +import optparse +import os +import sys +import openerp +import openerp.conf +import openerp.loglevels as loglevels +import logging +import openerp.release as release +import appdirs + +class MyOption (optparse.Option, object): + """ optparse Option with two additional attributes. + + The list of command line options (getopt.Option) is used to create the + list of the configuration file options. When reading the file, and then + reading the command line arguments, we don't want optparse.parse results + to override the configuration file values. But if we provide default + values to optparse, optparse will return them and we can't know if they + were really provided by the user or not. A solution is to not use + optparse's default attribute, but use a custom one (that will be copied + to create the default values of the configuration file). + + """ + def __init__(self, *opts, **attrs): + self.my_default = attrs.pop('my_default', None) + super(MyOption, self).__init__(*opts, **attrs) + +DEFAULT_LOG_HANDLER = ':INFO' +def _get_default_datadir(): + home = os.path.expanduser('~') + if os.path.isdir(home): + func = appdirs.user_data_dir + else: + if sys.platform in ['win32', 'darwin']: + func = appdirs.site_data_dir + else: + func = lambda **kwarg: "/var/lib/%s" % kwarg['appname'].lower() + # No "version" kwarg as session and filestore paths are shared against series + return func(appname=release.product_name, appauthor=release.author) + +def _deduplicate_loggers(loggers): + """ Avoid saving multiple logging levels for the same loggers to a save + file, that just takes space and the list can potentially grow unbounded + if for some odd reason people use :option`odoo.py --save`` all the time. + """ + # dict(iterable) -> the last item of iterable for any given key wins, + # which is what we want and expect. Output order should not matter as + # there are no duplicates within the output sequence + return ( + '{}:{}'.format(logger, level) + for logger, level in dict(it.split(':') for it in loggers).iteritems() + ) + + +class configmanager(object): + def __init__(self, fname=None): + """Constructor. + + :param fname: a shortcut allowing to instantiate :class:`configmanager` + from Python code without resorting to environment + variable + """ + # Options not exposed on the command line. Command line options will be added + # from optparse's parser. + self.options = { + 'admin_passwd': 'admin', + 'csv_internal_sep': ',', + 'publisher_warranty_url': 'http://services.openerp.com/publisher-warranty/', + 'reportgz': False, + 'root_path': None, + } + + # Not exposed in the configuration file. + self.blacklist_for_save = set([ + 'publisher_warranty_url', 'load_language', 'root_path', + 'init', 'save', 'config', 'update', 'stop_after_init', 'dev_mode' + ]) + + # dictionary mapping option destination (keys in self.options) to MyOptions. + self.casts = {} + + self.misc = {} + self.config_file = fname + + self._LOGLEVELS = dict([ + (getattr(loglevels, 'LOG_%s' % x), getattr(logging, x)) + for x in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET') + ]) + + version = "%s %s" % (release.description, release.version) + self.parser = parser = optparse.OptionParser(version=version, option_class=MyOption) + + # Server startup config + group = optparse.OptionGroup(parser, "Common options") + group.add_option("-c", "--config", dest="config", help="specify alternate config file") + group.add_option("-s", "--save", action="store_true", dest="save", default=False, + help="save configuration to ~/.openerp_serverrc") + group.add_option("-i", "--init", dest="init", help="install one or more modules (comma-separated list, use \"all\" for all modules), requires -d") + group.add_option("-u", "--update", dest="update", + help="update one or more modules (comma-separated list, use \"all\" for all modules). Requires -d.") + group.add_option("--without-demo", dest="without_demo", + help="disable loading demo data for modules to be installed (comma-separated, use \"all\" for all modules). Requires -d and -i. Default is %default", + my_default=False) + group.add_option("-P", "--import-partial", dest="import_partial", my_default='', + help="Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.") + group.add_option("--pidfile", dest="pidfile", help="file where the server pid will be stored") + group.add_option("--addons-path", dest="addons_path", + help="specify additional addons paths (separated by commas).", + action="callback", callback=self._check_addons_path, nargs=1, type="string") + group.add_option("--load", dest="server_wide_modules", help="Comma-separated list of server-wide modules default=web") + + group.add_option("-D", "--data-dir", dest="data_dir", my_default=_get_default_datadir(), + help="Directory where to store Odoo data") + parser.add_option_group(group) + + # XML-RPC / HTTP + group = optparse.OptionGroup(parser, "XML-RPC Configuration") + group.add_option("--xmlrpc-interface", dest="xmlrpc_interface", my_default='', + help="Specify the TCP IP address for the XML-RPC protocol. The empty string binds to all interfaces.") + group.add_option("--xmlrpc-port", dest="xmlrpc_port", my_default=8069, + help="specify the TCP port for the XML-RPC protocol", type="int") + group.add_option("--no-xmlrpc", dest="xmlrpc", action="store_false", my_default=True, + help="disable the XML-RPC protocol") + group.add_option("--proxy-mode", dest="proxy_mode", action="store_true", my_default=False, + help="Enable correct behavior when behind a reverse proxy") + group.add_option("--longpolling-port", dest="longpolling_port", my_default=8072, + help="specify the TCP port for longpolling requests", type="int") + parser.add_option_group(group) + + # WEB + group = optparse.OptionGroup(parser, "Web interface Configuration") + group.add_option("--db-filter", dest="dbfilter", my_default='.*', + help="Filter listed database", metavar="REGEXP") + parser.add_option_group(group) + + # Testing Group + group = optparse.OptionGroup(parser, "Testing Configuration") + group.add_option("--test-file", dest="test_file", my_default=False, + help="Launch a python or YML test file.") + group.add_option("--test-report-directory", dest="test_report_directory", my_default=False, + help="If set, will save sample of all reports in this directory.") + group.add_option("--test-enable", action="store_true", dest="test_enable", + my_default=False, help="Enable YAML and unit tests.") + group.add_option("--test-commit", action="store_true", dest="test_commit", + my_default=False, help="Commit database changes performed by YAML or XML tests.") + parser.add_option_group(group) + + # Logging Group + group = optparse.OptionGroup(parser, "Logging Configuration") + group.add_option("--logfile", dest="logfile", help="file where the server log will be stored") + group.add_option("--logrotate", dest="logrotate", action="store_true", my_default=False, help="enable logfile rotation") + group.add_option("--syslog", action="store_true", dest="syslog", my_default=False, help="Send the log to the syslog server") + group.add_option('--log-handler', action="append", default=[], my_default=DEFAULT_LOG_HANDLER, metavar="PREFIX:LEVEL", help='setup a handler at LEVEL for a given PREFIX. An empty PREFIX indicates the root logger. This option can be repeated. Example: "openerp.orm:DEBUG" or "werkzeug:CRITICAL" (default: ":INFO")') + group.add_option('--log-request', action="append_const", dest="log_handler", const="openerp.http.rpc.request:DEBUG", help='shortcut for --log-handler=openerp.http.rpc.request:DEBUG') + group.add_option('--log-response', action="append_const", dest="log_handler", const="openerp.http.rpc.response:DEBUG", help='shortcut for --log-handler=openerp.http.rpc.response:DEBUG') + group.add_option('--log-web', action="append_const", dest="log_handler", const="openerp.http:DEBUG", help='shortcut for --log-handler=openerp.http:DEBUG') + group.add_option('--log-sql', action="append_const", dest="log_handler", const="openerp.sql_db:DEBUG", help='shortcut for --log-handler=openerp.sql_db:DEBUG') + group.add_option('--log-db', dest='log_db', help="Logging database", my_default=False) + group.add_option('--log-db-level', dest='log_db_level', my_default='warning', help="Logging database level") + # For backward-compatibility, map the old log levels to something + # quite close. + levels = [ + 'info', 'debug_rpc', 'warn', 'test', 'critical', + 'debug_sql', 'error', 'debug', 'debug_rpc_answer', 'notset' + ] + group.add_option('--log-level', dest='log_level', type='choice', + choices=levels, my_default='info', + help='specify the level of the logging. Accepted values: %s.' % (levels,)) + + parser.add_option_group(group) + + # SMTP Group + group = optparse.OptionGroup(parser, "SMTP Configuration") + group.add_option('--email-from', dest='email_from', my_default=False, + help='specify the SMTP email address for sending email') + group.add_option('--smtp', dest='smtp_server', my_default='localhost', + help='specify the SMTP server for sending email') + group.add_option('--smtp-port', dest='smtp_port', my_default=25, + help='specify the SMTP port', type="int") + group.add_option('--smtp-ssl', dest='smtp_ssl', action='store_true', my_default=False, + help='if passed, SMTP connections will be encrypted with SSL (STARTTLS)') + group.add_option('--smtp-user', dest='smtp_user', my_default=False, + help='specify the SMTP username for sending email') + group.add_option('--smtp-password', dest='smtp_password', my_default=False, + help='specify the SMTP password for sending email') + parser.add_option_group(group) + + group = optparse.OptionGroup(parser, "Database related options") + group.add_option("-d", "--database", dest="db_name", my_default=False, + help="specify the database name") + group.add_option("-r", "--db_user", dest="db_user", my_default=False, + help="specify the database user name") + group.add_option("-w", "--db_password", dest="db_password", my_default=False, + help="specify the database password") + group.add_option("--pg_path", dest="pg_path", help="specify the pg executable path") + group.add_option("--db_host", dest="db_host", my_default=False, + help="specify the database host") + group.add_option("--db_port", dest="db_port", my_default=False, + help="specify the database port", type="int") + group.add_option("--db_maxconn", dest="db_maxconn", type='int', my_default=64, + help="specify the the maximum number of physical connections to posgresql") + group.add_option("--db-template", dest="db_template", my_default="template1", + help="specify a custom database template to create a new database") + parser.add_option_group(group) + + group = optparse.OptionGroup(parser, "Internationalisation options", + "Use these options to translate Odoo to another language." + "See i18n section of the user manual. Option '-d' is mandatory." + "Option '-l' is mandatory in case of importation" + ) + group.add_option('--load-language', dest="load_language", + help="specifies the languages for the translations you want to be loaded") + group.add_option('-l', "--language", dest="language", + help="specify the language of the translation file. Use it with --i18n-export or --i18n-import") + group.add_option("--i18n-export", dest="translate_out", + help="export all sentences to be translated to a CSV file, a PO file or a TGZ archive and exit") + group.add_option("--i18n-import", dest="translate_in", + help="import a CSV or a PO file with translations and exit. The '-l' option is required.") + group.add_option("--i18n-overwrite", dest="overwrite_existing_translations", action="store_true", my_default=False, + help="overwrites existing translation terms on updating a module or importing a CSV or a PO file.") + group.add_option("--modules", dest="translate_modules", + help="specify modules to export. Use in combination with --i18n-export") + parser.add_option_group(group) + + security = optparse.OptionGroup(parser, 'Security-related options') + security.add_option('--no-database-list', action="store_false", dest='list_db', my_default=True, + help="disable the ability to return the list of databases") + parser.add_option_group(security) + + # Advanced options + group = optparse.OptionGroup(parser, "Advanced options") + group.add_option('--dev', dest='dev_mode', type="string", + help="Enable developer mode. Param: List of options separated by comma. " + "Options : all, [pudb|wdb|ipdb|pdb], reload, qweb, werkzeug, xml") + group.add_option("--stop-after-init", action="store_true", dest="stop_after_init", my_default=False, + help="stop the server after its initialization") + group.add_option("--osv-memory-count-limit", dest="osv_memory_count_limit", my_default=False, + help="Force a limit on the maximum number of records kept in the virtual " + "osv_memory tables. The default is False, which means no count-based limit.", + type="int") + group.add_option("--osv-memory-age-limit", dest="osv_memory_age_limit", my_default=1.0, + help="Force a limit on the maximum age of records kept in the virtual " + "osv_memory tables. This is a decimal value expressed in hours, " + "and the default is 1 hour.", + type="float") + group.add_option("--max-cron-threads", dest="max_cron_threads", my_default=2, + help="Maximum number of threads processing concurrently cron jobs (default 2).", + type="int") + group.add_option("--unaccent", dest="unaccent", my_default=False, action="store_true", + help="Use the unaccent function provided by the database when available.") + group.add_option("--geoip-db", dest="geoip_database", my_default='/usr/share/GeoIP/GeoLiteCity.dat', + help="Absolute path to the GeoIP database file.") + parser.add_option_group(group) + + if os.name == 'posix': + group = optparse.OptionGroup(parser, "Multiprocessing options") + # TODO sensible default for the three following limits. + group.add_option("--workers", dest="workers", my_default=0, + help="Specify the number of workers, 0 disable prefork mode.", + type="int") + group.add_option("--limit-memory-soft", dest="limit_memory_soft", my_default=2048 * 1024 * 1024, + help="Maximum allowed virtual memory per worker, when reached the worker be reset after the current request (default 671088640 aka 640MB).", + type="int") + group.add_option("--limit-memory-hard", dest="limit_memory_hard", my_default=2560 * 1024 * 1024, + help="Maximum allowed virtual memory per worker, when reached, any memory allocation will fail (default 805306368 aka 768MB).", + type="int") + group.add_option("--limit-time-cpu", dest="limit_time_cpu", my_default=60, + help="Maximum allowed CPU time per request (default 60).", + type="int") + group.add_option("--limit-time-real", dest="limit_time_real", my_default=120, + help="Maximum allowed Real time per request (default 120).", + type="int") + group.add_option("--limit-time-real-cron", dest="limit_time_real_cron", my_default=-1, + help="Maximum allowed Real time per cron job. (default: --limit-time-real). " + "Set to 0 for no limit. ", + type="int") + group.add_option("--limit-request", dest="limit_request", my_default=8192, + help="Maximum number of request to be processed per worker (default 8192).", + type="int") + parser.add_option_group(group) + + # Copy all optparse options (i.e. MyOption) into self.options. + for group in parser.option_groups: + for option in group.option_list: + if option.dest not in self.options: + self.options[option.dest] = option.my_default + self.casts[option.dest] = option + + # generate default config + self._parse_config() + + def parse_config(self, args=None): + """ Parse the configuration file (if any) and the command-line + arguments. + + This method initializes openerp.tools.config and openerp.conf (the + former should be removed in the furture) with library-wide + configuration values. + + This method must be called before proper usage of this library can be + made. + + Typical usage of this method: + + openerp.tools.config.parse_config(sys.argv[1:]) + """ + self._parse_config(args) + openerp.netsvc.init_logger() + openerp.modules.module.initialize_sys_path() + + def _parse_config(self, args=None): + if args is None: + args = [] + opt, args = self.parser.parse_args(args) + + def die(cond, msg): + if cond: + self.parser.error(msg) + + # Ensures no illegitimate argument is silently discarded (avoids insidious "hyphen to dash" problem) + die(args, "unrecognized parameters: '%s'" % " ".join(args)) + + die(bool(opt.syslog) and bool(opt.logfile), + "the syslog and logfile options are exclusive") + + die(opt.translate_in and (not opt.language or not opt.db_name), + "the i18n-import option cannot be used without the language (-l) and the database (-d) options") + + die(opt.overwrite_existing_translations and not (opt.translate_in or opt.update), + "the i18n-overwrite option cannot be used without the i18n-import option or without the update option") + + die(opt.translate_out and (not opt.db_name), + "the i18n-export option cannot be used without the database (-d) option") + + # Check if the config file exists (-c used, but not -s) + die(not opt.save and opt.config and not os.access(opt.config, os.R_OK), + "The config file '%s' selected with -c/--config doesn't exist or is not readable, "\ + "use -s/--save if you want to generate it"% opt.config) + + # place/search the config file on Win32 near the server installation + # (../etc from the server) + # if the server is run by an unprivileged user, he has to specify location of a config file where he has the rights to write, + # else he won't be able to save the configurations, or even to start the server... + # TODO use appdirs + if os.name == 'nt': + rcfilepath = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'openerp-server.conf') + else: + rcfilepath = os.path.expanduser('~/.openerp_serverrc') + + self.rcfile = os.path.abspath( + self.config_file or opt.config or os.environ.get('OPENERP_SERVER') or rcfilepath) + self.load() + + # Verify that we want to log or not, if not the output will go to stdout + if self.options['logfile'] in ('None', 'False'): + self.options['logfile'] = False + # the same for the pidfile + if self.options['pidfile'] in ('None', 'False'): + self.options['pidfile'] = False + + # if defined dont take the configfile value even if the defined value is None + keys = ['xmlrpc_interface', 'xmlrpc_port', 'longpolling_port', + 'db_name', 'db_user', 'db_password', 'db_host', + 'db_port', 'db_template', 'logfile', 'pidfile', 'smtp_port', + 'email_from', 'smtp_server', 'smtp_user', 'smtp_password', + 'db_maxconn', 'import_partial', 'addons_path', + 'xmlrpc', 'syslog', 'without_demo', + 'dbfilter', 'log_level', 'log_db', + 'log_db_level', 'geoip_database', 'dev_mode' + ] + + for arg in keys: + # Copy the command-line argument (except the special case for log_handler, due to + # action=append requiring a real default, so we cannot use the my_default workaround) + if getattr(opt, arg): + self.options[arg] = getattr(opt, arg) + # ... or keep, but cast, the config file value. + elif isinstance(self.options[arg], basestring) and self.casts[arg].type in optparse.Option.TYPE_CHECKER: + self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg]) + + if isinstance(self.options['log_handler'], basestring): + self.options['log_handler'] = self.options['log_handler'].split(',') + self.options['log_handler'].extend(opt.log_handler) + + # if defined but None take the configfile value + keys = [ + 'language', 'translate_out', 'translate_in', 'overwrite_existing_translations', + 'dev_mode', 'smtp_ssl', 'load_language', + 'stop_after_init', 'logrotate', 'without_demo', 'xmlrpc', 'syslog', + 'list_db', 'proxy_mode', + 'test_file', 'test_enable', 'test_commit', 'test_report_directory', + 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent', + 'data_dir', + ] + + posix_keys = [ + 'workers', + 'limit_memory_hard', 'limit_memory_soft', + 'limit_time_cpu', 'limit_time_real', 'limit_request', 'limit_time_real_cron' + ] + + if os.name == 'posix': + keys += posix_keys + else: + self.options.update(dict.fromkeys(posix_keys, None)) + + # Copy the command-line arguments... + for arg in keys: + if getattr(opt, arg) is not None: + self.options[arg] = getattr(opt, arg) + # ... or keep, but cast, the config file value. + elif isinstance(self.options[arg], basestring) and self.casts[arg].type in optparse.Option.TYPE_CHECKER: + self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg]) + + self.options['root_path'] = os.path.abspath(os.path.expanduser(os.path.expandvars(os.path.dirname(openerp.__file__)))) + if not self.options['addons_path'] or self.options['addons_path']=='None': + default_addons = [] + base_addons = os.path.join(self.options['root_path'], 'addons') + if os.path.exists(base_addons): + default_addons.append(base_addons) + main_addons = os.path.abspath(os.path.join(self.options['root_path'], '../addons')) + if os.path.exists(main_addons): + default_addons.append(main_addons) + self.options['addons_path'] = ','.join(default_addons) + else: + self.options['addons_path'] = ",".join( + os.path.abspath(os.path.expanduser(os.path.expandvars(x.strip()))) + for x in self.options['addons_path'].split(',')) + + self.options['init'] = opt.init and dict.fromkeys(opt.init.split(','), 1) or {} + self.options['demo'] = (self.options['init'] + if not self.options['without_demo'] else {}) + self.options['update'] = opt.update and dict.fromkeys(opt.update.split(','), 1) or {} + self.options['translate_modules'] = opt.translate_modules and map(lambda m: m.strip(), opt.translate_modules.split(',')) or ['all'] + self.options['translate_modules'].sort() + + dev_split = opt.dev_mode and map(str.strip, opt.dev_mode.split(',')) or [] + self.options['dev_mode'] = 'all' in dev_split and dev_split + ['pdb', 'reload', 'qweb', 'werkzeug', 'xml'] or dev_split + + if opt.pg_path: + self.options['pg_path'] = opt.pg_path + + if self.options.get('language', False): + if len(self.options['language']) > 5: + raise Exception('ERROR: The Lang name must take max 5 chars, Eg: -lfr_BE') + + if opt.save: + self.save() + + openerp.conf.addons_paths = self.options['addons_path'].split(',') + if opt.server_wide_modules: + openerp.conf.server_wide_modules = map(lambda m: m.strip(), opt.server_wide_modules.split(',')) + else: + openerp.conf.server_wide_modules = ['web','web_kanban'] + + def _is_addons_path(self, path): + for f in os.listdir(path): + modpath = os.path.join(path, f) + if os.path.isdir(modpath): + def hasfile(filename): + return os.path.isfile(os.path.join(modpath, filename)) + if hasfile('__init__.py') and (hasfile('__openerp__.py') or hasfile('__terp__.py')): + return True + return False + + def _check_addons_path(self, option, opt, value, parser): + ad_paths = [] + for path in value.split(','): + path = path.strip() + res = os.path.abspath(os.path.expanduser(path)) + if not os.path.isdir(res): + raise optparse.OptionValueError("option %s: no such directory: %r" % (opt, path)) + if not self._is_addons_path(res): + raise optparse.OptionValueError("option %s: The addons-path %r does not seem to a be a valid Addons Directory!" % (opt, path)) + ad_paths.append(res) + + setattr(parser.values, option.dest, ",".join(ad_paths)) + + def load(self): + p = ConfigParser.ConfigParser() + try: + p.read([self.rcfile]) + for (name,value) in p.items('options'): + if value=='True' or value=='true': + value = True + if value=='False' or value=='false': + value = False + self.options[name] = value + #parse the other sections, as well + for sec in p.sections(): + if sec == 'options': + continue + if not self.misc.has_key(sec): + self.misc[sec]= {} + for (name, value) in p.items(sec): + if value=='True' or value=='true': + value = True + if value=='False' or value=='false': + value = False + self.misc[sec][name] = value + except IOError: + pass + except ConfigParser.NoSectionError: + pass + + def save(self): + p = ConfigParser.ConfigParser() + loglevelnames = dict(zip(self._LOGLEVELS.values(), self._LOGLEVELS.keys())) + p.add_section('options') + for opt in sorted(self.options.keys()): + if opt in ('version', 'language', 'translate_out', 'translate_in', 'overwrite_existing_translations', 'init', 'update'): + continue + if opt in self.blacklist_for_save: + continue + if opt in ('log_level',): + p.set('options', opt, loglevelnames.get(self.options[opt], self.options[opt])) + elif opt == 'log_handler': + p.set('options', opt, ','.join(_deduplicate_loggers(self.options[opt]))) + else: + p.set('options', opt, self.options[opt]) + + for sec in sorted(self.misc.keys()): + p.add_section(sec) + for opt in sorted(self.misc[sec].keys()): + p.set(sec,opt,self.misc[sec][opt]) + + # try to create the directories and write the file + try: + rc_exists = os.path.exists(self.rcfile) + if not rc_exists and not os.path.exists(os.path.dirname(self.rcfile)): + os.makedirs(os.path.dirname(self.rcfile)) + try: + p.write(file(self.rcfile, 'w')) + if not rc_exists: + os.chmod(self.rcfile, 0600) + except IOError: + sys.stderr.write("ERROR: couldn't write the config file\n") + + except OSError: + # what to do if impossible? + sys.stderr.write("ERROR: couldn't create the config directory\n") + + def get(self, key, default=None): + return self.options.get(key, default) + + def pop(self, key, default=None): + return self.options.pop(key, default) + + def get_misc(self, sect, key, default=None): + return self.misc.get(sect,{}).get(key, default) + + def __setitem__(self, key, value): + self.options[key] = value + if key in self.options and isinstance(self.options[key], basestring) and \ + key in self.casts and self.casts[key].type in optparse.Option.TYPE_CHECKER: + self.options[key] = optparse.Option.TYPE_CHECKER[self.casts[key].type](self.casts[key], key, self.options[key]) + + def __getitem__(self, key): + return self.options[key] + + @property + def addons_data_dir(self): + d = os.path.join(self['data_dir'], 'addons', release.series) + if not os.path.exists(d): + os.makedirs(d, 0700) + else: + assert os.access(d, os.W_OK), \ + "%s: directory is not writable" % d + return d + + @property + def session_dir(self): + d = os.path.join(self['data_dir'], 'sessions') + if not os.path.exists(d): + os.makedirs(d, 0700) + else: + assert os.access(d, os.W_OK), \ + "%s: directory is not writable" % d + return d + + def filestore(self, dbname): + return os.path.join(self['data_dir'], 'filestore', dbname) + +config = configmanager() diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py new file mode 100644 index 00000000000..e711e3828e3 --- /dev/null +++ b/odoo/tools/convert.py @@ -0,0 +1,921 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import cStringIO +import csv +import logging +import os.path +import re +import sys +import time + +from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta + +import pytz +from lxml import etree, builder + +import openerp +import openerp.release +from . import assertion_report +from .config import config +from .misc import file_open, unquote, ustr, SKIPPED_ELEMENT_TYPES +from .translate import _ +from .yaml_import import convert_yaml_import +from openerp import SUPERUSER_ID + +_logger = logging.getLogger(__name__) + +# Import of XML records requires the unsafe eval as well, +# almost everywhere, which is ok because it supposedly comes +# from trusted data, but at least we make it obvious now. +unsafe_eval = eval +from .safe_eval import safe_eval + +class ParseError(Exception): + def __init__(self, msg, text, filename, lineno): + self.msg = msg + self.text = text + self.filename = filename + self.lineno = lineno + + def __str__(self): + return '"%s" while parsing %s:%s, near\n%s' \ + % (self.msg, self.filename, self.lineno, self.text) + +class RecordDictWrapper(dict): + """ + Used to pass a record as locals in eval: + records do not strictly behave like dict, so we force them to. + """ + def __init__(self, record): + self.record = record + def __getitem__(self, key): + if key in self.record: + return self.record[key] + return dict.__getitem__(self, key) + +def _get_idref(self, env, model_str, idref): + idref2 = dict(idref, + time=time, + DateTime=datetime, + datetime=datetime, + timedelta=timedelta, + relativedelta=relativedelta, + version=openerp.release.major_version, + ref=self.id_get, + pytz=pytz) + if model_str: + idref2['obj'] = env[model_str].browse + return idref2 + +def _fix_multiple_roots(node): + """ + Surround the children of the ``node`` element of an XML field with a + single root "data" element, to prevent having a document with multiple + roots once parsed separately. + + XML nodes should have one root only, but we'd like to support + direct multiple roots in our partial documents (like inherited view architectures). + As a convention we'll surround multiple root with a container "data" element, to be + ignored later when parsing. + """ + real_nodes = [x for x in node if not isinstance(x, SKIPPED_ELEMENT_TYPES)] + if len(real_nodes) > 1: + data_node = etree.Element("data") + for child in node: + data_node.append(child) + node.append(data_node) + +def _eval_xml(self, node, env): + if node.tag in ('field','value'): + t = node.get('type','char') + f_model = node.get('model', '').encode('utf-8') + if node.get('search'): + f_search = node.get("search",'').encode('utf-8') + f_use = node.get("use",'id').encode('utf-8') + f_name = node.get("name",'').encode('utf-8') + idref2 = {} + if f_search: + idref2 = _get_idref(self, env, f_model, self.idref) + q = unsafe_eval(f_search, idref2) + ids = env[f_model].search(q).ids + if f_use != 'id': + ids = map(lambda x: x[f_use], env[f_model].browse(ids).read([f_use])) + _fields = env[f_model]._fields + if (f_name in _fields) and _fields[f_name].type == 'many2many': + return ids + f_val = False + if len(ids): + f_val = ids[0] + if isinstance(f_val, tuple): + f_val = f_val[0] + return f_val + a_eval = node.get('eval','') + if a_eval: + idref2 = _get_idref(self, env, f_model, self.idref) + try: + return unsafe_eval(a_eval, idref2) + except Exception: + logging.getLogger('openerp.tools.convert.init').error( + 'Could not eval(%s) for %s in %s', a_eval, node.get('name'), env.context) + raise + def _process(s): + matches = re.finditer(r'[^%]%\((.*?)\)[ds]', s) + done = [] + for m in matches: + found = m.group()[1:] + if found in done: + continue + done.append(found) + id = m.groups()[0] + if not id in self.idref: + self.idref[id] = self.id_get(id) + s = s.replace(found, str(self.idref[id])) + s = s.replace('%%', '%') # Quite wierd but it's for (somewhat) backward compatibility sake + return s + + if t == 'xml': + _fix_multiple_roots(node) + return '<?xml version="1.0"?>\n'\ + +_process("".join([etree.tostring(n, encoding='utf-8') for n in node])) + if t == 'html': + return _process("".join([etree.tostring(n, encoding='utf-8') for n in node])) + + data = node.text + if node.get('file'): + with file_open(node.get('file'), 'rb') as f: + data = f.read() + + if t == 'file': + from ..modules import module + path = data.strip() + if not module.get_module_resource(self.module, path): + raise IOError("No such file or directory: '%s' in %s" % ( + path, self.module)) + return '%s,%s' % (self.module, path) + + if t == 'char': + return data + + if t == 'base64': + return data.encode('base64') + + if t == 'int': + d = data.strip() + if d == 'None': + return None + return int(d) + + if t == 'float': + return float(data.strip()) + + if t in ('list','tuple'): + res=[] + for n in node.iterchildren(tag='value'): + res.append(_eval_xml(self, n, env)) + if t=='tuple': + return tuple(res) + return res + elif node.tag == "function": + args = [] + a_eval = node.get('eval','') + # FIXME: should probably be exclusive + if a_eval: + self.idref['ref'] = self.id_get + args = unsafe_eval(a_eval, self.idref) + for n in node: + return_val = _eval_xml(self, n, env) + if return_val is not None: + args.append(return_val) + model = env[node.get('model', '')] + method = node.get('name') + # this one still depends on the old API + return openerp.api.call_kw(model, method, args, {}) + elif node.tag == "test": + return node.text + + +def str2bool(value): + return value.lower() not in ('0', 'false', 'off') + + +class xml_import(object): + + @staticmethod + def nodeattr2bool(node, attr, default=False): + if not node.get(attr): + return default + val = node.get(attr).strip() + if not val: + return default + return str2bool(val) + + def isnoupdate(self, data_node=None): + return self.noupdate or (len(data_node) and self.nodeattr2bool(data_node, 'noupdate', False)) + + def get_context(self, data_node, node, eval_dict): + data_node_context = (len(data_node) and data_node.get('context','').encode('utf8')) + node_context = node.get("context",'').encode('utf8') + context = {} + for ctx in (data_node_context, node_context): + if ctx: + try: + ctx_res = unsafe_eval(ctx, eval_dict) + if isinstance(context, dict): + context.update(ctx_res) + else: + context = ctx_res + except NameError: + # Some contexts contain references that are only valid at runtime at + # client-side, so in that case we keep the original context string + # as it is. We also log it, just in case. + context = ctx + _logger.debug('Context value (%s) for element with id "%s" or its data node does not parse '\ + 'at server-side, keeping original string, in case it\'s meant for client side only', + ctx, node.get('id','n/a'), exc_info=True) + return context + + def get_uid(self, data_node, node): + node_uid = node.get('uid','') or (len(data_node) and data_node.get('uid','')) + if node_uid: + return self.id_get(node_uid) + return self.uid + + def _test_xml_id(self, xml_id): + id = xml_id + if '.' in xml_id: + module, id = xml_id.split('.', 1) + assert '.' not in id, """The ID reference "%s" must contain +maximum one dot. They are used to refer to other modules ID, in the +form: module.record_id""" % (xml_id,) + if module != self.module: + modcnt = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')]) + assert modcnt == 1, """The ID "%s" refers to an uninstalled module""" % (xml_id,) + + if len(id) > 64: + _logger.error('id: %s is to long (max: 64)', id) + + def _tag_delete(self, rec, data_node=None, mode=None): + d_model = rec.get("model") + d_search = rec.get("search",'').encode('utf-8') + d_id = rec.get("id") + records = self.env[d_model] + + if d_search: + idref = _get_idref(self, self.env, d_model, {}) + try: + records = records.search(unsafe_eval(d_search, idref)) + except ValueError: + _logger.warning('Skipping deletion for failed search `%r`', d_search, exc_info=True) + pass + if d_id: + try: + records += records.browse(self.id_get(d_id)) + except ValueError: + # d_id cannot be found. doesn't matter in this case + _logger.warning('Skipping deletion for missing XML ID `%r`', d_id, exc_info=True) + pass + if records: + records.unlink() + + def _remove_ir_values(self, name, value, model): + domain = [('name', '=', name), ('value', '=', value), ('model', '=', model)] + ir_values = self.env['ir.values'].search(domain) + if ir_values: + ir_values.unlink() + return True + + def _tag_report(self, rec, data_node=None, mode=None): + res = {} + for dest,f in (('name','string'),('model','model'),('report_name','name')): + res[dest] = rec.get(f,'').encode('utf8') + assert res[dest], "Attribute %s of report is empty !" % (f,) + for field,dest in (('rml','report_rml'),('file','report_rml'),('xml','report_xml'),('xsl','report_xsl'), + ('attachment','attachment'),('attachment_use','attachment_use'), ('usage','usage'), + ('report_type', 'report_type'), ('parser', 'parser')): + if rec.get(field): + res[dest] = rec.get(field).encode('utf8') + if rec.get('auto'): + res['auto'] = safe_eval(rec.get('auto','False')) + if rec.get('sxw'): + sxw_content = file_open(rec.get('sxw')).read() + res['report_sxw_content'] = sxw_content + if rec.get('header'): + res['header'] = safe_eval(rec.get('header','False')) + + res['multi'] = rec.get('multi') and safe_eval(rec.get('multi','False')) + + xml_id = rec.get('id','').encode('utf8') + self._test_xml_id(xml_id) + + if rec.get('groups'): + g_names = rec.get('groups','').split(',') + groups_value = [] + for group in g_names: + if group.startswith('-'): + group_id = self.id_get(group[1:]) + groups_value.append((3, group_id)) + else: + group_id = self.id_get(group) + groups_value.append((4, group_id)) + res['groups_id'] = groups_value + if rec.get('paperformat'): + pf_name = rec.get('paperformat') + pf_id = self.id_get(pf_name) + res['paperformat_id'] = pf_id + + id = self.env['ir.model.data']._update("ir.actions.report.xml", self.module, res, xml_id, noupdate=self.isnoupdate(data_node), mode=self.mode) + self.idref[xml_id] = int(id) + + if not rec.get('menu') or safe_eval(rec.get('menu','False')): + keyword = str(rec.get('keyword', 'client_print_multi')) + value = 'ir.actions.report.xml,'+str(id) + action = self.env['ir.values'].set_action(res['name'], keyword, res['model'], value) + self.env['ir.actions.report.xml'].browse(id).write({'ir_values_id': action.id}) + elif self.mode=='update' and safe_eval(rec.get('menu','False'))==False: + # Special check for report having attribute menu=False on update + value = 'ir.actions.report.xml,'+str(id) + self._remove_ir_values(res['name'], value, res['model']) + self.env['ir.actions.report.xml'].browse(id).write({'ir_values_id': False}) + return id + + def _tag_function(self, rec, data_node=None, mode=None): + if self.isnoupdate(data_node) and self.mode != 'init': + return + context = self.get_context(data_node, rec, {'ref': self.id_get}) + uid = self.get_uid(data_node, rec) + env = self.env(user=uid, context=context) + _eval_xml(self, rec, env) + return + + def _tag_act_window(self, rec, data_node=None, mode=None): + name = rec.get('name','').encode('utf-8') + xml_id = rec.get('id','').encode('utf8') + self._test_xml_id(xml_id) + type = rec.get('type','').encode('utf-8') or 'ir.actions.act_window' + view_id = False + if rec.get('view_id'): + view_id = self.id_get(rec.get('view_id','').encode('utf-8')) + domain = rec.get('domain','').encode('utf-8') or '[]' + res_model = rec.get('res_model','').encode('utf-8') + src_model = rec.get('src_model','').encode('utf-8') + view_type = rec.get('view_type','').encode('utf-8') or 'form' + view_mode = rec.get('view_mode','').encode('utf-8') or 'tree,form' + usage = rec.get('usage','').encode('utf-8') + limit = rec.get('limit','').encode('utf-8') + uid = self.uid + + # Act_window's 'domain' and 'context' contain mostly literals + # but they can also refer to the variables provided below + # in eval_context, so we need to eval() them before storing. + # Among the context variables, 'active_id' refers to + # the currently selected items in a list view, and only + # takes meaning at runtime on the client side. For this + # reason it must remain a bare variable in domain and context, + # even after eval() at server-side. We use the special 'unquote' + # class to achieve this effect: a string which has itself, unquoted, + # as representation. + active_id = unquote("active_id") + active_ids = unquote("active_ids") + active_model = unquote("active_model") + + # Include all locals() in eval_context, for backwards compatibility + eval_context = { + 'name': name, + 'xml_id': xml_id, + 'type': type, + 'view_id': view_id, + 'domain': domain, + 'res_model': res_model, + 'src_model': src_model, + 'view_type': view_type, + 'view_mode': view_mode, + 'usage': usage, + 'limit': limit, + 'uid' : uid, + 'active_id': active_id, + 'active_ids': active_ids, + 'active_model': active_model, + 'ref': self.id_get, + } + context = self.get_context(data_node, rec, eval_context) + + try: + domain = unsafe_eval(domain, eval_context) + except NameError: + # Some domains contain references that are only valid at runtime at + # client-side, so in that case we keep the original domain string + # as it is. We also log it, just in case. + _logger.debug('Domain value (%s) for element with id "%s" does not parse '\ + 'at server-side, keeping original string, in case it\'s meant for client side only', + domain, xml_id or 'n/a', exc_info=True) + res = { + 'name': name, + 'type': type, + 'view_id': view_id, + 'domain': domain, + 'context': context, + 'res_model': res_model, + 'src_model': src_model, + 'view_type': view_type, + 'view_mode': view_mode, + 'usage': usage, + 'limit': limit, + } + + if rec.get('groups'): + g_names = rec.get('groups','').split(',') + groups_value = [] + for group in g_names: + if group.startswith('-'): + group_id = self.id_get(group[1:]) + groups_value.append((3, group_id)) + else: + group_id = self.id_get(group) + groups_value.append((4, group_id)) + res['groups_id'] = groups_value + + if rec.get('target'): + res['target'] = rec.get('target','') + if rec.get('multi'): + res['multi'] = safe_eval(rec.get('multi', 'False')) + id = self.env['ir.model.data']._update('ir.actions.act_window', self.module, res, xml_id, noupdate=self.isnoupdate(data_node), mode=self.mode) + self.idref[xml_id] = int(id) + + if src_model: + #keyword = 'client_action_relate' + res_id = False + model = src_model + if isinstance(model, (list, tuple)): + model, res_id = model + keyword = rec.get('key2','').encode('utf-8') or 'client_action_relate' + value = 'ir.actions.act_window,'+str(id) + replace = rec.get('replace','') or True + self.env['ir.values'].set_action(xml_id, action_slot=keyword, model=model, action=value, res_id=res_id) + # TODO add remove ir.model.data + + def _tag_ir_set(self, rec, data_node=None, mode=None): + """ + .. deprecated:: 9.0 + + Use the <record> notation with ``ir.values`` as model instead. + """ + if self.mode != 'init': + return + res = {} + for field in rec.findall('./field'): + f_name = field.get("name",'').encode('utf-8') + f_val = _eval_xml(self, field, self.env) + res[f_name] = f_val + ir_values = self.env['ir.values'] + for model in res['models']: + res_id = False + if isinstance(model, (list, tuple)): + model, res_id = model + if res['key'] == 'default': + ir_values.set_default(model, field_name=res['name'], value=res['value'], condition=res['key2']) + elif res['key'] == 'action': + ir_values.set_action(res['name'], action_slot=res['key2'], model=model, action=res['value'], res_id=res_id) + + def _tag_workflow(self, rec, data_node=None, mode=None): + if self.isnoupdate(data_node) and self.mode != 'init': + return + model = rec.get('model').encode('ascii') + w_ref = rec.get('ref') + if w_ref: + id = self.id_get(w_ref) + else: + number_children = len(rec) + assert number_children > 0,\ + 'You must define a child node if you dont give a ref' + assert number_children == 1,\ + 'Only one child node is accepted (%d given)' % number_children + id = _eval_xml(self, rec[0], self.env) + + uid = self.get_uid(data_node, rec) + record = self.env(user=uid)[model].browse(id) + record.signal_workflow(rec.get('action').encode('ascii')) + + def _tag_menuitem(self, rec, data_node=None, mode=None): + rec_id = rec.get("id",'').encode('ascii') + self._test_xml_id(rec_id) + + # The parent attribute was specified, if non-empty determine its ID, otherwise + # explicitly make a top-level menu + if rec.get('parent'): + menu_parent_id = self.id_get(rec.get('parent','')) + else: + # we get here with <menuitem parent="">, explicit clear of parent, or + # if no parent attribute at all but menu name is not a menu path + menu_parent_id = False + values = {'parent_id': menu_parent_id} + if rec.get('name'): + values['name'] = rec.get('name') + try: + res = [ self.id_get(rec.get('id','')) ] + except: + res = None + + if rec.get('action'): + a_action = rec.get('action','').encode('utf8') + + # determine the type of action + action_type, action_id = self.model_id_get(a_action) + action_type = action_type.split('.')[-1] # keep only type part + values['action'] = "ir.actions.%s,%d" % (action_type, action_id) + + if not values.get('name') and action_type in ('act_window', 'wizard', 'url', 'client', 'server'): + a_table = 'ir_act_%s' % action_type.replace('act_', '') + self.cr.execute('select name from "%s" where id=%%s' % a_table, (int(action_id),)) + resw = self.cr.fetchone() + if resw: + values['name'] = resw[0] + + if not values.get('name'): + # ensure menu has a name + values['name'] = rec_id or '?' + + if rec.get('load_xmlid'): + values['load_xmlid'] = True + if rec.get('sequence'): + values['sequence'] = int(rec.get('sequence')) + + if rec.get('groups'): + g_names = rec.get('groups','').split(',') + groups_value = [] + for group in g_names: + if group.startswith('-'): + group_id = self.id_get(group[1:]) + groups_value.append((3, group_id)) + else: + group_id = self.id_get(group) + groups_value.append((4, group_id)) + values['groups_id'] = groups_value + + if not values.get('parent_id'): + if rec.get('web_icon'): + values['web_icon'] = rec.get('web_icon') + + pid = self.env['ir.model.data']._update('ir.ui.menu', self.module, values, rec_id, noupdate=self.isnoupdate(data_node), mode=self.mode, res_id=res and res[0] or False) + + if rec_id and pid: + self.idref[rec_id] = int(pid) + + return 'ir.ui.menu', pid + + def _assert_equals(self, f1, f2, prec=4): + return not round(f1 - f2, prec) + + def _tag_assert(self, rec, data_node=None, mode=None): + if self.isnoupdate(data_node) and self.mode != 'init': + return + + rec_model = rec.get("model",'').encode('ascii') + rec_id = rec.get("id",'').encode('ascii') + self._test_xml_id(rec_id) + rec_src = rec.get("search",'').encode('utf8') + rec_src_count = rec.get("count") + + rec_string = rec.get("string",'').encode('utf8') or 'unknown' + + records = None + eval_dict = {'ref': self.id_get} + context = self.get_context(data_node, rec, eval_dict) + uid = self.get_uid(data_node, rec) + env = self.env(user=uid, context=context) + if rec_id: + records = env[rec_model].browse(self.id_get(rec_id)) + elif rec_src: + q = unsafe_eval(rec_src, eval_dict) + records = env[rec_model].search(q) + if rec_src_count: + count = int(rec_src_count) + if len(records) != count: + self.assertion_report.record_failure() + msg = 'assertion "%s" failed!\n' \ + ' Incorrect search count:\n' \ + ' expected count: %d\n' \ + ' obtained count: %d\n' \ + % (rec_string, count, len(records)) + _logger.error(msg) + return + + assert records is not None,\ + 'You must give either an id or a search criteria' + ref = self.id_get + for record in records: + globals_dict = RecordDictWrapper(record) + globals_dict['floatEqual'] = self._assert_equals + globals_dict['ref'] = ref + globals_dict['_ref'] = ref + for test in rec.findall('./test'): + f_expr = test.get("expr",'').encode('utf-8') + env = self.env(user=uid, context=context) + expected_value = _eval_xml(self, test, env) or True + expression_value = unsafe_eval(f_expr, globals_dict) + if expression_value != expected_value: # assertion failed + self.assertion_report.record_failure() + msg = 'assertion "%s" failed!\n' \ + ' xmltag: %s\n' \ + ' expected value: %r\n' \ + ' obtained value: %r\n' \ + % (rec_string, etree.tostring(test), expected_value, expression_value) + _logger.error(msg) + return + else: # all tests were successful for this assertion tag (no break) + self.assertion_report.record_success() + + def _tag_record(self, rec, data_node=None, mode=None): + rec_model = rec.get("model").encode('ascii') + model = self.env[rec_model] + rec_id = rec.get("id",'').encode('ascii') + rec_context = rec.get("context", {}) + if rec_context: + rec_context = unsafe_eval(rec_context) + + if self.xml_filename and rec_id: + rec_context['install_mode_data'] = dict( + xml_file=self.xml_filename, + xml_id=rec_id, + model=rec_model, + module=self.module + ) + + self._test_xml_id(rec_id) + # in update mode, the record won't be updated if the data node explicitely + # opt-out using @noupdate="1". A second check will be performed in + # ir.model.data#_update() using the record's ir.model.data `noupdate` field. + if self.isnoupdate(data_node) and self.mode != 'init': + # check if the xml record has no id, skip + if not rec_id: + return None + + if '.' in rec_id: + module,rec_id2 = rec_id.split('.') + else: + module = self.module + rec_id2 = rec_id + id = self.env['ir.model.data']._update_dummy(rec_model, module, rec_id2) + if id: + # if the resource already exists, don't update it but store + # its database id (can be useful) + self.idref[rec_id] = int(id) + return None + elif not self.nodeattr2bool(rec, 'forcecreate', True): + # if it doesn't exist and we shouldn't create it, skip it + return None + # else create it normally + + res = {} + for field in rec.findall('./field'): + #TODO: most of this code is duplicated above (in _eval_xml)... + f_name = field.get("name").encode('utf-8') + f_ref = field.get("ref",'').encode('utf-8') + f_search = field.get("search",'').encode('utf-8') + f_model = field.get("model",'').encode('utf-8') + if not f_model and f_name in model._fields: + f_model = model._fields[f_name].comodel_name + f_use = field.get("use",'').encode('utf-8') or 'id' + f_val = False + + if f_search: + q = unsafe_eval(f_search, self.idref) + assert f_model, 'Define an attribute model="..." in your .XML file !' + # browse the objects searched + s = self.env[f_model].search(q) + # column definitions of the "local" object + _fields = self.env[rec_model]._fields + # if the current field is many2many + if (f_name in _fields) and _fields[f_name].type == 'many2many': + f_val = [(6, 0, map(lambda x: x[f_use], s))] + elif len(s): + # otherwise (we are probably in a many2one field), + # take the first element of the search + f_val = s[0][f_use] + elif f_ref: + if f_name in model._fields and model._fields[f_name].type == 'reference': + val = self.model_id_get(f_ref) + f_val = val[0] + ',' + str(val[1]) + else: + f_val = self.id_get(f_ref) + else: + f_val = _eval_xml(self, field, self.env) + if f_name in model._fields: + if model._fields[f_name].type == 'integer': + f_val = int(f_val) + elif model._fields[f_name].type in ['float', 'monetary']: + f_val = float(f_val) + elif model._fields[f_name].type == 'boolean' and isinstance(f_val, basestring): + f_val = str2bool(f_val) + res[f_name] = f_val + + id = self.env(context=rec_context)['ir.model.data']._update(rec_model, self.module, res, rec_id or False, not self.isnoupdate(data_node), noupdate=self.isnoupdate(data_node), mode=self.mode) + if rec_id: + self.idref[rec_id] = int(id) + if config.get('import_partial'): + self.cr.commit() + return rec_model, id + + def _tag_template(self, el, data_node=None, mode=None): + # This helper transforms a <template> element into a <record> and forwards it + tpl_id = el.get('id', el.get('t-name', '')).encode('ascii') + full_tpl_id = tpl_id + if '.' not in full_tpl_id: + full_tpl_id = '%s.%s' % (self.module, tpl_id) + # set the full template name for qweb <module>.<id> + if not el.get('inherit_id'): + el.set('t-name', full_tpl_id) + el.tag = 't' + else: + el.tag = 'data' + el.attrib.pop('id', None) + + record_attrs = { + 'id': tpl_id, + 'model': 'ir.ui.view', + } + for att in ['forcecreate', 'context']: + if att in el.keys(): + record_attrs[att] = el.attrib.pop(att) + + Field = builder.E.field + name = el.get('name', tpl_id) + + record = etree.Element('record', attrib=record_attrs) + record.append(Field(name, name='name')) + record.append(Field(full_tpl_id, name='key')) + record.append(Field("qweb", name='type')) + record.append(Field(el.get('priority', "16"), name='priority')) + if 'inherit_id' in el.attrib: + record.append(Field(name='inherit_id', ref=el.get('inherit_id'))) + if 'website_id' in el.attrib: + record.append(Field(name='website_id', ref=el.get('website_id'))) + if 'key' in el.attrib: + record.append(Field(el.get('key'), name='key')) + if el.get('active') in ("True", "False"): + view_id = self.id_get(tpl_id, raise_if_not_found=False) + if mode != "update" or not view_id: + record.append(Field(name='active', eval=el.get('active'))) + if el.get('customize_show') in ("True", "False"): + record.append(Field(name='customize_show', eval=el.get('customize_show'))) + groups = el.attrib.pop('groups', None) + if groups: + grp_lst = map(lambda x: "ref('%s')" % x, groups.split(',')) + record.append(Field(name="groups_id", eval="[(6, 0, ["+', '.join(grp_lst)+"])]")) + if el.attrib.pop('page', None) == 'True': + record.append(Field(name="page", eval="True")) + if el.get('primary') == 'True': + # Pseudo clone mode, we'll set the t-name to the full canonical xmlid + el.append( + builder.E.xpath( + builder.E.attribute(full_tpl_id, name='t-name'), + expr=".", + position="attributes", + ) + ) + record.append(Field('primary', name='mode')) + # inject complete <template> element (after changing node name) into + # the ``arch`` field + record.append(Field(el, name="arch", type="xml")) + + return self._tag_record(record, data_node) + + def id_get(self, id_str, raise_if_not_found=True): + if id_str in self.idref: + return self.idref[id_str] + res = self.model_id_get(id_str, raise_if_not_found) + return res and res[1] + + def model_id_get(self, id_str, raise_if_not_found=True): + if '.' not in id_str: + id_str = '%s.%s' % (self.module, id_str) + return self.env['ir.model.data'].xmlid_to_res_model_res_id(id_str, raise_if_not_found=raise_if_not_found) + + def parse(self, de, mode=None): + roots = ['openerp','data','odoo'] + if de.tag not in roots: + raise Exception("Root xml tag must be <openerp>, <odoo> or <data>.") + for rec in de: + if rec.tag in roots: + self.parse(rec, mode) + elif rec.tag in self._tags: + try: + self._tags[rec.tag](rec, de, mode=mode) + except Exception, e: + self.cr.rollback() + exc_info = sys.exc_info() + raise ParseError, (ustr(e), etree.tostring(rec).rstrip(), rec.getroottree().docinfo.URL, rec.sourceline), exc_info[2] + return True + + def __init__(self, cr, module, idref, mode, report=None, noupdate=False, xml_filename=None): + self.mode = mode + self.module = module + self.env = openerp.api.Environment(cr, SUPERUSER_ID, {}) + self.cr = cr + self.uid = SUPERUSER_ID + self.idref = idref + if report is None: + report = assertion_report.assertion_report() + self.assertion_report = report + self.noupdate = noupdate + self.xml_filename = xml_filename + self._tags = { + 'record': self._tag_record, + 'delete': self._tag_delete, + 'function': self._tag_function, + 'menuitem': self._tag_menuitem, + 'template': self._tag_template, + 'workflow': self._tag_workflow, + 'report': self._tag_report, + 'ir_set': self._tag_ir_set, # deprecated:: 9.0 + 'act_window': self._tag_act_window, + 'assert': self._tag_assert, + } + +def convert_file(cr, module, filename, idref, mode='update', noupdate=False, kind=None, report=None, pathname=None): + if pathname is None: + pathname = os.path.join(module, filename) + fp = file_open(pathname) + ext = os.path.splitext(filename)[1].lower() + + try: + if ext == '.csv': + convert_csv_import(cr, module, pathname, fp.read(), idref, mode, noupdate) + elif ext == '.sql': + convert_sql_import(cr, fp) + elif ext == '.yml': + convert_yaml_import(cr, module, fp, kind, idref, mode, noupdate, report) + elif ext == '.xml': + convert_xml_import(cr, module, fp, idref, mode, noupdate, report) + elif ext == '.js': + pass # .js files are valid but ignored here. + else: + raise ValueError("Can't load unknown file type %s.", filename) + finally: + fp.close() + +def convert_sql_import(cr, fp): + cr.execute(fp.read()) + +def convert_csv_import(cr, module, fname, csvcontent, idref=None, mode='init', + noupdate=False): + '''Import csv file : + quote: " + delimiter: , + encoding: utf-8''' + if not idref: + idref={} + model = ('.'.join(fname.split('.')[:-1]).split('-'))[0] + #remove folder path from model + head, model = os.path.split(model) + + input = cStringIO.StringIO(csvcontent) #FIXME + reader = csv.reader(input, quotechar='"', delimiter=',') + fields = reader.next() + + if not (mode == 'init' or 'id' in fields): + _logger.error("Import specification does not contain 'id' and we are in init mode, Cannot continue.") + return + + datas = [] + for line in reader: + if not (line and any(line)): + continue + try: + datas.append(map(ustr, line)) + except Exception: + _logger.error("Cannot import the line: %s", line) + + context = { + 'mode': mode, + 'module': module, + 'noupdate': noupdate, + } + env = openerp.api.Environment(cr, SUPERUSER_ID, context) + result = env[model].load(fields, datas) + if any(msg['type'] == 'error' for msg in result['messages']): + # Report failed import and abort module install + warning_msg = "\n".join(msg['message'] for msg in result['messages']) + raise Exception(_('Module loading %s failed: file %s could not be processed:\n %s') % (module, fname, warning_msg)) + +def convert_xml_import(cr, module, xmlfile, idref=None, mode='init', noupdate=False, report=None): + doc = etree.parse(xmlfile) + relaxng = etree.RelaxNG( + etree.parse(os.path.join(config['root_path'],'import_xml.rng' ))) + try: + relaxng.assert_(doc) + except Exception: + _logger.info('The XML file does not fit the required schema !', exc_info=True) + _logger.info(ustr(relaxng.error_log.last_error)) + raise + + if idref is None: + idref={} + if isinstance(xmlfile, file): + xml_filename = xmlfile.name + else: + xml_filename = xmlfile + obj = xml_import(cr, module, idref, mode, report=report, noupdate=noupdate, xml_filename=xml_filename) + obj.parse(doc.getroot(), mode=mode) + return True diff --git a/odoo/tools/debugger.py b/odoo/tools/debugger.py new file mode 100644 index 00000000000..55ac46e8ddb --- /dev/null +++ b/odoo/tools/debugger.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. +import importlib +import logging +import types + +_logger = logging.getLogger(__name__) +SUPPORTED_DEBUGGER = {'pdb', 'ipdb', 'wdb', 'pudb'} + + +def post_mortem(config, info): + if config['dev_mode'] and isinstance(info[2], types.TracebackType): + debug = next((opt for opt in config['dev_mode'] if opt in SUPPORTED_DEBUGGER), None) + if debug: + try: + # Try to import the xpdb from config (pdb, ipdb, pudb, ...) + importlib.import_module(debug).post_mortem(info[2]) + except ImportError: + _logger.error('Error while importing %s.' % debug) diff --git a/odoo/tools/float_utils.py b/odoo/tools/float_utils.py new file mode 100644 index 00000000000..67777145dba --- /dev/null +++ b/odoo/tools/float_utils.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import math + +def _float_check_precision(precision_digits=None, precision_rounding=None): + assert (precision_digits is not None or precision_rounding is not None) and \ + not (precision_digits and precision_rounding),\ + "exactly one of precision_digits and precision_rounding must be specified" + if precision_digits is not None: + return 10 ** -precision_digits + return precision_rounding + +def float_round(value, precision_digits=None, precision_rounding=None, rounding_method='HALF-UP'): + """Return ``value`` rounded to ``precision_digits`` decimal digits, + minimizing IEEE-754 floating point representation errors, and applying + the tie-breaking rule selected with ``rounding_method``, by default + HALF-UP (away from zero). + Precision must be given by ``precision_digits`` or ``precision_rounding``, + not both! + + :param float value: the value to round + :param int precision_digits: number of fractional digits to round to. + :param float precision_rounding: decimal number representing the minimum + non-zero value at the desired precision (for example, 0.01 for a + 2-digit precision). + :param rounding_method: the rounding method used: 'HALF-UP' or 'UP', the first + one rounding up to the closest number with the rule that number>=0.5 is + rounded up to 1, and the latest one always rounding up. + :return: rounded float + """ + rounding_factor = _float_check_precision(precision_digits=precision_digits, + precision_rounding=precision_rounding) + if rounding_factor == 0 or value == 0: return 0.0 + + # NORMALIZE - ROUND - DENORMALIZE + # In order to easily support rounding to arbitrary 'steps' (e.g. coin values), + # we normalize the value before rounding it as an integer, and de-normalize + # after rounding: e.g. float_round(1.3, precision_rounding=.5) == 1.5 + + # TIE-BREAKING: HALF-UP (for normal rounding) + # We want to apply HALF-UP tie-breaking rules, i.e. 0.5 rounds away from 0. + # Due to IEE754 float/double representation limits, the approximation of the + # real value may be slightly below the tie limit, resulting in an error of + # 1 unit in the last place (ulp) after rounding. + # For example 2.675 == 2.6749999999999998. + # To correct this, we add a very small epsilon value, scaled to the + # the order of magnitude of the value, to tip the tie-break in the right + # direction. + # Credit: discussion with OpenERP community members on bug 882036 + + normalized_value = value / rounding_factor # normalize + epsilon_magnitude = math.log(abs(normalized_value), 2) + epsilon = 2**(epsilon_magnitude-53) + if rounding_method == 'HALF-UP': + normalized_value += cmp(normalized_value,0) * epsilon + rounded_value = round(normalized_value) # round to integer + + # TIE-BREAKING: UP (for ceiling operations) + # When rounding the value up, we instead subtract the epsilon value + # as the the approximation of the real value may be slightly *above* the + # tie limit, this would result in incorrectly rounding up to the next number + # The math.ceil operation is applied on the absolute value in order to + # round "away from zero" and not "towards infinity", then the sign is + # restored. + + elif rounding_method == 'UP': + sign = cmp(normalized_value, 0) + normalized_value -= sign*epsilon + rounded_value = math.ceil(abs(normalized_value))*sign # ceil to integer + + result = rounded_value * rounding_factor # de-normalize + return result + +def float_is_zero(value, precision_digits=None, precision_rounding=None): + """Returns true if ``value`` is small enough to be treated as + zero at the given precision (smaller than the corresponding *epsilon*). + The precision (``10**-precision_digits`` or ``precision_rounding``) + is used as the zero *epsilon*: values less than that are considered + to be zero. + Precision must be given by ``precision_digits`` or ``precision_rounding``, + not both! + + Warning: ``float_is_zero(value1-value2)`` is not equivalent to + ``float_compare(value1,value2) == 0``, as the former will round after + computing the difference, while the latter will round before, giving + different results for e.g. 0.006 and 0.002 at 2 digits precision. + + :param int precision_digits: number of fractional digits to round to. + :param float precision_rounding: decimal number representing the minimum + non-zero value at the desired precision (for example, 0.01 for a + 2-digit precision). + :param float value: value to compare with the precision's zero + :return: True if ``value`` is considered zero + """ + epsilon = _float_check_precision(precision_digits=precision_digits, + precision_rounding=precision_rounding) + return abs(float_round(value, precision_rounding=epsilon)) < epsilon + +def float_compare(value1, value2, precision_digits=None, precision_rounding=None): + """Compare ``value1`` and ``value2`` after rounding them according to the + given precision. A value is considered lower/greater than another value + if their rounded value is different. This is not the same as having a + non-zero difference! + Precision must be given by ``precision_digits`` or ``precision_rounding``, + not both! + + Example: 1.432 and 1.431 are equal at 2 digits precision, + so this method would return 0 + However 0.006 and 0.002 are considered different (this method returns 1) + because they respectively round to 0.01 and 0.0, even though + 0.006-0.002 = 0.004 which would be considered zero at 2 digits precision. + + Warning: ``float_is_zero(value1-value2)`` is not equivalent to + ``float_compare(value1,value2) == 0``, as the former will round after + computing the difference, while the latter will round before, giving + different results for e.g. 0.006 and 0.002 at 2 digits precision. + + :param int precision_digits: number of fractional digits to round to. + :param float precision_rounding: decimal number representing the minimum + non-zero value at the desired precision (for example, 0.01 for a + 2-digit precision). + :param float value1: first value to compare + :param float value2: second value to compare + :return: (resp.) -1, 0 or 1, if ``value1`` is (resp.) lower than, + equal to, or greater than ``value2``, at the given precision. + """ + rounding_factor = _float_check_precision(precision_digits=precision_digits, + precision_rounding=precision_rounding) + value1 = float_round(value1, precision_rounding=rounding_factor) + value2 = float_round(value2, precision_rounding=rounding_factor) + delta = value1 - value2 + if float_is_zero(delta, precision_rounding=rounding_factor): return 0 + return -1 if delta < 0.0 else 1 + +def float_repr(value, precision_digits): + """Returns a string representation of a float with the + the given number of fractional digits. This should not be + used to perform a rounding operation (this is done via + :meth:`~.float_round`), but only to produce a suitable + string representation for a float. + + :param int precision_digits: number of fractional digits to + include in the output + """ + # Can't use str() here because it seems to have an intrisic + # rounding to 12 significant digits, which causes a loss of + # precision. e.g. str(123456789.1234) == str(123456789.123)!! + return ("%%.%sf" % precision_digits) % value + +_float_repr = float_repr + +class float_precision(float): + """ A class for float values that carry precision digits. This is a thin + layer on top of ``float``, and the precision digits are not propagated + to the result of arithmetic operations. This class is used when + converting monetary values to cache, and for serializing them to the + database. + """ + __slots__ = ['precision_digits'] + + def __new__(cls, value, precision_digits): + obj = super(float_precision, cls).__new__(cls, value) + obj.precision_digits = precision_digits + return obj + + def float_repr(self): + return _float_repr(self, self.precision_digits) + + +if __name__ == "__main__": + + import time + start = time.time() + count = 0 + errors = 0 + + def try_round(amount, expected, precision_digits=3): + global count, errors; count += 1 + result = float_repr(float_round(amount, precision_digits=precision_digits), + precision_digits=precision_digits) + if result != expected: + errors += 1 + print '###!!! Rounding error: got %s , expected %s' % (result, expected) + + # Extended float range test, inspired by Cloves Almeida's test on bug #882036. + fractions = [.0, .015, .01499, .675, .67499, .4555, .4555, .45555] + expecteds = ['.00', '.02', '.01', '.68', '.67', '.46', '.456', '.4556'] + precisions = [2, 2, 2, 2, 2, 2, 3, 4] + for magnitude in range(7): + for i in xrange(len(fractions)): + frac, exp, prec = fractions[i], expecteds[i], precisions[i] + for sign in [-1,1]: + for x in xrange(0,10000,97): + n = x * 10**magnitude + f = sign * (n + frac) + f_exp = ('-' if f != 0 and sign == -1 else '') + str(n) + exp + try_round(f, f_exp, precision_digits=prec) + + stop = time.time() + + # Micro-bench results: + # 47130 round calls in 0.422306060791 secs, with Python 2.6.7 on Core i3 x64 + # with decimal: + # 47130 round calls in 6.612248100021 secs, with Python 2.6.7 on Core i3 x64 + print count, " round calls, ", errors, "errors, done in ", (stop-start), 'secs' diff --git a/odoo/tools/func.py b/odoo/tools/func.py new file mode 100644 index 00000000000..8f49864eda2 --- /dev/null +++ b/odoo/tools/func.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +__all__ = ['synchronized', 'lazy_classproperty', 'lazy_property', + 'classproperty', 'conditional'] + +from functools import wraps +from inspect import getsourcefile + + +class lazy_property(object): + """ Decorator for a lazy property of an object, i.e., an object attribute + that is determined by the result of a method call evaluated once. To + reevaluate the property, simply delete the attribute on the object, and + get it again. + """ + def __init__(self, fget): + self.fget = fget + + def __get__(self, obj, cls): + if obj is None: + return self + value = self.fget(obj) + setattr(obj, self.fget.__name__, value) + return value + + @property + def __doc__(self): + return self.fget.__doc__ + + @staticmethod + def reset_all(obj): + """ Reset all lazy properties on the instance `obj`. """ + cls = type(obj) + obj_dict = vars(obj) + for name in obj_dict.keys(): + if isinstance(getattr(cls, name, None), lazy_property): + obj_dict.pop(name) + +class lazy_classproperty(lazy_property): + """ Similar to :class:`lazy_property`, but for classes. """ + def __get__(self, obj, cls): + val = self.fget(cls) + setattr(cls, self.fget.__name__, val) + return val + +def conditional(condition, decorator): + """ Decorator for a conditionally applied decorator. + + Example: + + @conditional(get_config('use_cache'), ormcache) + def fn(): + pass + """ + if condition: + return decorator + else: + return lambda fn: fn + +def synchronized(lock_attr='_lock'): + def decorator(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + lock = getattr(self, lock_attr) + try: + lock.acquire() + return func(self, *args, **kwargs) + finally: + lock.release() + return wrapper + return decorator + +def frame_codeinfo(fframe, back=0): + """ Return a (filename, line) pair for a previous frame . + @return (filename, lineno) where lineno is either int or string=='' + """ + + try: + if not fframe: + return "<unknown>", '' + for i in range(back): + fframe = fframe.f_back + try: + fname = getsourcefile(fframe) + except TypeError: + fname = '<builtin>' + lineno = fframe.f_lineno or '' + return fname, lineno + except Exception: + return "<unknown>", '' + +def compose(a, b): + """ Composes the callables ``a`` and ``b``. ``compose(a, b)(*args)`` is + equivalent to ``a(b(*args))``. + + Can be used as a decorator by partially applying ``a``:: + + @partial(compose, a) + def b(): + ... + """ + @wraps(b) + def wrapper(*args, **kwargs): + return a(b(*args, **kwargs)) + return wrapper + + +class _ClassProperty(property): + def __get__(self, cls, owner): + return self.fget.__get__(None, owner)() + +def classproperty(func): + return _ClassProperty(classmethod(func)) diff --git a/odoo/tools/graph.py b/odoo/tools/graph.py new file mode 100755 index 00000000000..454a1dce98f --- /dev/null +++ b/odoo/tools/graph.py @@ -0,0 +1,754 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import operator +import math + +class graph(object): + def __init__(self, nodes, transitions, no_ancester=None): + """Initialize graph's object + + @param nodes list of ids of nodes in the graph + @param transitions list of edges in the graph in the form (source_node, destination_node) + @param no_ancester list of nodes with no incoming edges + """ + + self.nodes = nodes or [] + self.edges = transitions or [] + self.no_ancester = no_ancester or {} + trans = {} + + for t in transitions: + trans.setdefault(t[0], []) + trans[t[0]].append(t[1]) + self.transitions = trans + self.result = {} + + + def init_rank(self): + """Computes rank of the nodes of the graph by finding initial feasible tree + """ + self.edge_wt = {} + for link in self.links: + self.edge_wt[link] = self.result[link[1]]['x'] - self.result[link[0]]['x'] + + tot_node = len(self.partial_order) + #do until all the nodes in the component are searched + while self.tight_tree()<tot_node: + list_node = [] + list_edge = [] + + for node in self.nodes: + if node not in self.reachable_nodes: + list_node.append(node) + + for edge in self.edge_wt: + if edge not in self.tree_edges: + list_edge.append(edge) + + slack = 100 + + for edge in list_edge: + if ((edge[0] in self.reachable_nodes and edge[1] not in self.reachable_nodes) or + (edge[1] in self.reachable_nodes and edge[0] not in self.reachable_nodes)): + if slack > self.edge_wt[edge]-1: + slack = self.edge_wt[edge]-1 + new_edge = edge + + if new_edge[0] not in self.reachable_nodes: + delta = -(self.edge_wt[new_edge]-1) + else: + delta = self.edge_wt[new_edge]-1 + + for node in self.result: + if node in self.reachable_nodes: + self.result[node]['x'] += delta + + for edge in self.edge_wt: + self.edge_wt[edge] = self.result[edge[1]]['x'] - self.result[edge[0]]['x'] + + self.init_cutvalues() + + + def tight_tree(self): + self.reachable_nodes = [] + self.tree_edges = [] + self.reachable_node(self.start) + return len(self.reachable_nodes) + + + def reachable_node(self, node): + """Find the nodes of the graph which are only 1 rank apart from each other + """ + + if node not in self.reachable_nodes: + self.reachable_nodes.append(node) + for edge in self.edge_wt: + if edge[0]==node: + if self.edge_wt[edge]==1: + self.tree_edges.append(edge) + if edge[1] not in self.reachable_nodes: + self.reachable_nodes.append(edge[1]) + self.reachable_node(edge[1]) + + + def init_cutvalues(self): + """Initailize cut values of edges of the feasible tree. + Edges with negative cut-values are removed from the tree to optimize rank assignment + """ + self.cut_edges = {} + self.head_nodes = [] + i=0 + + for edge in self.tree_edges: + self.head_nodes = [] + rest_edges = [] + rest_edges += self.tree_edges + del rest_edges[i] + self.head_component(self.start, rest_edges) + i+=1 + positive = 0 + negative = 0 + for source_node in self.transitions: + if source_node in self.head_nodes: + for dest_node in self.transitions[source_node]: + if dest_node not in self.head_nodes: + negative+=1 + else: + for dest_node in self.transitions[source_node]: + if dest_node in self.head_nodes: + positive+=1 + + self.cut_edges[edge] = positive - negative + + + def head_component(self, node, rest_edges): + """Find nodes which are reachable from the starting node, after removing an edge + """ + if node not in self.head_nodes: + self.head_nodes.append(node) + + for edge in rest_edges: + if edge[0]==node: + self.head_component(edge[1],rest_edges) + + + def process_ranking(self, node, level=0): + """Computes initial feasible ranking after making graph acyclic with depth-first search + """ + + if node not in self.result: + self.result[node] = {'y': None, 'x':level, 'mark':0} + else: + if level > self.result[node]['x']: + self.result[node]['x'] = level + + if self.result[node]['mark']==0: + self.result[node]['mark'] = 1 + for sec_end in self.transitions.get(node, []): + self.process_ranking(sec_end, level+1) + + + def make_acyclic(self, parent, node, level, tree): + """Computes Partial-order of the nodes with depth-first search + """ + + if node not in self.partial_order: + self.partial_order[node] = {'level':level, 'mark':0} + if parent: + tree.append((parent, node)) + + if self.partial_order[node]['mark']==0: + self.partial_order[node]['mark'] = 1 + for sec_end in self.transitions.get(node, []): + self.links.append((node, sec_end)) + self.make_acyclic(node, sec_end, level+1, tree) + + return tree + + + def rev_edges(self, tree): + """reverse the direction of the edges whose source-node-partail_order> destination-node-partail_order + to make the graph acyclic + """ + Is_Cyclic = False + i=0 + for link in self.links: + src = link[0] + des = link[1] + edge_len = self.partial_order[des]['level'] - self.partial_order[src]['level'] + if edge_len < 0: + del self.links[i] + self.links.insert(i, (des, src)) + self.transitions[src].remove(des) + self.transitions.setdefault(des, []).append(src) + Is_Cyclic = True + elif math.fabs(edge_len) > 1: + Is_Cyclic = True + i += 1 + + return Is_Cyclic + + def exchange(self, e, f): + """Exchange edges to make feasible-tree optimized + :param e: edge with negative cut-value + :param f: new edge with minimum slack-value + """ + del self.tree_edges[self.tree_edges.index(e)] + self.tree_edges.append(f) + self.init_cutvalues() + + + def enter_edge(self, edge): + """Finds a new_edge with minimum slack value to replace an edge with negative cut-value + + @param edge edge with negative cut-value + """ + + self.head_nodes = [] + rest_edges = [] + rest_edges += self.tree_edges + del rest_edges[rest_edges.index(edge)] + self.head_component(self.start, rest_edges) + + if edge[1] in self.head_nodes: + l = [] + for node in self.result: + if node not in self.head_nodes: + l.append(node) + self.head_nodes = l + + slack = 100 + new_edge = edge + for source_node in self.transitions: + if source_node in self.head_nodes: + for dest_node in self.transitions[source_node]: + if dest_node not in self.head_nodes: + if slack>(self.edge_wt[edge]-1): + slack = self.edge_wt[edge]-1 + new_edge = (source_node, dest_node) + + return new_edge + + + def leave_edge(self): + """Returns the edge with negative cut_value(if exists) + """ + if self.critical_edges: + for edge in self.critical_edges: + self.cut_edges[edge] = 0 + + for edge in self.cut_edges: + if self.cut_edges[edge]<0: + return edge + + return None + + + def finalize_rank(self, node, level): + self.result[node]['x'] = level + for destination in self.optimal_edges.get(node, []): + self.finalize_rank(destination, level+1) + + + def normalize(self): + """The ranks are normalized by setting the least rank to zero. + """ + + least_rank = min(map(lambda x: x['x'], self.result.values())) + + if least_rank!=0: + for node in self.result: + self.result[node]['x']-=least_rank + + + def make_chain(self): + """Edges between nodes more than one rank apart are replaced by chains of unit + length edges between temporary nodes. + """ + + for edge in self.edge_wt: + if self.edge_wt[edge]>1: + self.transitions[edge[0]].remove(edge[1]) + start = self.result[edge[0]]['x'] + end = self.result[edge[1]]['x'] + + for rank in range(start+1, end): + if not self.result.get((rank, 'temp'), False): + self.result[(rank, 'temp')] = {'y': None, 'x': rank, 'mark': 0} + + for rank in range(start, end): + if start==rank: + self.transitions[edge[0]].append((rank+1, 'temp')) + elif rank==end-1: + self.transitions.setdefault((rank, 'temp'), []).append(edge[1]) + else: + self.transitions.setdefault((rank, 'temp'), []).append((rank+1, 'temp')) + + + def init_order(self, node, level): + """Initialize orders the nodes in each rank with depth-first search + """ + if not self.result[node]['y']: + self.result[node]['y'] = self.order[level] + self.order[level] += 1 + + for sec_end in self.transitions.get(node, []): + if node!=sec_end: + self.init_order(sec_end, self.result[sec_end]['x']) + + + def order_heuristic(self): + for i in range(12): + self.wmedian() + + + def wmedian(self): + """Applies median heuristic to find optimzed order of the nodes with in their ranks + """ + for level in self.levels: + + node_median = [] + nodes = self.levels[level] + for node in nodes: + node_median.append((node, self.median_value(node, level-1))) + + sort_list = sorted(node_median, key=operator.itemgetter(1)) + + new_list = [tuple[0] for tuple in sort_list] + + self.levels[level] = new_list + order = 0 + for node in nodes: + self.result[node]['y'] = order + order +=1 + + + def median_value(self, node, adj_rank): + """Returns median value of a vertex , defined as the median position of the adjacent vertices + + @param node node to process + @param adj_rank rank 1 less than the node's rank + """ + adj_nodes = self.adj_position(node, adj_rank) + l = len(adj_nodes) + m = l/2 + + if l==0: + return -1.0 + elif l%2 == 1: + return adj_nodes[m]#median of the middle element + elif l==2: + return (adj_nodes[0]+adj_nodes[1])/2 + else: + left = adj_nodes[m-1] - adj_nodes[0] + right = adj_nodes[l-1] - adj_nodes[m] + return ((adj_nodes[m-1]*right) + (adj_nodes[m]*left))/(left+right) + + + def adj_position(self, node, adj_rank): + """Returns list of the present positions of the nodes adjacent to node in the given adjacent rank. + + @param node node to process + @param adj_rank rank 1 less than the node's rank + """ + + pre_level_nodes = self.levels.get(adj_rank, []) + adj_nodes = [] + + if pre_level_nodes: + for src in pre_level_nodes: + if self.transitions.get(src) and node in self.transitions[src]: + adj_nodes.append(self.result[src]['y']) + + return adj_nodes + + + def preprocess_order(self): + levels = {} + + for r in self.partial_order: + l = self.result[r]['x'] + levels.setdefault(l,[]) + levels[l].append(r) + + self.levels = levels + + + def graph_order(self): + """Finds actual-order of the nodes with respect to maximum number of nodes in a rank in component + """ + mid_pos = 0.0 + max_level = max(map(lambda x: len(x), self.levels.values())) + + for level in self.levels: + if level: + no = len(self.levels[level]) + factor = (max_level - no) * 0.10 + list = self.levels[level] + list.reverse() + + if no%2==0: + first_half = list[no/2:] + factor = -factor + else: + first_half = list[no/2+1:] + if max_level==1:#for the case when horizontal graph is there + self.result[list[no/2]]['y'] = mid_pos + (self.result[list[no/2]]['x']%2 * 0.5) + else: + self.result[list[no/2]]['y'] = mid_pos + factor + + last_half = list[:no/2] + + i=1 + for node in first_half: + self.result[node]['y'] = mid_pos - (i + factor) + i += 1 + + i=1 + for node in last_half: + self.result[node]['y'] = mid_pos + (i + factor) + i += 1 + else: + self.max_order += max_level+1 + mid_pos = self.result[self.start]['y'] + + + def tree_order(self, node, last=0): + mid_pos = self.result[node]['y'] + l = self.transitions.get(node, []) + l.reverse() + no = len(l) + + rest = no%2 + first_half = l[no/2+rest:] + last_half = l[:no/2] + + for i, child in enumerate(first_half): + self.result[child]['y'] = mid_pos - (i+1 - (0 if rest else 0.5)) + + if self.transitions.get(child, False): + if last: + self.result[child]['y'] = last + len(self.transitions[child])/2 + 1 + last = self.tree_order(child, last) + + if rest: + mid_node = l[no/2] + self.result[mid_node]['y'] = mid_pos + + if self.transitions.get(mid_node, False): + if last: + self.result[mid_node]['y'] = last + len(self.transitions[mid_node])/2 + 1 + if node!=mid_node: + last = self.tree_order(mid_node) + else: + if last: + self.result[mid_node]['y'] = last + 1 + self.result[node]['y'] = self.result[mid_node]['y'] + mid_pos = self.result[node]['y'] + + i=1 + last_child = None + for child in last_half: + self.result[child]['y'] = mid_pos + (i - (0 if rest else 0.5)) + last_child = child + i += 1 + if self.transitions.get(child, False): + if last: + self.result[child]['y'] = last + len(self.transitions[child])/2 + 1 + if node!=child: + last = self.tree_order(child, last) + + if last_child: + last = self.result[last_child]['y'] + + return last + + + def process_order(self): + """Finds actual-order of the nodes with respect to maximum number of nodes in a rank in component + """ + + if self.Is_Cyclic: + max_level = max(map(lambda x: len(x), self.levels.values())) + + if max_level%2: + self.result[self.start]['y'] = (max_level+1)/2 + self.max_order + (self.max_order and 1) + else: + self.result[self.start]['y'] = max_level /2 + self.max_order + (self.max_order and 1) + + self.graph_order() + + else: + self.result[self.start]['y'] = 0 + self.tree_order(self.start, 0) + min_order = math.fabs(min(map(lambda x: x['y'], self.result.values()))) + + index = self.start_nodes.index(self.start) + same = False + + roots = [] + if index>0: + for start in self.start_nodes[:index]: + same = True + for edge in self.tree_list[start][1:]: + if edge in self.tree_list[self.start]: + continue + else: + same = False + break + if same: + roots.append(start) + + if roots: + min_order += self.max_order + else: + min_order += self.max_order + 1 + + for level in self.levels: + for node in self.levels[level]: + self.result[node]['y'] += min_order + + if roots: + roots.append(self.start) + one_level_el = self.tree_list[self.start][0][1] + base = self.result[one_level_el]['y']# * 2 / (index + 2) + + + no = len(roots) + first_half = roots[:no/2] + + if no%2==0: + last_half = roots[no/2:] + else: + last_half = roots[no/2+1:] + + factor = -math.floor(no/2) + for start in first_half: + self.result[start]['y'] = base + factor + factor += 1 + + if no%2: + self.result[roots[no/2]]['y'] = base + factor + factor +=1 + + for start in last_half: + self.result[start]['y'] = base + factor + factor += 1 + + self.max_order = max(map(lambda x: x['y'], self.result.values())) + + def find_starts(self): + """Finds other start nodes of the graph in the case when graph is disconneted + """ + rem_nodes = [] + for node in self.nodes: + if not self.partial_order.get(node): + rem_nodes.append(node) + cnt = 0 + while True: + if len(rem_nodes)==1: + self.start_nodes.append(rem_nodes[0]) + break + else: + count = 0 + new_start = rem_nodes[0] + largest_tree = [] + + for node in rem_nodes: + self.partial_order = {} + tree = self.make_acyclic(None, node, 0, []) + if len(tree)+1 > count: + count = len(tree) + 1 + new_start = node + largest_tree = tree + else: + if not largest_tree: + new_start = rem_nodes[0] + rem_nodes.remove(new_start) + + self.start_nodes.append(new_start) + + + for edge in largest_tree: + if edge[0] in rem_nodes: + rem_nodes.remove(edge[0]) + if edge[1] in rem_nodes: + rem_nodes.remove(edge[1]) + + if not rem_nodes: + break + + + def rank(self): + """Finds the optimized rank of the nodes using Network-simplex algorithm + """ + self.levels = {} + self.critical_edges = [] + self.partial_order = {} + self.links = [] + self.Is_Cyclic = False + + self.tree_list[self.start] = self.make_acyclic(None, self.start, 0, []) + self.Is_Cyclic = self.rev_edges(self.tree_list[self.start]) + self.process_ranking(self.start) + self.init_rank() + + #make cut values of all tree edges to 0 to optimize feasible tree + e = self.leave_edge() + + while e : + f = self.enter_edge(e) + if e==f: + self.critical_edges.append(e) + else: + self.exchange(e,f) + e = self.leave_edge() + + #finalize rank using optimum feasible tree +# self.optimal_edges = {} +# for edge in self.tree_edges: +# source = self.optimal_edges.setdefault(edge[0], []) +# source.append(edge[1]) + +# self.finalize_rank(self.start, 0) + + #normalization + self.normalize() + for edge in self.edge_wt: + self.edge_wt[edge] = self.result[edge[1]]['x'] - self.result[edge[0]]['x'] + + def order_in_rank(self): + """Finds optimized order of the nodes within their ranks using median heuristic + """ + + self.make_chain() + self.preprocess_order() + self.order = {} + max_rank = max(map(lambda x: x, self.levels.keys())) + + for i in range(max_rank+1): + self.order[i] = 0 + + self.init_order(self.start, self.result[self.start]['x']) + + for level in self.levels: + self.levels[level].sort(lambda x, y: cmp(self.result[x]['y'], self.result[y]['y'])) + + self.order_heuristic() + self.process_order() + + def process(self, starting_node): + """Process the graph to find ranks and order of the nodes + + @param starting_node node from where to start the graph search + """ + + self.start_nodes = starting_node or [] + self.partial_order = {} + self.links = [] + self.tree_list = {} + + if self.nodes: + if self.start_nodes: + #add dummy edges to the nodes which does not have any incoming edges + tree = self.make_acyclic(None, self.start_nodes[0], 0, []) + + for node in self.no_ancester: + for sec_node in self.transitions.get(node, []): + if sec_node in self.partial_order.keys(): + self.transitions[self.start_nodes[0]].append(node) + break + + self.partial_order = {} + tree = self.make_acyclic(None, self.start_nodes[0], 0, []) + + + # if graph is disconnected or no start-node is given + #than to find starting_node for each component of the node + if len(self.nodes) > len(self.partial_order): + self.find_starts() + + self.max_order = 0 + #for each component of the graph find ranks and order of the nodes + for s in self.start_nodes: + self.start = s + self.rank() # First step:Netwoek simplex algorithm + self.order_in_rank() #Second step: ordering nodes within ranks + + + def __str__(self): + result = '' + for l in self.levels: + result += 'PosY: ' + str(l) + '\n' + for node in self.levels[l]: + result += '\tPosX: '+ str(self.result[node]['y']) + ' - Node:' + str(node) + "\n" + return result + + + def scale(self, maxx, maxy, nwidth=0, nheight=0, margin=20): + """Computes actual co-ordiantes of the nodes + """ + + #for flat edges ie. source an destination nodes are on the same rank + for src in self.transitions: + for des in self.transitions[src]: + if self.result[des]['x'] - self.result[src]['x'] == 0: + self.result[src]['x'] += 0.08 + self.result[des]['x'] -= 0.08 + + factorX = maxx + nheight + factorY = maxy + nwidth + + for node in self.result: + self.result[node]['y'] = (self.result[node]['y']) * factorX + margin + self.result[node]['x'] = (self.result[node]['x']) * factorY + margin + + + def result_get(self): + return self.result + +if __name__=='__main__': + starting_node = ['profile'] # put here nodes with flow_start=True + nodes = ['project','account','hr','base','product','mrp','test','profile'] + transitions = [ + ('profile','mrp'), + ('mrp','project'), + ('project','product'), + ('mrp','hr'), + ('mrp','test'), + ('project','account'), + ('project','hr'), + ('product','base'), + ('account','product'), + ('account','test'), + ('account','base'), + ('hr','base'), + ('test','base') + ] + + radius = 20 + g = graph(nodes, transitions) + g.process(starting_node) + g.scale(radius*3,radius*3, radius, radius) + + from PIL import Image + from PIL import ImageDraw + img = Image.new("RGB", (800, 600), "#ffffff") + draw = ImageDraw.Draw(img) + + result = g.result_get() + node_res = {} + for node in nodes: + node_res[node] = result[node] + + for name,node in node_res.items(): + + draw.arc( (int(node['y']-radius), int(node['x']-radius),int(node['y']+radius), int(node['x']+radius) ), 0, 360, (128,128,128)) + draw.text( (int(node['y']), int(node['x'])), str(name), (128,128,128)) + + + for t in transitions: + draw.line( (int(node_res[t[0]]['y']), int(node_res[t[0]]['x']),int(node_res[t[1]]['y']),int(node_res[t[1]]['x'])),(128,128,128) ) + img.save("graph.png", "PNG") diff --git a/odoo/tools/image.py b/odoo/tools/image.py new file mode 100644 index 00000000000..f17623cbe6e --- /dev/null +++ b/odoo/tools/image.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +from PIL import Image +from PIL import ImageEnhance +from random import randrange + +# ---------------------------------------- +# Image resizing +# ---------------------------------------- + +def image_resize_image(base64_source, size=(1024, 1024), encoding='base64', filetype=None, avoid_if_small=False): + """ Function to resize an image. The image will be resized to the given + size, while keeping the aspect ratios, and holes in the image will be + filled with transparent background. The image will not be stretched if + smaller than the expected size. + Steps of the resizing: + - Compute width and height if not specified. + - if avoid_if_small: if both image sizes are smaller than the requested + sizes, the original image is returned. This is used to avoid adding + transparent content around images that we do not want to alter but + just resize if too big. This is used for example when storing images + in the 'image' field: we keep the original image, resized to a maximal + size, without adding transparent content around it if smaller. + - create a thumbnail of the source image through using the thumbnail + function. Aspect ratios are preserved when using it. Note that if the + source image is smaller than the expected size, it will not be + extended, but filled to match the size. + - create a transparent background that will hold the final image. + - paste the thumbnail on the transparent background and center it. + + :param base64_source: base64-encoded version of the source + image; if False, returns False + :param size: 2-tuple(width, height). A None value for any of width or + height mean an automatically computed value based respectivelly + on height or width of the source image. + :param encoding: the output encoding + :param filetype: the output filetype, by default the source image's + :type filetype: str, any PIL image format (supported for creation) + :param avoid_if_small: do not resize if image height and width + are smaller than the expected size. + """ + if not base64_source: + return False + if size == (None, None): + return base64_source + image_stream = StringIO.StringIO(base64_source.decode(encoding)) + image = Image.open(image_stream) + # store filetype here, as Image.new below will lose image.format + filetype = (filetype or image.format).upper() + + filetype = { + 'BMP': 'PNG', + }.get(filetype, filetype) + + asked_width, asked_height = size + if asked_width is None: + asked_width = int(image.size[0] * (float(asked_height) / image.size[1])) + if asked_height is None: + asked_height = int(image.size[1] * (float(asked_width) / image.size[0])) + size = asked_width, asked_height + + # check image size: do not create a thumbnail if avoiding smaller images + if avoid_if_small and image.size[0] <= size[0] and image.size[1] <= size[1]: + return base64_source + + if image.size != size: + image = image_resize_and_sharpen(image, size) + if image.mode not in ["1", "L", "P", "RGB", "RGBA"]: + image = image.convert("RGB") + + background_stream = StringIO.StringIO() + image.save(background_stream, filetype) + return background_stream.getvalue().encode(encoding) + +def image_resize_and_sharpen(image, size, preserve_aspect_ratio=False, factor=2.0): + """ + Create a thumbnail by resizing while keeping ratio. + A sharpen filter is applied for a better looking result. + + :param image: PIL.Image.Image() + :param size: 2-tuple(width, height) + :param preserve_aspect_ratio: boolean (default: False) + :param factor: Sharpen factor (default: 2.0) + """ + if image.mode != 'RGBA': + image = image.convert('RGBA') + image.thumbnail(size, Image.ANTIALIAS) + if preserve_aspect_ratio: + size = image.size + sharpener = ImageEnhance.Sharpness(image) + resized_image = sharpener.enhance(factor) + # create a transparent image for background and paste the image on it + image = Image.new('RGBA', size, (255, 255, 255, 0)) + image.paste(resized_image, ((size[0] - resized_image.size[0]) / 2, (size[1] - resized_image.size[1]) / 2)) + return image + +def image_save_for_web(image, fp=None, format=None): + """ + Save image optimized for web usage. + + :param image: PIL.Image.Image() + :param fp: File name or file object. If not specified, a bytestring is returned. + :param format: File format if could not be deduced from image. + """ + opt = dict(format=image.format or format) + if image.format == 'PNG': + opt.update(optimize=True) + alpha = False + if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info): + alpha = image.convert('RGBA').split()[-1] + if image.mode != 'P': + # Floyd Steinberg dithering by default + image = image.convert('RGBA').convert('P', palette=Image.WEB, colors=256) + if alpha: + image.putalpha(alpha) + elif image.format == 'JPEG': + opt.update(optimize=True, quality=80) + if fp: + image.save(fp, **opt) + else: + img = StringIO.StringIO() + image.save(img, **opt) + return img.getvalue() + +def image_resize_image_big(base64_source, size=(1024, 1024), encoding='base64', filetype=None, avoid_if_small=True): + """ Wrapper on image_resize_image, to resize images larger than the standard + 'big' image size: 1024x1024px. + :param size, encoding, filetype, avoid_if_small: refer to image_resize_image + """ + return image_resize_image(base64_source, size, encoding, filetype, avoid_if_small) + +def image_resize_image_medium(base64_source, size=(128, 128), encoding='base64', filetype=None, avoid_if_small=False): + """ Wrapper on image_resize_image, to resize to the standard 'medium' + image size: 180x180. + :param size, encoding, filetype, avoid_if_small: refer to image_resize_image + """ + return image_resize_image(base64_source, size, encoding, filetype, avoid_if_small) + +def image_resize_image_small(base64_source, size=(64, 64), encoding='base64', filetype=None, avoid_if_small=False): + """ Wrapper on image_resize_image, to resize to the standard 'small' image + size: 50x50. + :param size, encoding, filetype, avoid_if_small: refer to image_resize_image + """ + return image_resize_image(base64_source, size, encoding, filetype, avoid_if_small) + +# ---------------------------------------- +# Crop Image +# ---------------------------------------- +def crop_image(data, type='top', ratio=False, thumbnail_ratio=None, image_format="PNG"): + """ Used for cropping image and create thumbnail + :param data: base64 data of image. + :param type: Used for cropping position possible + Possible Values : 'top', 'center', 'bottom' + :param ratio: Cropping ratio + e.g for (4,3), (16,9), (16,10) etc + send ratio(1,1) to generate square image + :param thumbnail_ratio: It is size reduce ratio for thumbnail + e.g. thumbnail_ratio=2 will reduce your 500x500 image converted in to 250x250 + :param image_format: return image format PNG,JPEG etc + """ + if not data: + return False + image_stream = Image.open(StringIO.StringIO(data.decode('base64'))) + output_stream = StringIO.StringIO() + w, h = image_stream.size + new_h = h + new_w = w + + if ratio: + w_ratio, h_ratio = ratio + new_h = (w * h_ratio) / w_ratio + new_w = w + if new_h > h: + new_h = h + new_w = (h * w_ratio) / h_ratio + + if type == "top": + cropped_image = image_stream.crop((0, 0, new_w, new_h)) + cropped_image.save(output_stream, format=image_format) + elif type == "center": + cropped_image = image_stream.crop(((w - new_w) / 2, (h - new_h) / 2, (w + new_w) / 2, (h + new_h) / 2)) + cropped_image.save(output_stream, format=image_format) + elif type == "bottom": + cropped_image = image_stream.crop((0, h - new_h, new_w, h)) + cropped_image.save(output_stream, format=image_format) + else: + raise ValueError('ERROR: invalid value for crop_type') + # TDE FIXME: should not have a ratio, makes no sense -> should have maximum width (std: 64; 256 px) + if thumbnail_ratio: + thumb_image = Image.open(StringIO.StringIO(output_stream.getvalue())) + thumb_image.thumbnail((new_w / thumbnail_ratio, new_h / thumbnail_ratio), Image.ANTIALIAS) + thumb_image.save(output_stream, image_format) + return output_stream.getvalue().encode('base64') + +# ---------------------------------------- +# Colors +# --------------------------------------- + +def image_colorize(original, randomize=True, color=(255, 255, 255)): + """ Add a color to the transparent background of an image. + :param original: file object on the original image file + :param randomize: randomize the background color + :param color: background-color, if not randomize + """ + # create a new image, based on the original one + original = Image.open(StringIO.StringIO(original)) + image = Image.new('RGB', original.size) + # generate the background color, past it as background + if randomize: + color = (randrange(32, 224, 24), randrange(32, 224, 24), randrange(32, 224, 24)) + image.paste(color) + image.paste(original, mask=original) + # return the new image + buffer = StringIO.StringIO() + image.save(buffer, 'PNG') + return buffer.getvalue() + +# ---------------------------------------- +# Misc image tools +# --------------------------------------- + +def image_get_resized_images(base64_source, return_big=False, return_medium=True, return_small=True, + big_name='image', medium_name='image_medium', small_name='image_small', + avoid_resize_big=True, avoid_resize_medium=False, avoid_resize_small=False): + """ Standard tool function that returns a dictionary containing the + big, medium and small versions of the source image. This function + is meant to be used for the methods of functional fields for + models using images. + + Default parameters are given to be used for the getter of functional + image fields, for example with res.users or res.partner. It returns + only image_medium and image_small values, to update those fields. + + :param base64_source: base64-encoded version of the source + image; if False, all returnes values will be False + :param return_{..}: if set, computes and return the related resizing + of the image + :param {..}_name: key of the resized image in the return dictionary; + 'image', 'image_medium' and 'image_small' by default. + :param avoid_resize_[..]: see avoid_if_small parameter + :return return_dict: dictionary with resized images, depending on + previous parameters. + """ + return_dict = dict() + if return_big: + return_dict[big_name] = image_resize_image_big(base64_source, avoid_if_small=avoid_resize_big) + if return_medium: + return_dict[medium_name] = image_resize_image_medium(base64_source, avoid_if_small=avoid_resize_medium) + if return_small: + return_dict[small_name] = image_resize_image_small(base64_source, avoid_if_small=avoid_resize_small) + return return_dict + +def image_resize_images(vals, big_name='image', medium_name='image_medium', small_name='image_small'): + """ Update ``vals`` with image fields resized as expected. """ + if big_name in vals: + vals.update(image_get_resized_images(vals[big_name], + return_big=True, return_medium=True, return_small=True, + big_name=big_name, medium_name=medium_name, small_name=small_name, + avoid_resize_big=True, avoid_resize_medium=False, avoid_resize_small=False)) + elif medium_name in vals: + vals.update(image_get_resized_images(vals[medium_name], + return_big=True, return_medium=True, return_small=True, + big_name=big_name, medium_name=medium_name, small_name=small_name, + avoid_resize_big=True, avoid_resize_medium=True, avoid_resize_small=False)) + elif small_name in vals: + vals.update(image_get_resized_images(vals[small_name], + return_big=True, return_medium=True, return_small=True, + big_name=big_name, medium_name=medium_name, small_name=small_name, + avoid_resize_big=True, avoid_resize_medium=True, avoid_resize_small=True)) + + +if __name__=="__main__": + import sys + + assert len(sys.argv)==3, 'Usage to Test: image.py SRC.png DEST.png' + + img = file(sys.argv[1],'rb').read().encode('base64') + new = image_resize_image(img, (128,100)) + file(sys.argv[2], 'wb').write(new.decode('base64')) diff --git a/odoo/tools/import_email.py b/odoo/tools/import_email.py new file mode 100644 index 00000000000..0e7926aeb0a --- /dev/null +++ b/odoo/tools/import_email.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import os, sys +import re +import smtplib +import email, mimetypes +from email.header import decode_header +from email.mime.text import MIMEText +import xmlrpclib + +warn_msg = """ +Bonjour, + +Le message avec le sujet "%s" n'a pu être archivé dans l'ERP. + +""".decode('utf-8') + +class EmailParser(object): + + def __init__(self, headers, dispatcher): + self.headers = headers + self.dispatcher = dispatcher + + def parse(self, msg): + dispatcher((self.headers, msg)) + +class CommandDispatcher(object): + + def __init__(self, receiver): + self.receiver = receiver + + def __call__(self, request): + return self.receiver(request) + +class RPCProxy(object): + + def __init__(self, uid, passwd, host='localhost', port=8069, path='object'): + self.rpc = xmlrpclib.ServerProxy('http://%s:%s/%s' % (host, port, path)) + self.user_id = uid + self.passwd = passwd + + def __call__(self, request): + return self.rpc.execute(self.user_id, self.passwd, *request) + +class ReceiverEmail2Event(object): + + email_re = re.compile(r""" + ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part + @ # mandatory @ sign + [a-zA-Z0-9][\w\.-]* # domain must start with a letter + \. + [a-z]{2,3} # TLD + ) + """, re.VERBOSE) + + project_re = re.compile(r"^ *\[?(\d{4}\.?\d{0,3})\]?", re.UNICODE) + + + def __init__(self, rpc): + self.rpc = rpc + + def get_addresses(self, headers, msg): + hcontent = '' + for header in [h for h in headers if msg.has_key(h)]: + hcontent += msg[header] + return self.email_re.findall(hcontent) + + def get_partners(self, headers, msg): + alladdresses = self.get_addresses(headers, msg) + address_ids = self.rpc(('res.partner', 'search', [('email', 'in', alladdresses)])) + addresses = self.rpc(('res.partner', 'read', address_ids)) + return [x['partner_id'][0] for x in addresses] + + def __call__(self, request): + headers, msg = request + partners = self.get_partners(headers, msg) + subject = u'' + for string, charset in decode_header(msg['Subject']): + if charset: + subject += string.decode(charset) + else: + subject += unicode(string) + if partners: + self.save_mail(msg, subject, partners) + else: + warning = MIMEText((warn_msg % (subject,)).encode('utf-8'), 'plain', 'utf-8') + warning['Subject'] = 'Message de OpenERP' + warning['From'] = 'erp@steel-sa.com' + warning['To'] = msg['From'] + s = smtplib.SMTP() + s.connect() + s.sendmail('erp@steel-sa.com', self.email_re.findall(msg['From']), warning.as_string()) + s.close() + + if msg.is_multipart(): + for message in [m for m in msg.get_payload() if m.get_content_type() == 'message/rfc822']: + self((headers, message.get_payload()[0])) + + def save_mail(self, msg, subject, partners): + counter, description = 1, u'' + if msg.is_multipart(): + for part in msg.get_payload(): + stockdir = os.path.join('emails', msg['Message-Id'][1:-1]) + newdir = os.path.join('/tmp', stockdir) + filename = part.get_filename() + if not filename: + ext = mimetypes.guess_extension(part.get_type()) + if not ext: + ext = '.bin' + filename = 'part-%03d%s' % (counter, ext) + + if part.get_content_maintype() == 'multipart': + continue + elif part.get_content_maintype() == 'text': + if part.get_content_subtype() == 'plain': + description += part.get_payload(decode=1).decode(part.get_charsets()[0]) + description += u'\n\nVous trouverez les éventuels fichiers dans le répertoire: %s' % stockdir + continue + else: + description += u'\n\nCe message est en "%s", vous trouverez ce texte dans le répertoire: %s' % (part.get_content_type(), stockdir) + elif part.get_content_type() == 'message/rfc822': + continue + if not os.path.isdir(newdir): + os.mkdir(newdir) + + counter += 1 + fd = file(os.path.join(newdir, filename), 'w') + fd.write(part.get_payload(decode=1)) + fd.close() + else: + description = msg.get_payload(decode=1).decode(msg.get_charsets()[0]) + + project = self.project_re.search(subject) + if project: + project = project.groups()[0] + else: + project = '' + + for partner in partners: + self.rpc(('res.partner.event', 'create', {'name' : subject, 'partner_id' : partner, 'description' : description, 'project' : project})) + + +if __name__ == '__main__': + rpc_dispatcher = CommandDispatcher(RPCProxy(4, 'admin')) + dispatcher = CommandDispatcher(ReceiverEmail2Event(rpc_dispatcher)) + parser = EmailParser(['To', 'Cc', 'From'], dispatcher) + parser.parse(email.message_from_file(sys.stdin)) diff --git a/odoo/tools/lru.py b/odoo/tools/lru.py new file mode 100644 index 00000000000..b0ea21bba44 --- /dev/null +++ b/odoo/tools/lru.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# taken from http://code.activestate.com/recipes/252524-length-limited-o1-lru-cache-implementation/ +import threading +from func import synchronized + +__all__ = ['LRU'] + +class LRUNode(object): + __slots__ = ['prev', 'next', 'me'] + def __init__(self, prev, me): + self.prev = prev + self.me = me + self.next = None + +class LRU(object): + """ + Implementation of a length-limited O(1) LRU queue. + Built for and used by PyPE: + http://pype.sourceforge.net + Copyright 2003 Josiah Carlson. + """ + def __init__(self, count, pairs=[]): + self._lock = threading.RLock() + self.count = max(count, 1) + self.d = {} + self.first = None + self.last = None + for key, value in pairs: + self[key] = value + + @synchronized() + def __contains__(self, obj): + return obj in self.d + + def get(self, obj, val=None): + try: + return self[obj] + except KeyError: + return val + + @synchronized() + def __getitem__(self, obj): + a = self.d[obj].me + self[a[0]] = a[1] + return a[1] + + @synchronized() + def __setitem__(self, obj, val): + if obj in self.d: + del self[obj] + nobj = LRUNode(self.last, (obj, val)) + if self.first is None: + self.first = nobj + if self.last: + self.last.next = nobj + self.last = nobj + self.d[obj] = nobj + if len(self.d) > self.count: + if self.first == self.last: + self.first = None + self.last = None + return + a = self.first + a.next.prev = None + self.first = a.next + a.next = None + del self.d[a.me[0]] + del a + + @synchronized() + def __delitem__(self, obj): + nobj = self.d[obj] + if nobj.prev: + nobj.prev.next = nobj.next + else: + self.first = nobj.next + if nobj.next: + nobj.next.prev = nobj.prev + else: + self.last = nobj.prev + del self.d[obj] + + @synchronized() + def __iter__(self): + cur = self.first + while cur is not None: + cur2 = cur.next + yield cur.me[1] + cur = cur2 + + @synchronized() + def __len__(self): + return len(self.d) + + @synchronized() + def iteritems(self): + cur = self.first + while cur is not None: + cur2 = cur.next + yield cur.me + cur = cur2 + + @synchronized() + def iterkeys(self): + return iter(self.d) + + @synchronized() + def itervalues(self): + for i,j in self.iteritems(): + yield j + + @synchronized() + def keys(self): + return self.d.keys() + + @synchronized() + def pop(self,key): + v=self[key] + del self[key] + return v + + @synchronized() + def clear(self): + self.d = {} + self.first = None + self.last = None diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py new file mode 100644 index 00000000000..da4ec6fb716 --- /dev/null +++ b/odoo/tools/mail.py @@ -0,0 +1,523 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import cgi +import logging +import lxml.html.clean as clean +import random +import re +import socket +import threading +import time + +from email.header import decode_header +from email.utils import getaddresses, formataddr +from lxml import etree + +import openerp +from openerp.loglevels import ustr + +_logger = logging.getLogger(__name__) + + +#---------------------------------------------------------- +# HTML Sanitizer +#---------------------------------------------------------- + +tags_to_kill = ["script", "head", "meta", "title", "link", "style", "frame", "iframe", "base", "object", "embed"] +tags_to_remove = ['html', 'body'] + +# allow new semantic HTML5 tags +allowed_tags = clean.defs.tags | frozenset('article section header footer hgroup nav aside figure main'.split() + [etree.Comment]) +safe_attrs = clean.defs.safe_attrs | frozenset( + ['style', + 'data-o-mail-quote', # quote detection + 'data-oe-model', 'data-oe-id', 'data-oe-field', 'data-oe-type', 'data-oe-expression', 'data-oe-translation-id', 'data-oe-nodeid', + 'data-publish', 'data-id', 'data-res_id', 'data-member_id', 'data-view-id' + ]) + + +class _Cleaner(clean.Cleaner): + + _style_re = re.compile('''([\w-]+)\s*:\s*((?:[^;"']|"[^"]*"|'[^']*')+)''') + + _style_whitelist = [ + 'font-size', 'font-family', 'background-color', 'color', 'text-align', + 'padding', 'padding-top', 'padding-left', 'padding-bottom', 'padding-right', + 'margin', 'margin-top', 'margin-left', 'margin-bottom', 'margin-right' + # box model + 'border', 'border-color', 'border-radius', 'height', 'margin', 'padding', 'width', 'max-width', 'min-width', + # tables + 'border-collapse', 'border-spacing', 'caption-side', 'empty-cells', 'table-layout'] + + strip_classes = False + sanitize_style = False + + def __call__(self, doc): + # perform quote detection before cleaning and class removal + for el in doc.iter(): + self.tag_quote(el) + + super(_Cleaner, self).__call__(doc) + + # if we keep attributes but still remove classes + if not getattr(self, 'safe_attrs_only', False) and self.strip_classes: + for el in doc.iter(): + self.strip_class(el) + + # if we keep style attribute, sanitize them + if not self.style and self.sanitize_style: + for el in doc.iter(): + self.parse_style(el) + + def tag_quote(self, el): + def _create_new_node(tag, text, tail=None, attrs=None): + new_node = etree.Element(tag) + new_node.text = text + new_node.tail = tail + if attrs: + for key, val in attrs.iteritems(): + new_node.set(key, val) + return new_node + + def _tag_matching_regex_in_text(regex, node, tag='span', attrs=None): + text = node.text or '' + if not re.search(regex, text): + return + + child_node = None + idx, node_idx = 0, 0 + for item in re.finditer(regex, text): + new_node = _create_new_node(tag, text[item.start():item.end()], None, attrs) + if child_node is None: + node.text = text[idx:item.start()] + new_node.tail = text[item.end():] + node.insert(node_idx, new_node) + else: + child_node.tail = text[idx:item.start()] + new_node.tail = text[item.end():] + node.insert(node_idx, new_node) + child_node = new_node + idx = item.end() + node_idx = node_idx + 1 + + el_class = el.get('class', '') or '' + el_id = el.get('id', '') or '' + + # gmail or yahoo // # outlook, html // # msoffice + if ('gmail_extra' in el_class or 'yahoo_quoted' in el_class) or \ + (el.tag == 'hr' and ('stopSpelling' in el_class or 'stopSpelling' in el_id)) or \ + ('SkyDrivePlaceholder' in el_class or 'SkyDrivePlaceholder' in el_class): + el.set('data-o-mail-quote', '1') + if el.getparent() is not None: + el.getparent().set('data-o-mail-quote-container', '1') + + # html signature (-- <br />blah) + signature_begin = re.compile(r"((?:(?:^|\n)[-]{2}[\s]?$))") + if el.text and el.find('br') is not None and re.search(signature_begin, el.text): + el.set('data-o-mail-quote', '1') + if el.getparent() is not None: + el.getparent().set('data-o-mail-quote-container', '1') + + # text-based quotes (>, >>) and signatures (-- Signature) + text_complete_regex = re.compile(r"((?:\n[>]+[^\n\r]*)+|(?:(?:^|\n)[-]{2}[\s]?[\r\n]{1,2}[\s\S]+))") + if not el.get('data-o-mail-quote'): + _tag_matching_regex_in_text(text_complete_regex, el, 'span', {'data-o-mail-quote': '1'}) + + if el.tag == 'blockquote': + # remove single node + el.set('data-o-mail-quote-node', '1') + el.set('data-o-mail-quote', '1') + if el.getparent() is not None and (el.getparent().get('data-o-mail-quote') or el.getparent().get('data-o-mail-quote-container')) and not el.getparent().get('data-o-mail-quote-node'): + el.set('data-o-mail-quote', '1') + + def strip_class(self, el): + if el.attrib.get('class'): + del el.attrib['class'] + + def parse_style(self, el): + attributes = el.attrib + styling = attributes.get('style') + if styling: + valid_styles = {} + styles = self._style_re.findall(styling) + for style in styles: + if style[0].lower() in self._style_whitelist: + valid_styles[style[0].lower()] = style[1] + if valid_styles: + el.attrib['style'] = '; '.join('%s: %s' % (key, val) for (key, val) in valid_styles.iteritems()) + else: + del el.attrib['style'] + + def allow_element(self, el): + if el.tag == 'object' and el.get('type') == "image/svg+xml": + return True + return super(_Cleaner, self).allow_element(el) + + +def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=False, sanitize_style=False, strip_style=False, strip_classes=False): + if not src: + return src + src = ustr(src, errors='replace') + # html: remove encoding attribute inside tags + doctype = re.compile(r'(<[^>]*\s)(encoding=(["\'][^"\']*?["\']|[^\s\n\r>]+)(\s[^>]*|/)?>)', re.IGNORECASE | re.DOTALL) + src = doctype.sub(r"", src) + + logger = logging.getLogger(__name__ + '.html_sanitize') + + # html encode email tags + part = re.compile(r"(<(([^a<>]|a[^<>\s])[^<>]*)@[^<>]+>)", re.IGNORECASE | re.DOTALL) + # remove results containing cite="mid:email_like@address" (ex: blockquote cite) + # cite_except = re.compile(r"^((?!cite[\s]*=['\"]).)*$", re.IGNORECASE) + src = part.sub(lambda m: ('cite=' not in m.group(1) and 'alt=' not in m.group(1)) and cgi.escape(m.group(1)) or m.group(1), src) + # html encode mako tags <% ... %> to decode them later and keep them alive, otherwise they are stripped by the cleaner + src = src.replace('<%', cgi.escape('<%')) + src = src.replace('%>', cgi.escape('%>')) + + kwargs = { + 'page_structure': True, + 'style': strip_style, # True = remove style tags/attrs + 'sanitize_style': sanitize_style, # True = sanitize styling + 'forms': True, # True = remove form tags + 'remove_unknown_tags': False, + 'comments': False, + 'processing_instructions': False + } + if sanitize_tags: + kwargs['allow_tags'] = allowed_tags + if etree.LXML_VERSION >= (2, 3, 1): + # kill_tags attribute has been added in version 2.3.1 + kwargs.update({ + 'kill_tags': tags_to_kill, + 'remove_tags': tags_to_remove, + }) + else: + kwargs['remove_tags'] = tags_to_kill + tags_to_remove + + if sanitize_attributes and etree.LXML_VERSION >= (3, 1, 0): # lxml < 3.1.0 does not allow to specify safe_attrs. We keep all attributes in order to keep "style" + if strip_classes: + current_safe_attrs = safe_attrs - frozenset(['class']) + else: + current_safe_attrs = safe_attrs + kwargs.update({ + 'safe_attrs_only': True, + 'safe_attrs': current_safe_attrs, + }) + else: + kwargs.update({ + 'safe_attrs_only': False, # keep oe-data attributes + style + 'strip_classes': strip_classes, # remove classes, even when keeping other attributes + }) + + try: + # some corner cases make the parser crash (such as <SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT> in test_mail) + cleaner = _Cleaner(**kwargs) + cleaned = cleaner.clean_html(src) + # MAKO compatibility: $, { and } inside quotes are escaped, preventing correct mako execution + cleaned = cleaned.replace('%24', '$') + cleaned = cleaned.replace('%7B', '{') + cleaned = cleaned.replace('%7D', '}') + cleaned = cleaned.replace('%20', ' ') + cleaned = cleaned.replace('%5B', '[') + cleaned = cleaned.replace('%5D', ']') + cleaned = cleaned.replace('%7C', '|') + cleaned = cleaned.replace('<%', '<%') + cleaned = cleaned.replace('%>', '%>') + except etree.ParserError, e: + if 'empty' in str(e): + return "" + if not silent: + raise + logger.warning('ParserError obtained when sanitizing %r', src, exc_info=True) + cleaned = '<p>ParserError when sanitizing</p>' + except Exception: + if not silent: + raise + logger.warning('unknown error obtained when sanitizing %r', src, exc_info=True) + cleaned = '<p>Unknown error when sanitizing</p>' + + # this is ugly, but lxml/etree tostring want to put everything in a 'div' that breaks the editor -> remove that + if cleaned.startswith('<div>') and cleaned.endswith('</div>'): + cleaned = cleaned[5:-6] + + return cleaned + +#---------------------------------------------------------- +# HTML/Text management +#---------------------------------------------------------- + +def html_keep_url(text): + """ Transform the url into clickable link with <a/> tag """ + idx = 0 + final = '' + link_tags = re.compile(r"""(?<!["'])((ftp|http|https):\/\/(\w+:{0,1}\w*@)?([^\s<"']+)(:[0-9]+)?(\/|\/([^\s<"']))?)(?![^\s<"']*["']|[^\s<"']*</a>)""") + for item in re.finditer(link_tags, text): + final += text[idx:item.start()] + final += '<a href="%s" target="_blank">%s</a>' % (item.group(0), item.group(0)) + idx = item.end() + final += text[idx:] + return final + +def html2plaintext(html, body_id=None, encoding='utf-8'): + """ From an HTML text, convert the HTML to plain text. + If @param body_id is provided then this is the tag where the + body (not necessarily <body>) starts. + """ + ## (c) Fry-IT, www.fry-it.com, 2007 + ## <peter@fry-it.com> + ## download here: http://www.peterbe.com/plog/html2plaintext + + html = ustr(html) + + if not html: + return '' + + tree = etree.fromstring(html, parser=etree.HTMLParser()) + + if body_id is not None: + source = tree.xpath('//*[@id=%s]' % (body_id,)) + else: + source = tree.xpath('//body') + if len(source): + tree = source[0] + + url_index = [] + i = 0 + for link in tree.findall('.//a'): + url = link.get('href') + if url: + i += 1 + link.tag = 'span' + link.text = '%s [%s]' % (link.text, i) + url_index.append(url) + + html = ustr(etree.tostring(tree, encoding=encoding)) + # \r char is converted into , must remove it + html = html.replace(' ', '') + + html = html.replace('<strong>', '*').replace('</strong>', '*') + html = html.replace('<b>', '*').replace('</b>', '*') + html = html.replace('<h3>', '*').replace('</h3>', '*') + html = html.replace('<h2>', '**').replace('</h2>', '**') + html = html.replace('<h1>', '**').replace('</h1>', '**') + html = html.replace('<em>', '/').replace('</em>', '/') + html = html.replace('<tr>', '\n') + html = html.replace('</p>', '\n') + html = re.sub('<br\s*/?>', '\n', html) + html = re.sub('<.*?>', ' ', html) + html = html.replace(' ' * 2, ' ') + html = html.replace('>', '>') + html = html.replace('<', '<') + html = html.replace('&', '&') + + # strip all lines + html = '\n'.join([x.strip() for x in html.splitlines()]) + html = html.replace('\n' * 2, '\n') + + for i, url in enumerate(url_index): + if i == 0: + html += '\n\n' + html += ustr('[%s] %s\n') % (i + 1, url) + + return html + +def plaintext2html(text, container_tag=False): + """ Convert plaintext into html. Content of the text is escaped to manage + html entities, using cgi.escape(). + - all \n,\r are replaced by <br /> + - enclose content into <p> + - convert url into clickable link + - 2 or more consecutive <br /> are considered as paragraph breaks + + :param string container_tag: container of the html; by default the + content is embedded into a <div> + """ + text = cgi.escape(ustr(text)) + + # 1. replace \n and \r + text = text.replace('\n', '<br/>') + text = text.replace('\r', '<br/>') + + # 2. clickable links + text = html_keep_url(text) + + # 3-4: form paragraphs + idx = 0 + final = '<p>' + br_tags = re.compile(r'(([<]\s*[bB][rR]\s*\/?[>]\s*){2,})') + for item in re.finditer(br_tags, text): + final += text[idx:item.start()] + '</p><p>' + idx = item.end() + final += text[idx:] + '</p>' + + # 5. container + if container_tag: + final = '<%s>%s</%s>' % (container_tag, final, container_tag) + return ustr(final) + +def append_content_to_html(html, content, plaintext=True, preserve=False, container_tag=False): + """ Append extra content at the end of an HTML snippet, trying + to locate the end of the HTML document (</body>, </html>, or + EOF), and converting the provided content in html unless ``plaintext`` + is False. + Content conversion can be done in two ways: + - wrapping it into a pre (preserve=True) + - use plaintext2html (preserve=False, using container_tag to wrap the + whole content) + A side-effect of this method is to coerce all HTML tags to + lowercase in ``html``, and strip enclosing <html> or <body> tags in + content if ``plaintext`` is False. + + :param str html: html tagsoup (doesn't have to be XHTML) + :param str content: extra content to append + :param bool plaintext: whether content is plaintext and should + be wrapped in a <pre/> tag. + :param bool preserve: if content is plaintext, wrap it into a <pre> + instead of converting it into html + """ + html = ustr(html) + if plaintext and preserve: + content = u'\n<pre>%s</pre>\n' % ustr(content) + elif plaintext: + content = '\n%s\n' % plaintext2html(content, container_tag) + else: + content = re.sub(r'(?i)(</?(?:html|body|head|!\s*DOCTYPE)[^>]*>)', '', content) + content = u'\n%s\n' % ustr(content) + # Force all tags to lowercase + html = re.sub(r'(</?)\W*(\w+)([ >])', + lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html) + insert_location = html.find('</body>') + if insert_location == -1: + insert_location = html.find('</html>') + if insert_location == -1: + return '%s%s' % (html, content) + return '%s%s%s' % (html[:insert_location], content, html[insert_location:]) + +#---------------------------------------------------------- +# Emails +#---------------------------------------------------------- + +# matches any email in a body of text +email_re = re.compile(r"""([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63})""", re.VERBOSE) + +# matches a string containing only one email +single_email_re = re.compile(r"""^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63}$""", re.VERBOSE) + +# update command in emails body +command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) + +# Updated in 7.0 to match the model name as well +# Typical form of references is <timestamp-openerp-record_id-model_name@domain> +# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain +reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?[^>]*@([^>]*)>", re.UNICODE) +discussion_re = re.compile("<.*-open(?:object|erp)-private[^>]*@([^>]*)>", re.UNICODE) + +mail_header_msgid_re = re.compile('<[^<>]+>') + + +def generate_tracking_message_id(res_id): + """Returns a string that can be used in the Message-ID RFC822 header field + + Used to track the replies related to a given object thanks to the "In-Reply-To" + or "References" fields that Mail User Agents will set. + """ + try: + rnd = random.SystemRandom().random() + except NotImplementedError: + rnd = random.random() + rndstr = ("%.15f" % rnd)[2:] + return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname()) + +def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False, + attachments=None, message_id=None, references=None, openobject_id=False, debug=False, subtype='plain', headers=None, + smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None): + """Low-level function for sending an email (deprecated). + + :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead. + :param email_from: A string used to fill the `From` header, if falsy, + config['email_from'] is used instead. Also used for + the `Reply-To` header if `reply_to` is not provided + :param email_to: a sequence of addresses to send the mail to. + """ + + # If not cr, get cr from current thread database + local_cr = None + if not cr: + db_name = getattr(threading.currentThread(), 'dbname', None) + if db_name: + local_cr = cr = openerp.registry(db_name).cursor() + else: + raise Exception("No database cursor found, please pass one explicitly") + + # Send Email + try: + mail_server_pool = openerp.registry(cr.dbname)['ir.mail_server'] + res = False + # Pack Message into MIME Object + email_msg = mail_server_pool.build_email(email_from, email_to, subject, body, email_cc, email_bcc, reply_to, + attachments, message_id, references, openobject_id, subtype, headers=headers) + + res = mail_server_pool.send_email(cr, uid or 1, email_msg, mail_server_id=None, + smtp_server=smtp_server, smtp_port=smtp_port, smtp_user=smtp_user, smtp_password=smtp_password, + smtp_encryption=('ssl' if ssl else None), smtp_debug=debug) + except Exception: + _logger.exception("tools.email_send failed to deliver email") + return False + finally: + if local_cr: + cr.close() + return res + +def email_split(text): + """ Return a list of the email addresses found in ``text`` """ + if not text: + return [] + return [addr[1] for addr in getaddresses([text]) + # getaddresses() returns '' when email parsing fails, and + # sometimes returns emails without at least '@'. The '@' + # is strictly required in RFC2822's `addr-spec`. + if addr[1] + if '@' in addr[1]] + +def email_split_and_format(text): + """ Return a list of email addresses found in ``text``, formatted using + formataddr. """ + if not text: + return [] + return [formataddr((addr[0], addr[1])) for addr in getaddresses([text]) + # getaddresses() returns '' when email parsing fails, and + # sometimes returns emails without at least '@'. The '@' + # is strictly required in RFC2822's `addr-spec`. + if addr[1] + if '@' in addr[1]] + +def email_references(references): + ref_match, model, thread_id, hostname, is_private = False, False, False, False, False + if references: + ref_match = reference_re.search(references) + if ref_match: + model = ref_match.group(2) + thread_id = int(ref_match.group(1)) + hostname = ref_match.group(3) + else: + ref_match = discussion_re.search(references) + if ref_match: + is_private = True + return (ref_match, model, thread_id, hostname, is_private) + +# was mail_message.decode() +def decode_smtp_header(smtp_header): + """Returns unicode() string conversion of the given encoded smtp header + text. email.header decode_header method return a decoded string and its + charset for each decoded par of the header. This method unicodes the + decoded header and join them in a complete string. """ + if smtp_header: + text = decode_header(smtp_header.replace('\r', '')) + # The joining space will not be needed as of Python 3.3 + # See https://hg.python.org/cpython/rev/8c03fe231877 + return ' '.join([ustr(x[0], x[1]) for x in text]) + return u'' + +# was mail_thread.decode_header() +def decode_message_header(message, header, separator=' '): + return separator.join(map(decode_smtp_header, filter(None, message.get_all(header, [])))) diff --git a/odoo/tools/mimetypes.py b/odoo/tools/mimetypes.py new file mode 100644 index 00000000000..581dcb27bd6 --- /dev/null +++ b/odoo/tools/mimetypes.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +""" +Mimetypes-related utilities + +# TODO: reexport stdlib mimetypes? +""" +import collections +import io +import logging +import re +import zipfile + +__all__ = ['guess_mimetype'] + +_logger = logging.getLogger(__name__) + +# We define our own guess_mimetype implementation and if magic is available we +# use it instead. + +# discriminants for zip-based file formats +_ooxml_dirs = { + 'word/': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'pt/': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'xl/': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +} +def _check_ooxml(data): + with io.BytesIO(data) as f, zipfile.ZipFile(f) as z: + filenames = z.namelist() + # OOXML documents should have a [Content_Types].xml file for early + # check that we're interested in this thing at all + if '[Content_Types].xml' not in filenames: + return False + + # then there is a directory whose name denotes the type of the file: + # word, pt (powerpoint) or xl (excel) + for dirname, mime in _ooxml_dirs.iteritems(): + if any(entry.startswith(dirname) for entry in filenames): + return mime + + return False + + +# checks that a string looks kinda sorta like a mimetype +_mime_validator = re.compile(r""" + [\w-]+ # type-name + / # subtype separator + [\w-]+ # registration facet or subtype + (?:\.[\w-]+)* # optional faceted name + (?:\+[\w-]+)? # optional structured syntax specifier +""", re.VERBOSE) +def _check_open_container_format(data): + with io.BytesIO(data) as f, zipfile.ZipFile(f) as z: + # OCF zip files must contain a ``mimetype`` entry + if 'mimetype' not in z.namelist(): + return False + + # it holds the exact mimetype for the file + marcel = z.read('mimetype') + # check that it's not too long (RFC6838 § 4.2 restricts type and + # subtype to 127 characters each + separator, strongly recommends + # limiting them to 64 but does not require it) and that it looks a lot + # like a valid mime type + if len(marcel) < 256 and _mime_validator.match(marcel): + return marcel + + return False + +_xls_pattern = re.compile(""" + \x09\x08\x10\x00\x00\x06\x05\x00 + | \xFD\xFF\xFF\xFF(\x10|\x1F|\x20|"|\\#|\\(|\\)) +""", re.VERBOSE) +_ppt_pattern = re.compile(""" + \x00\x6E\x1E\xF0 + | \x0F\x00\xE8\x03 + | \xA0\x46\x1D\xF0 + | \xFD\xFF\xFF\xFF(\x0E|\x1C|\x43)\x00\x00\x00 +""", re.VERBOSE) +def _check_olecf(data): + """ Pre-OOXML Office formats are OLE Compound Files which all use the same + file signature ("magic bytes") and should have a subheader at offset 512 + (0x200). + + Subheaders taken from http://www.garykessler.net/library/file_sigs.html + according to which Mac office files *may* have different subheaders. We'll + ignore that. + """ + offset = 0x200 + if data.startswith('\xEC\xA5\xC1\x00', offset): + return 'application/msword' + # the _xls_pattern stuff doesn't seem to work correctly (the test file + # only has a bunch of \xf* at offset 0x200), that apparently works + elif 'Microsoft Excel' in data: + return 'application/vnd.ms-excel' + elif _ppt_pattern.match(data, offset): + return 'application/vnd.ms-powerpoint' + return False + +# for "master" formats with many subformats, discriminants is a list of +# functions, tried in order and the first non-falsy value returned is the +# selected mime type. If all functions return falsy values, the master +# mimetype is returned. +_Entry = collections.namedtuple('_Entry', ['mimetype', 'signatures', 'discriminants']) +_mime_mappings = ( + # pdf + _Entry('application/pdf', ['%PDF'], []), + # jpg, jpeg, png, gif, bmp + _Entry('image/jpeg', ['\xFF\xD8\xFF\xE0', '\xFF\xD8\xFF\xE2', '\xFF\xD8\xFF\xE3', '\xFF\xD8\xFF\xE1'], []), + _Entry('image/png', ['\x89PNG\r\n\x1A\n'], []), + _Entry('image/gif', ['GIF87a', 'GIF89a'], []), + _Entry('image/bmp', ['BM'], []), + # OLECF files in general (Word, Excel, PPT, default to word because why not?) + _Entry('application/msword', ['\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x0D\x44\x4F\x43'], [ + _check_olecf + ]), + # zip, but will include jar, odt, ods, odp, docx, xlsx, pptx, apk + _Entry('application/zip', ['PK\x03\x04'], [_check_ooxml, _check_open_container_format]), +) +def guess_mimetype(bin_data, default='application/octet-stream'): + """ Attempts to guess the mime type of the provided binary data, similar + to but significantly more limited than libmagic + + :param str bin_data: binary data to try and guess a mime type for + :returns: matched mimetype or ``application/octet-stream`` if none matched + """ + # by default, guess the type using the magic number of file hex signature (like magic, but more limited) + # see http://www.filesignatures.net/ for file signatures + for entry in _mime_mappings: + for signature in entry.signatures: + if bin_data.startswith(signature): + for discriminant in entry.discriminants: + try: + guess = discriminant(bin_data) + if guess: return guess + except Exception: + # log-and-next + _logger.getChild('guess_mimetype').warn( + "Sub-checker '%s' of type '%s' failed", + discriminant.__name__, entry.mimetype, + exc_info=True + ) + # if no discriminant or no discriminant matches, return + # primary mime type + return entry.mimetype + return default + + +try: + import magic +except ImportError: + magic = None +else: + # There are 2 python libs named 'magic' with incompatible api. + + # magic from pypi https://pypi.python.org/pypi/python-magic/ + if hasattr(magic,'from_buffer'): + guess_mimetype = lambda bin_data: magic.from_buffer(bin_data, mime=True) + # magic from file(1) https://packages.debian.org/squeeze/python-magic + elif hasattr(magic,'open'): + ms = magic.open(magic.MAGIC_MIME_TYPE) + ms.load() + guess_mimetype = ms.buffer diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py new file mode 100644 index 00000000000..2dbe2bad382 --- /dev/null +++ b/odoo/tools/misc.py @@ -0,0 +1,1178 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + + +""" +Miscellaneous tools used by OpenERP. +""" + +from functools import wraps +import cPickle +import cProfile +from contextlib import contextmanager +import subprocess +import logging +import os +import passlib.utils +import re +import socket +import sys +import threading +import time +import werkzeug.utils +import zipfile +from cStringIO import StringIO +from collections import defaultdict, Hashable, Iterable, Mapping, OrderedDict +from itertools import islice, izip, groupby +from lxml import etree +from which import which +from threading import local +import traceback +import csv +from operator import itemgetter + +try: + from html2text import html2text +except ImportError: + html2text = None + +from config import config +from cache import * +from .parse_version import parse_version + +import openerp +# get_encodings, ustr and exception_to_unicode were originally from tools.misc. +# There are moved to loglevels until we refactor tools. +from openerp.loglevels import get_encodings, ustr, exception_to_unicode # noqa + +_logger = logging.getLogger(__name__) + +# List of etree._Element subclasses that we choose to ignore when parsing XML. +# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones. +SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase, etree._Entity) + +# Configure default global parser +etree.set_default_parser(etree.XMLParser(resolve_entities=False)) + +#---------------------------------------------------------- +# Subprocesses +#---------------------------------------------------------- + +def find_in_path(name): + path = os.environ.get('PATH', os.defpath).split(os.pathsep) + if config.get('bin_path') and config['bin_path'] != 'None': + path.append(config['bin_path']) + return which(name, path=os.pathsep.join(path)) + +def _exec_pipe(prog, args, env=None): + cmd = (prog,) + args + # on win32, passing close_fds=True is not compatible + # with redirecting std[in/err/out] + close_fds = os.name=="posix" + pop = subprocess.Popen(cmd, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=close_fds, env=env) + return pop.stdin, pop.stdout + +def exec_command_pipe(name, *args): + prog = find_in_path(name) + if not prog: + raise Exception('Command `%s` not found.' % name) + return _exec_pipe(prog, args) + +#---------------------------------------------------------- +# Postgres subprocesses +#---------------------------------------------------------- + +def find_pg_tool(name): + path = None + if config['pg_path'] and config['pg_path'] != 'None': + path = config['pg_path'] + try: + return which(name, path=path) + except IOError: + raise Exception('Command `%s` not found.' % name) + +def exec_pg_environ(): + """ + Force the database PostgreSQL environment variables to the database + configuration of Odoo. + + Note: On systems where pg_restore/pg_dump require an explicit password + (i.e. on Windows where TCP sockets are used), it is necessary to pass the + postgres user password in the PGPASSWORD environment variable or in a + special .pgpass file. + + See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html + """ + env = os.environ.copy() + if openerp.tools.config['db_host']: + env['PGHOST'] = openerp.tools.config['db_host'] + if openerp.tools.config['db_port']: + env['PGPORT'] = str(openerp.tools.config['db_port']) + if openerp.tools.config['db_user']: + env['PGUSER'] = openerp.tools.config['db_user'] + if openerp.tools.config['db_password']: + env['PGPASSWORD'] = openerp.tools.config['db_password'] + return env + +def exec_pg_command(name, *args): + prog = find_pg_tool(name) + env = exec_pg_environ() + with open(os.devnull) as dn: + args2 = (prog,) + args + rc = subprocess.call(args2, env=env, stdout=dn, stderr=subprocess.STDOUT) + if rc: + raise Exception('Postgres subprocess %s error %s' % (args2, rc)) + +def exec_pg_command_pipe(name, *args): + prog = find_pg_tool(name) + env = exec_pg_environ() + return _exec_pipe(prog, args, env) + +#---------------------------------------------------------- +# File paths +#---------------------------------------------------------- +#file_path_root = os.getcwd() +#file_path_addons = os.path.join(file_path_root, 'addons') + +def file_open(name, mode="r", subdir='addons', pathinfo=False): + """Open a file from the OpenERP root, using a subdir folder. + + Example:: + + >>> file_open('hr/report/timesheer.xsl') + >>> file_open('addons/hr/report/timesheet.xsl') + >>> file_open('../../base/report/rml_template.xsl', subdir='addons/hr/report', pathinfo=True) + + @param name name of the file + @param mode file open mode + @param subdir subdirectory + @param pathinfo if True returns tuple (fileobject, filepath) + + @return fileobject if pathinfo is False else (fileobject, filepath) + """ + import openerp.modules as addons + adps = addons.module.ad_paths + rtp = os.path.normcase(os.path.abspath(config['root_path'])) + + basename = name + + if os.path.isabs(name): + # It is an absolute path + # Is it below 'addons_path' or 'root_path'? + name = os.path.normcase(os.path.normpath(name)) + for root in adps + [rtp]: + root = os.path.normcase(os.path.normpath(root)) + os.sep + if name.startswith(root): + base = root.rstrip(os.sep) + name = name[len(base) + 1:] + break + else: + # It is outside the OpenERP root: skip zipfile lookup. + base, name = os.path.split(name) + return _fileopen(name, mode=mode, basedir=base, pathinfo=pathinfo, basename=basename) + + if name.replace(os.sep, '/').startswith('addons/'): + subdir = 'addons' + name2 = name[7:] + elif subdir: + name = os.path.join(subdir, name) + if name.replace(os.sep, '/').startswith('addons/'): + subdir = 'addons' + name2 = name[7:] + else: + name2 = name + + # First, try to locate in addons_path + if subdir: + for adp in adps: + try: + return _fileopen(name2, mode=mode, basedir=adp, + pathinfo=pathinfo, basename=basename) + except IOError: + pass + + # Second, try to locate in root_path + return _fileopen(name, mode=mode, basedir=rtp, pathinfo=pathinfo, basename=basename) + + +def _fileopen(path, mode, basedir, pathinfo, basename=None): + name = os.path.normpath(os.path.join(basedir, path)) + + if basename is None: + basename = name + # Give higher priority to module directories, which is + # a more common case than zipped modules. + if os.path.isfile(name): + fo = open(name, mode) + if pathinfo: + return fo, name + return fo + + # Support for loading modules in zipped form. + # This will not work for zipped modules that are sitting + # outside of known addons paths. + head = os.path.normpath(path) + zipname = False + while os.sep in head: + head, tail = os.path.split(head) + if not tail: + break + if zipname: + zipname = os.path.join(tail, zipname) + else: + zipname = tail + zpath = os.path.join(basedir, head + '.zip') + if zipfile.is_zipfile(zpath): + from cStringIO import StringIO + zfile = zipfile.ZipFile(zpath) + try: + fo = StringIO() + fo.write(zfile.read(os.path.join( + os.path.basename(head), zipname).replace( + os.sep, '/'))) + fo.seek(0) + if pathinfo: + return fo, name + return fo + except Exception: + pass + # Not found + if name.endswith('.rml'): + raise IOError('Report %r doesn\'t exist or deleted' % basename) + raise IOError('File not found: %s' % basename) + + +#---------------------------------------------------------- +# iterables +#---------------------------------------------------------- +def flatten(list): + """Flatten a list of elements into a uniqu list + Author: Christophe Simonis (christophe@tinyerp.com) + + Examples:: + >>> flatten(['a']) + ['a'] + >>> flatten('b') + ['b'] + >>> flatten( [] ) + [] + >>> flatten( [[], [[]]] ) + [] + >>> flatten( [[['a','b'], 'c'], 'd', ['e', [], 'f']] ) + ['a', 'b', 'c', 'd', 'e', 'f'] + >>> t = (1,2,(3,), [4, 5, [6, [7], (8, 9), ([10, 11, (12, 13)]), [14, [], (15,)], []]]) + >>> flatten(t) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + """ + + def isiterable(x): + return hasattr(x, "__iter__") + + r = [] + for e in list: + if isiterable(e): + map(r.append, flatten(e)) + else: + r.append(e) + return r + +def reverse_enumerate(l): + """Like enumerate but in the other sens + + Usage:: + >>> a = ['a', 'b', 'c'] + >>> it = reverse_enumerate(a) + >>> it.next() + (2, 'c') + >>> it.next() + (1, 'b') + >>> it.next() + (0, 'a') + >>> it.next() + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + """ + return izip(xrange(len(l)-1, -1, -1), reversed(l)) + +def partition(pred, elems): + """ Return a pair equivalent to: + ``filter(pred, elems), filter(lambda x: not pred(x), elems)` """ + yes, nos = [], [] + for elem in elems: + (yes if pred(elem) else nos).append(elem) + return yes, nos + +def topological_sort(elems): + """ Return a list of elements sorted so that their dependencies are listed + before them in the result. + + :param elems: specifies the elements to sort with their dependencies; it is + a dictionary like `{element: dependencies}` where `dependencies` is a + collection of elements that must appear before `element`. The elements + of `dependencies` are not required to appear in `elems`; they will + simply not appear in the result. + + :returns: a list with the keys of `elems` sorted according to their + specification. + """ + # the algorithm is inspired by [Tarjan 1976], + # http://en.wikipedia.org/wiki/Topological_sorting#Algorithms + result = [] + visited = set() + + def visit(n): + if n not in visited: + visited.add(n) + if n in elems: + # first visit all dependencies of n, then append n to result + map(visit, elems[n]) + result.append(n) + + map(visit, elems) + + return result + + +try: + import xlwt + + # add some sanitizations to respect the excel sheet name restrictions + # as the sheet name is often translatable, can not control the input + class PatchedWorkbook(xlwt.Workbook): + def add_sheet(self, name): + # invalid Excel character: []:*?/\ + name = re.sub(r'[\[\]:*?/\\]', '', name) + + # maximum size is 31 characters + name = name[:31] + return super(PatchedWorkbook, self).add_sheet(name) + + xlwt.Workbook = PatchedWorkbook + +except ImportError: + xlwt = None + + +class UpdateableStr(local): + """ Class that stores an updateable string (used in wizards) + """ + + def __init__(self, string=''): + self.string = string + + def __str__(self): + return str(self.string) + + def __repr__(self): + return str(self.string) + + def __nonzero__(self): + return bool(self.string) + + +class UpdateableDict(local): + """Stores an updateable dict to use in wizards + """ + + def __init__(self, dict=None): + if dict is None: + dict = {} + self.dict = dict + + def __str__(self): + return str(self.dict) + + def __repr__(self): + return str(self.dict) + + def clear(self): + return self.dict.clear() + + def keys(self): + return self.dict.keys() + + def __setitem__(self, i, y): + self.dict.__setitem__(i, y) + + def __getitem__(self, i): + return self.dict.__getitem__(i) + + def copy(self): + return self.dict.copy() + + def iteritems(self): + return self.dict.iteritems() + + def iterkeys(self): + return self.dict.iterkeys() + + def itervalues(self): + return self.dict.itervalues() + + def pop(self, k, d=None): + return self.dict.pop(k, d) + + def popitem(self): + return self.dict.popitem() + + def setdefault(self, k, d=None): + return self.dict.setdefault(k, d) + + def update(self, E, **F): + return self.dict.update(E, F) + + def values(self): + return self.dict.values() + + def get(self, k, d=None): + return self.dict.get(k, d) + + def has_key(self, k): + return self.dict.has_key(k) + + def items(self): + return self.dict.items() + + def __cmp__(self, y): + return self.dict.__cmp__(y) + + def __contains__(self, k): + return self.dict.__contains__(k) + + def __delitem__(self, y): + return self.dict.__delitem__(y) + + def __eq__(self, y): + return self.dict.__eq__(y) + + def __ge__(self, y): + return self.dict.__ge__(y) + + def __gt__(self, y): + return self.dict.__gt__(y) + + def __hash__(self): + return self.dict.__hash__() + + def __iter__(self): + return self.dict.__iter__() + + def __le__(self, y): + return self.dict.__le__(y) + + def __len__(self): + return self.dict.__len__() + + def __lt__(self, y): + return self.dict.__lt__(y) + + def __ne__(self, y): + return self.dict.__ne__(y) + +def to_xml(s): + return s.replace('&','&').replace('<','<').replace('>','>') + +def get_iso_codes(lang): + if lang.find('_') != -1: + if lang.split('_')[0] == lang.split('_')[1].lower(): + lang = lang.split('_')[0] + return lang + +def scan_languages(): + """ Returns all languages supported by OpenERP for translation + + :returns: a list of (lang_code, lang_name) pairs + :rtype: [(str, unicode)] + """ + csvpath = openerp.modules.module.get_resource_path('base', 'res', 'res.lang.csv') + try: + # read (code, name) from languages in base/res/res.lang.csv + result = [] + with open(csvpath) as csvfile: + reader = csv.reader(csvfile, delimiter=',', quotechar='"') + fields = reader.next() + code_index = fields.index("code") + name_index = fields.index("name") + for row in reader: + result.append((ustr(row[code_index]), ustr(row[name_index]))) + except Exception: + _logger.error("Could not read %s", csvpath) + result = [] + + return sorted(result or [('en_US', u'English')], key=itemgetter(1)) + +def get_user_companies(cr, user): + def _get_company_children(cr, ids): + if not ids: + return [] + cr.execute('SELECT id FROM res_company WHERE parent_id IN %s', (tuple(ids),)) + res = [x[0] for x in cr.fetchall()] + res.extend(_get_company_children(cr, res)) + return res + cr.execute('SELECT company_id FROM res_users WHERE id=%s', (user,)) + user_comp = cr.fetchone()[0] + if not user_comp: + return [] + return [user_comp] + _get_company_children(cr, [user_comp]) + +def mod10r(number): + """ + Input number : account or invoice number + Output return: the same number completed with the recursive mod10 + key + """ + codec=[0,9,4,6,8,2,7,1,3,5] + report = 0 + result="" + for digit in number: + result += digit + if digit.isdigit(): + report = codec[ (int(digit) + report) % 10 ] + return result + str((10 - report) % 10) + +def str2bool(s, default=None): + s = ustr(s).lower() + y = 'y yes 1 true t on'.split() + n = 'n no 0 false f off'.split() + if s not in (y + n): + if default is None: + raise ValueError('Use 0/1/yes/no/true/false/on/off') + return bool(default) + return s in y + +def human_size(sz): + """ + Return the size in a human readable format + """ + if not sz: + return False + units = ('bytes', 'Kb', 'Mb', 'Gb') + if isinstance(sz,basestring): + sz=len(sz) + s, i = float(sz), 0 + while s >= 1024 and i < len(units)-1: + s /= 1024 + i += 1 + return "%0.2f %s" % (s, units[i]) + +def logged(f): + @wraps(f) + def wrapper(*args, **kwargs): + from pprint import pformat + + vector = ['Call -> function: %r' % f] + for i, arg in enumerate(args): + vector.append(' arg %02d: %s' % (i, pformat(arg))) + for key, value in kwargs.items(): + vector.append(' kwarg %10s: %s' % (key, pformat(value))) + + timeb4 = time.time() + res = f(*args, **kwargs) + + vector.append(' result: %s' % pformat(res)) + vector.append(' time delta: %s' % (time.time() - timeb4)) + _logger.debug('\n'.join(vector)) + return res + + return wrapper + +class profile(object): + def __init__(self, fname=None): + self.fname = fname + + def __call__(self, f): + @wraps(f) + def wrapper(*args, **kwargs): + profile = cProfile.Profile() + result = profile.runcall(f, *args, **kwargs) + profile.dump_stats(self.fname or ("%s.cprof" % (f.func_name,))) + return result + + return wrapper + +def detect_ip_addr(): + """Try a very crude method to figure out a valid external + IP or hostname for the current machine. Don't rely on this + for binding to an interface, but it could be used as basis + for constructing a remote URL to the server. + """ + def _detect_ip_addr(): + from array import array + from struct import pack, unpack + + try: + import fcntl + except ImportError: + fcntl = None + + ip_addr = None + + if not fcntl: # not UNIX: + host = socket.gethostname() + ip_addr = socket.gethostbyname(host) + else: # UNIX: + # get all interfaces: + nbytes = 128 * 32 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + names = array('B', '\0' * nbytes) + #print 'names: ', names + outbytes = unpack('iL', fcntl.ioctl( s.fileno(), 0x8912, pack('iL', nbytes, names.buffer_info()[0])))[0] + namestr = names.tostring() + + # try 64 bit kernel: + for i in range(0, outbytes, 40): + name = namestr[i:i+16].split('\0', 1)[0] + if name != 'lo': + ip_addr = socket.inet_ntoa(namestr[i+20:i+24]) + break + + # try 32 bit kernel: + if ip_addr is None: + ifaces = filter(None, [namestr[i:i+32].split('\0', 1)[0] for i in range(0, outbytes, 32)]) + + for ifname in [iface for iface in ifaces if iface != 'lo']: + ip_addr = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915, pack('256s', ifname[:15]))[20:24]) + break + + return ip_addr or 'localhost' + + try: + ip_addr = _detect_ip_addr() + except Exception: + ip_addr = 'localhost' + return ip_addr + +DEFAULT_SERVER_DATE_FORMAT = "%Y-%m-%d" +DEFAULT_SERVER_TIME_FORMAT = "%H:%M:%S" +DEFAULT_SERVER_DATETIME_FORMAT = "%s %s" % ( + DEFAULT_SERVER_DATE_FORMAT, + DEFAULT_SERVER_TIME_FORMAT) + +# Python's strftime supports only the format directives +# that are available on the platform's libc, so in order to +# be cross-platform we map to the directives required by +# the C standard (1989 version), always available on platforms +# with a C standard implementation. +DATETIME_FORMATS_MAP = { + '%C': '', # century + '%D': '%m/%d/%Y', # modified %y->%Y + '%e': '%d', + '%E': '', # special modifier + '%F': '%Y-%m-%d', + '%g': '%Y', # modified %y->%Y + '%G': '%Y', + '%h': '%b', + '%k': '%H', + '%l': '%I', + '%n': '\n', + '%O': '', # special modifier + '%P': '%p', + '%R': '%H:%M', + '%r': '%I:%M:%S %p', + '%s': '', #num of seconds since epoch + '%T': '%H:%M:%S', + '%t': ' ', # tab + '%u': ' %w', + '%V': '%W', + '%y': '%Y', # Even if %y works, it's ambiguous, so we should use %Y + '%+': '%Y-%m-%d %H:%M:%S', + + # %Z is a special case that causes 2 problems at least: + # - the timezone names we use (in res_user.context_tz) come + # from pytz, but not all these names are recognized by + # strptime(), so we cannot convert in both directions + # when such a timezone is selected and %Z is in the format + # - %Z is replaced by an empty string in strftime() when + # there is not tzinfo in a datetime value (e.g when the user + # did not pick a context_tz). The resulting string does not + # parse back if the format requires %Z. + # As a consequence, we strip it completely from format strings. + # The user can always have a look at the context_tz in + # preferences to check the timezone. + '%z': '', + '%Z': '', +} + +POSIX_TO_LDML = { + 'a': 'E', + 'A': 'EEEE', + 'b': 'MMM', + 'B': 'MMMM', + #'c': '', + 'd': 'dd', + 'H': 'HH', + 'I': 'hh', + 'j': 'DDD', + 'm': 'MM', + 'M': 'mm', + 'p': 'a', + 'S': 'ss', + 'U': 'w', + 'w': 'e', + 'W': 'w', + 'y': 'yy', + 'Y': 'yyyy', + # see comments above, and babel's format_datetime assumes an UTC timezone + # for naive datetime objects + #'z': 'Z', + #'Z': 'z', +} + +def posix_to_ldml(fmt, locale): + """ Converts a posix/strftime pattern into an LDML date format pattern. + + :param fmt: non-extended C89/C90 strftime pattern + :param locale: babel locale used for locale-specific conversions (e.g. %x and %X) + :return: unicode + """ + buf = [] + pc = False + quoted = [] + + for c in fmt: + # LDML date format patterns uses letters, so letters must be quoted + if not pc and c.isalpha(): + quoted.append(c if c != "'" else "''") + continue + if quoted: + buf.append("'") + buf.append(''.join(quoted)) + buf.append("'") + quoted = [] + + if pc: + if c == '%': # escaped percent + buf.append('%') + elif c == 'x': # date format, short seems to match + buf.append(locale.date_formats['short'].pattern) + elif c == 'X': # time format, seems to include seconds. short does not + buf.append(locale.time_formats['medium'].pattern) + else: # look up format char in static mapping + buf.append(POSIX_TO_LDML[c]) + pc = False + elif c == '%': + pc = True + else: + buf.append(c) + + # flush anything remaining in quoted buffer + if quoted: + buf.append("'") + buf.append(''.join(quoted)) + buf.append("'") + + return ''.join(buf) + +def split_every(n, iterable, piece_maker=tuple): + """Splits an iterable into length-n pieces. The last piece will be shorter + if ``n`` does not evenly divide the iterable length. + @param ``piece_maker``: function to build the pieces + from the slices (tuple,list,...) + """ + iterator = iter(iterable) + piece = piece_maker(islice(iterator, n)) + while piece: + yield piece + piece = piece_maker(islice(iterator, n)) + +if __name__ == '__main__': + import doctest + doctest.testmod() + +class upload_data_thread(threading.Thread): + def __init__(self, email, data, type): + self.args = [('email',email),('type',type),('data',data)] + super(upload_data_thread,self).__init__() + def run(self): + try: + import urllib + args = urllib.urlencode(self.args) + fp = urllib.urlopen('http://www.openerp.com/scripts/survey.php', args) + fp.read() + fp.close() + except Exception: + pass + +def upload_data(email, data, type='SURVEY'): + a = upload_data_thread(email, data, type) + a.start() + return True + +def get_and_group_by_field(cr, uid, obj, ids, field, context=None): + """ Read the values of ``field´´ for the given ``ids´´ and group ids by value. + + :param string field: name of the field we want to read and group by + :return: mapping of field values to the list of ids that have it + :rtype: dict + """ + res = {} + for record in obj.read(cr, uid, ids, [field], context=context): + key = record[field] + res.setdefault(key[0] if isinstance(key, tuple) else key, []).append(record['id']) + return res + +def get_and_group_by_company(cr, uid, obj, ids, context=None): + return get_and_group_by_field(cr, uid, obj, ids, field='company_id', context=context) + +# port of python 2.6's attrgetter with support for dotted notation +def resolve_attr(obj, attr): + for name in attr.split("."): + obj = getattr(obj, name) + return obj + +def attrgetter(*items): + if len(items) == 1: + attr = items[0] + def g(obj): + return resolve_attr(obj, attr) + else: + def g(obj): + return tuple(resolve_attr(obj, attr) for attr in items) + return g + +class unquote(str): + """A subclass of str that implements repr() without enclosing quotation marks + or escaping, keeping the original string untouched. The name come from Lisp's unquote. + One of the uses for this is to preserve or insert bare variable names within dicts during eval() + of a dict's repr(). Use with care. + + Some examples (notice that there are never quotes surrounding + the ``active_id`` name: + + >>> unquote('active_id') + active_id + >>> d = {'test': unquote('active_id')} + >>> d + {'test': active_id} + >>> print d + {'test': active_id} + """ + def __repr__(self): + return self + +class UnquoteEvalContext(defaultdict): + """Defaultdict-based evaluation context that returns + an ``unquote`` string for any missing name used during + the evaluation. + Mostly useful for evaluating OpenERP domains/contexts that + may refer to names that are unknown at the time of eval, + so that when the context/domain is converted back to a string, + the original names are preserved. + + **Warning**: using an ``UnquoteEvalContext`` as context for ``eval()`` or + ``safe_eval()`` will shadow the builtins, which may cause other + failures, depending on what is evaluated. + + Example (notice that ``section_id`` is preserved in the final + result) : + + >>> context_str = "{'default_user_id': uid, 'default_section_id': section_id}" + >>> eval(context_str, UnquoteEvalContext(uid=1)) + {'default_user_id': 1, 'default_section_id': section_id} + + """ + def __init__(self, *args, **kwargs): + super(UnquoteEvalContext, self).__init__(None, *args, **kwargs) + + def __missing__(self, key): + return unquote(key) + + +class mute_logger(object): + """Temporary suppress the logging. + Can be used as context manager or decorator. + + @mute_logger('openerp.plic.ploc') + def do_stuff(): + blahblah() + + with mute_logger('openerp.foo.bar'): + do_suff() + + """ + def __init__(self, *loggers): + self.loggers = loggers + + def filter(self, record): + return 0 + + def __enter__(self): + for logger in self.loggers: + assert isinstance(logger, basestring),\ + "A logger name must be a string, got %s" % type(logger) + logging.getLogger(logger).addFilter(self) + + def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): + for logger in self.loggers: + logging.getLogger(logger).removeFilter(self) + + def __call__(self, func): + @wraps(func) + def deco(*args, **kwargs): + with self: + return func(*args, **kwargs) + return deco + +_ph = object() +class CountingStream(object): + """ Stream wrapper counting the number of element it has yielded. Similar + role to ``enumerate``, but for use when the iteration process of the stream + isn't fully under caller control (the stream can be iterated from multiple + points including within a library) + + ``start`` allows overriding the starting index (the index before the first + item is returned). + + On each iteration (call to :meth:`~.next`), increases its :attr:`~.index` + by one. + + .. attribute:: index + + ``int``, index of the last yielded element in the stream. If the stream + has ended, will give an index 1-past the stream + """ + def __init__(self, stream, start=-1): + self.stream = iter(stream) + self.index = start + self.stopped = False + def __iter__(self): + return self + def next(self): + if self.stopped: raise StopIteration() + self.index += 1 + val = next(self.stream, _ph) + if val is _ph: + self.stopped = True + raise StopIteration() + return val + +def stripped_sys_argv(*strip_args): + """Return sys.argv with some arguments stripped, suitable for reexecution or subprocesses""" + strip_args = sorted(set(strip_args) | set(['-s', '--save', '-u', '--update', '-i', '--init', '--i18n-overwrite'])) + assert all(config.parser.has_option(s) for s in strip_args) + takes_value = dict((s, config.parser.get_option(s).takes_value()) for s in strip_args) + + longs, shorts = list(tuple(y) for _, y in groupby(strip_args, lambda x: x.startswith('--'))) + longs_eq = tuple(l + '=' for l in longs if takes_value[l]) + + args = sys.argv[:] + + def strip(args, i): + return args[i].startswith(shorts) \ + or args[i].startswith(longs_eq) or (args[i] in longs) \ + or (i >= 1 and (args[i - 1] in strip_args) and takes_value[args[i - 1]]) + + return [x for i, x in enumerate(args) if not strip(args, i)] + +class ConstantMapping(Mapping): + """ + An immutable mapping returning the provided value for every single key. + + Useful for default value to methods + """ + __slots__ = ['_value'] + def __init__(self, val): + self._value = val + + def __len__(self): + """ + defaultdict updates its length for each individually requested key, is + that really useful? + """ + return 0 + + def __iter__(self): + """ + same as len, defaultdict udpates its iterable keyset with each key + requested, is there a point for this? + """ + return iter([]) + + def __getitem__(self, item): + return self._value + + +def dumpstacks(sig=None, frame=None): + """ Signal handler: dump a stack trace for each existing thread.""" + code = [] + + def extract_stack(stack): + for filename, lineno, name, line in traceback.extract_stack(stack): + yield 'File: "%s", line %d, in %s' % (filename, lineno, name) + if line: + yield " %s" % (line.strip(),) + + # code from http://stackoverflow.com/questions/132058/getting-stack-trace-from-a-running-python-application#answer-2569696 + # modified for python 2.5 compatibility + threads_info = {th.ident: {'name': th.name, + 'uid': getattr(th, 'uid', 'n/a'), + 'dbname': getattr(th, 'dbname', 'n/a')} + for th in threading.enumerate()} + for threadId, stack in sys._current_frames().items(): + thread_info = threads_info.get(threadId, {}) + code.append("\n# Thread: %s (id:%s) (db:%s) (uid:%s)" % + (thread_info.get('name', 'n/a'), + threadId, + thread_info.get('dbname', 'n/a'), + thread_info.get('uid', 'n/a'))) + for line in extract_stack(stack): + code.append(line) + + if openerp.evented: + # code from http://stackoverflow.com/questions/12510648/in-gevent-how-can-i-dump-stack-traces-of-all-running-greenlets + import gc + from greenlet import greenlet + for ob in gc.get_objects(): + if not isinstance(ob, greenlet) or not ob: + continue + code.append("\n# Greenlet: %r" % (ob,)) + for line in extract_stack(ob.gr_frame): + code.append(line) + + _logger.info("\n".join(code)) + +def freehash(arg): + try: + return hash(arg) + except Exception: + if isinstance(arg, Mapping): + return hash(frozendict(arg)) + elif isinstance(arg, Iterable): + return hash(frozenset(map(freehash, arg))) + else: + return id(arg) + +class frozendict(dict): + """ An implementation of an immutable dictionary. """ + def __delitem__(self, key): + raise NotImplementedError("'__delitem__' not supported on frozendict") + def __setitem__(self, key, val): + raise NotImplementedError("'__setitem__' not supported on frozendict") + def clear(self): + raise NotImplementedError("'clear' not supported on frozendict") + def pop(self, key, default=None): + raise NotImplementedError("'pop' not supported on frozendict") + def popitem(self): + raise NotImplementedError("'popitem' not supported on frozendict") + def setdefault(self, key, default=None): + raise NotImplementedError("'setdefault' not supported on frozendict") + def update(self, *args, **kwargs): + raise NotImplementedError("'update' not supported on frozendict") + def __hash__(self): + return hash(frozenset((key, freehash(val)) for key, val in self.iteritems())) + +class Collector(Mapping): + """ A mapping from keys to lists. This is essentially a space optimization + for ``defaultdict(list)``. + """ + __slots__ = ['_map'] + def __init__(self): + self._map = {} + def add(self, key, val): + vals = self._map.setdefault(key, []) + if val not in vals: + vals.append(val) + def __getitem__(self, key): + return self._map.get(key, ()) + def __iter__(self): + return iter(self._map) + def __len__(self): + return len(self._map) + +class OrderedSet(OrderedDict): + """ A set collection that remembers the elements first insertion order. """ + def __init__(self, seq=()): + super(OrderedSet, self).__init__([(x, None) for x in seq]) + + def add(self, elem): + self[elem] = None + + def discard(self, elem): + self.pop(elem, None) + +class LastOrderedSet(OrderedSet): + """ A set collection that remembers the elements last insertion order. """ + def add(self, elem): + OrderedSet.discard(self, elem) + OrderedSet.add(self, elem) + +@contextmanager +def ignore(*exc): + try: + yield + except exc: + pass + +# Avoid DeprecationWarning while still remaining compatible with werkzeug pre-0.9 +if parse_version(getattr(werkzeug, '__version__', '0.0')) < parse_version('0.9.0'): + def html_escape(text): + return werkzeug.utils.escape(text, quote=True) +else: + def html_escape(text): + return werkzeug.utils.escape(text) + +def formatLang(env, value, digits=None, grouping=True, monetary=False, dp=False, currency_obj=False): + """ + Assuming 'Account' decimal.precision=3: + formatLang(value) -> digits=2 (default) + formatLang(value, digits=4) -> digits=4 + formatLang(value, dp='Account') -> digits=3 + formatLang(value, digits=5, dp='Account') -> digits=5 + """ + + if digits is None: + digits = DEFAULT_DIGITS = 2 + if dp: + decimal_precision_obj = env['decimal.precision'] + digits = decimal_precision_obj.precision_get(dp) + elif (hasattr(value, '_field') and isinstance(value._field, (float_field, function_field)) and value._field.digits): + digits = value._field.digits[1] + if not digits and digits is not 0: + digits = DEFAULT_DIGITS + + if isinstance(value, (str, unicode)) and not value: + return '' + + lang = env.user.company_id.partner_id.lang or 'en_US' + lang_objs = env['res.lang'].search([('code', '=', lang)]) + if not lang_objs: + lang_objs = env['res.lang'].search([], limit=1) + lang_obj = lang_objs[0] + + res = lang_obj.format('%.' + str(digits) + 'f', value, grouping=grouping, monetary=monetary) + + if currency_obj and currency_obj.symbol: + if currency_obj.position == 'after': + res = '%s %s' % (res, currency_obj.symbol) + elif currency_obj and currency_obj.position == 'before': + res = '%s %s' % (currency_obj.symbol, res) + return res + +def _consteq(str1, str2): + """ Constant-time string comparison. Suitable to compare bytestrings of fixed, + known length only, because length difference is optimized. """ + return len(str1) == len(str2) and sum(ord(x)^ord(y) for x, y in zip(str1, str2)) == 0 + +consteq = getattr(passlib.utils, 'consteq', _consteq) + +class Pickle(object): + @classmethod + def load(cls, stream, errors=False): + unpickler = cPickle.Unpickler(stream) + # pickle builtins: str/unicode, int/long, float, bool, tuple, list, dict, None + unpickler.find_global = None + try: + return unpickler.load() + except Exception: + _logger.warning('Failed unpickling data, returning default: %r', errors, exc_info=True) + return errors + + @classmethod + def loads(cls, text): + return cls.load(StringIO(text)) + + dumps = cPickle.dumps + dump = cPickle.dump + +pickle = Pickle diff --git a/odoo/tools/osutil.py b/odoo/tools/osutil.py new file mode 100644 index 00000000000..e7cd0dda4b0 --- /dev/null +++ b/odoo/tools/osutil.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +""" +Some functions related to the os and os.path module +""" + +from contextlib import contextmanager +import os +from os.path import join as opj +import shutil +import tempfile +import zipfile + +if os.name == 'nt': + import ctypes + import win32service as ws + import win32serviceutil as wsu + + +def listdir(dir, recursive=False): + """Allow to recursively get the file listing""" + dir = os.path.normpath(dir) + if not recursive: + return os.listdir(dir) + + res = [] + for root, dirs, files in walksymlinks(dir): + root = root[len(dir)+1:] + res.extend([opj(root, f) for f in files]) + return res + +def walksymlinks(top, topdown=True, onerror=None): + """ + same as os.walk but follow symlinks + attention: all symlinks are walked before all normals directories + """ + for dirpath, dirnames, filenames in os.walk(top, topdown, onerror): + if topdown: + yield dirpath, dirnames, filenames + + symlinks = filter(lambda dirname: os.path.islink(os.path.join(dirpath, dirname)), dirnames) + for s in symlinks: + for x in walksymlinks(os.path.join(dirpath, s), topdown, onerror): + yield x + + if not topdown: + yield dirpath, dirnames, filenames + +@contextmanager +def tempdir(): + tmpdir = tempfile.mkdtemp() + try: + yield tmpdir + finally: + shutil.rmtree(tmpdir) + +def zip_dir(path, stream, include_dir=True, fnct_sort=None): # TODO add ignore list + """ + : param fnct_sort : Function to be passed to "key" parameter of built-in + python sorted() to provide flexibility of sorting files + inside ZIP archive according to specific requirements. + """ + path = os.path.normpath(path) + len_prefix = len(os.path.dirname(path)) if include_dir else len(path) + if len_prefix: + len_prefix += 1 + + with zipfile.ZipFile(stream, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zipf: + for dirpath, dirnames, filenames in os.walk(path): + filenames = sorted(filenames, key=fnct_sort) + for fname in filenames: + bname, ext = os.path.splitext(fname) + ext = ext or bname + if ext not in ['.pyc', '.pyo', '.swp', '.DS_Store']: + path = os.path.normpath(os.path.join(dirpath, fname)) + if os.path.isfile(path): + zipf.write(path, path[len_prefix:]) + + +if os.name != 'nt': + getppid = os.getppid + is_running_as_nt_service = lambda: False +else: + # based on http://mail.python.org/pipermail/python-win32/2007-June/006174.html + _TH32CS_SNAPPROCESS = 0x00000002 + class _PROCESSENTRY32(ctypes.Structure): + _fields_ = [("dwSize", ctypes.c_ulong), + ("cntUsage", ctypes.c_ulong), + ("th32ProcessID", ctypes.c_ulong), + ("th32DefaultHeapID", ctypes.c_ulong), + ("th32ModuleID", ctypes.c_ulong), + ("cntThreads", ctypes.c_ulong), + ("th32ParentProcessID", ctypes.c_ulong), + ("pcPriClassBase", ctypes.c_ulong), + ("dwFlags", ctypes.c_ulong), + ("szExeFile", ctypes.c_char * 260)] + + def getppid(): + CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot + Process32First = ctypes.windll.kernel32.Process32First + Process32Next = ctypes.windll.kernel32.Process32Next + CloseHandle = ctypes.windll.kernel32.CloseHandle + hProcessSnap = CreateToolhelp32Snapshot(_TH32CS_SNAPPROCESS, 0) + current_pid = os.getpid() + try: + pe32 = _PROCESSENTRY32() + pe32.dwSize = ctypes.sizeof(_PROCESSENTRY32) + if not Process32First(hProcessSnap, ctypes.byref(pe32)): + raise OSError('Failed getting first process.') + while True: + if pe32.th32ProcessID == current_pid: + return pe32.th32ParentProcessID + if not Process32Next(hProcessSnap, ctypes.byref(pe32)): + return None + finally: + CloseHandle(hProcessSnap) + + from contextlib import contextmanager + from openerp.release import nt_service_name + + def is_running_as_nt_service(): + @contextmanager + def close_srv(srv): + try: + yield srv + finally: + ws.CloseServiceHandle(srv) + + try: + with close_srv(ws.OpenSCManager(None, None, ws.SC_MANAGER_ALL_ACCESS)) as hscm: + with close_srv(wsu.SmartOpenService(hscm, nt_service_name, ws.SERVICE_ALL_ACCESS)) as hs: + info = ws.QueryServiceStatusEx(hs) + return info['ProcessId'] == getppid() + except Exception: + return False + +if __name__ == '__main__': + from pprint import pprint as pp + pp(listdir('../report', True)) diff --git a/odoo/tools/parse_version.py b/odoo/tools/parse_version.py new file mode 100644 index 00000000000..90b1b98e57f --- /dev/null +++ b/odoo/tools/parse_version.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +## this functions are taken from the setuptools package (version 0.6c8) +## http://peak.telecommunity.com/DevCenter/PkgResources#parsing-utilities + +import re + +component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) +replace = {'pre':'c', 'preview':'c','-':'final-','_':'final-','rc':'c','dev':'@','saas':'','~':''}.get + +def _parse_version_parts(s): + for part in component_re.split(s): + part = replace(part,part) + if not part or part=='.': + continue + if part[:1] in '0123456789': + yield part.zfill(8) # pad for numeric comparison + else: + yield '*'+part + + yield '*final' # ensure that alpha/beta/candidate are before final + +def parse_version(s): + """Convert a version string to a chronologically-sortable key + + This is a rough cross between distutils' StrictVersion and LooseVersion; + if you give it versions that would work with StrictVersion, then it behaves + the same; otherwise it acts like a slightly-smarter LooseVersion. It is + *possible* to create pathological version coding schemes that will fool + this parser, but they should be very rare in practice. + + The returned value will be a tuple of strings. Numeric portions of the + version are padded to 8 digits so they will compare numerically, but + without relying on how numbers compare relative to strings. Dots are + dropped, but dashes are retained. Trailing zeros between alpha segments + or dashes are suppressed, so that e.g. "2.4.0" is considered the same as + "2.4". Alphanumeric parts are lower-cased. + + The algorithm assumes that strings like "-" and any alpha string that + alphabetically follows "final" represents a "patch level". So, "2.4-1" + is assumed to be a branch or patch of "2.4", and therefore "2.4.1" is + considered newer than "2.4-1", whic in turn is newer than "2.4". + + Strings like "a", "b", "c", "alpha", "beta", "candidate" and so on (that + come before "final" alphabetically) are assumed to be pre-release versions, + so that the version "2.4" is considered newer than "2.4a1". + + Finally, to handle miscellaneous cases, the strings "pre", "preview", and + "rc" are treated as if they were "c", i.e. as though they were release + candidates, and therefore are not as new as a version string that does not + contain them. + """ + parts = [] + for part in _parse_version_parts((s or '0.1').lower()): + if part.startswith('*'): + if part<'*final': # remove '-' before a prerelease tag + while parts and parts[-1]=='*final-': parts.pop() + # remove trailing zeros from each series of numeric parts + while parts and parts[-1]=='00000000': + parts.pop() + parts.append(part) + return tuple(parts) + +if __name__ == '__main__': + + def cmp(a, b): + msg = '%s < %s == %s' % (a, b, a < b) + assert a < b, msg + return b + + def chk(lst, verbose=False): + pvs = [] + for v in lst: + pv = parse_version(v) + pvs.append(pv) + if verbose: + print v, pv + reduce(cmp, pvs) + + chk(('0', '4.2', '4.2.3.4', '5.0.0-alpha', '5.0.0-rc1', '5.0.0-rc1.1', '5.0.0_rc2', '5.0.0_rc3', '5.0.0'), False) + chk(('5.0.0-0_rc3', '5.0.0-1dev', '5.0.0-1'), False) + diff --git a/odoo/tools/pdf_utils.py b/odoo/tools/pdf_utils.py new file mode 100644 index 00000000000..bcdcdbb63fc --- /dev/null +++ b/odoo/tools/pdf_utils.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +""" Copyright (c) 2003-2007 LOGILAB S.A. (Paris, FRANCE). + http://www.logilab.fr/ -- mailto:contact@logilab.fr + +manipulate pdf and fdf files. pdftk recommended. + +Notes regarding pdftk, pdf forms and fdf files (form definition file) +fields names can be extracted with: + pdftk orig.pdf generate_fdf output truc.fdf +to merge fdf and pdf: + pdftk orig.pdf fill_form test.fdf output result.pdf [flatten] +without flatten, one could further edit the resulting form. +with flatten, everything is turned into text. +""" + +from __future__ import with_statement + +import os +import tempfile + + +HEAD="""%FDF-1.2 +%\xE2\xE3\xCF\xD3 +1 0 obj +<< +/FDF +<< +/Fields [ +""" + +TAIL="""] +>> +>> +endobj +trailer + +<< +/Root 1 0 R +>> +%%EOF +""" + +def output_field(f): + return "\xfe\xff" + "".join( [ "\x00"+c for c in f ] ) + +def extract_keys(lines): + keys = [] + for line in lines: + if line.startswith('/V'): + pass #print 'value',line + elif line.startswith('/T'): + key = line[7:-2] + key = ''.join(key.split('\x00')) + keys.append( key ) + return keys + +def write_field(out, key, value): + out.write("<<\n") + if value: + out.write("/V (%s)\n" %value) + else: + out.write("/V /\n") + out.write("/T (%s)\n" % output_field(key) ) + out.write(">> \n") + +def write_fields(out, fields): + out.write(HEAD) + for key in fields: + value = fields[key] + write_field(out, key, value) +# write_field(out, key+"a", value) # pour copie-carbone sur autres pages + out.write(TAIL) + +def extract_keys_from_pdf(filename): + # what about using 'pdftk filename dump_data_fields' and parsing the output ? + tmp_file = tempfile.mkstemp(".fdf")[1] + try: + os.system('pdftk %s generate_fdf output \"%s\"' % (filename, tmp_file)) + with open(tmp_file, "r") as ofile: + lines = ofile.readlines() + finally: + try: + os.remove(tmp_file) + except Exception: + pass # nothing to do + return extract_keys(lines) + + +def fill_pdf(infile, outfile, fields): + tmp_file = tempfile.mkstemp(".fdf")[1] + try: + with open(tmp_file, "w") as ofile: + write_fields(ofile, fields) + os.system('pdftk %s fill_form \"%s\" output %s flatten' % (infile, tmp_file, outfile)) + finally: + try: + os.remove(tmp_file) + except Exception: + pass # nothing to do + +def testfill_pdf(infile, outfile): + keys = extract_keys_from_pdf(infile) + fields = [] + for key in keys: + fields.append( (key, key, '') ) + fill_pdf(infile, outfile, fields) diff --git a/odoo/tools/safe_eval.py b/odoo/tools/safe_eval.py new file mode 100644 index 00000000000..2b4a0a07bc7 --- /dev/null +++ b/odoo/tools/safe_eval.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +""" +safe_eval module - methods intended to provide more restricted alternatives to + evaluate simple and/or untrusted code. + +Methods in this module are typically used as alternatives to eval() to parse +OpenERP domain strings, conditions and expressions, mostly based on locals +condition/math builtins. +""" + +# Module partially ripped from/inspired by several different sources: +# - http://code.activestate.com/recipes/286134/ +# - safe_eval in lp:~xrg/openobject-server/optimize-5.0 +# - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad + +from opcode import HAVE_ARGUMENT, opmap +from psycopg2 import OperationalError +from types import CodeType +import logging +import werkzeug + +from .misc import ustr + +import openerp + +unsafe_eval = eval + +__all__ = ['test_expr', 'safe_eval', 'const_eval'] + +# The time module is usually already provided in the safe_eval environment +# but some code, e.g. datetime.datetime.now() (Windows/Python 2.5.2, bug +# lp:703841), does import time. +_ALLOWED_MODULES = ['_strptime', 'math', 'time'] + +_UNSAFE_ATTRIBUTES = ['f_builtins', 'f_globals', 'f_locals', 'gi_frame', + 'co_code', 'func_globals'] + +_CONST_OPCODES = set(opmap[x] for x in [ + 'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP', 'DUP_TOPX', + 'POP_BLOCK','SETUP_LOOP', 'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE', + 'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR', 'STORE_MAP'] if x in opmap) + +_EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [ + 'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT', + 'UNARY_INVERT', 'BINARY_POWER', 'BINARY_MULTIPLY', + 'BINARY_DIVIDE', 'BINARY_FLOOR_DIVIDE', 'BINARY_TRUE_DIVIDE', + 'BINARY_MODULO', 'BINARY_ADD', 'BINARY_SUBTRACT', 'BINARY_SUBSCR', + 'BINARY_LSHIFT', 'BINARY_RSHIFT', 'BINARY_AND', 'BINARY_XOR', + 'BINARY_OR', 'INPLACE_ADD', 'INPLACE_SUBTRACT', 'INPLACE_MULTIPLY', + 'INPLACE_DIVIDE', 'INPLACE_REMAINDER', 'INPLACE_POWER', + 'INPLACE_LEFTSHIFT', 'INPLACE_RIGHTSHIFT', 'INPLACE_AND', + 'INPLACE_XOR','INPLACE_OR' +] if x in opmap)) + +_SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [ + 'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR', + 'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'DELETE_NAME', + 'JUMP_FORWARD', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE', 'JUMP_ABSOLUTE', + 'MAKE_FUNCTION', 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BREAK_LOOP', + 'CONTINUE_LOOP', 'RAISE_VARARGS', 'YIELD_VALUE', + # New in Python 2.7 - http://bugs.python.org/issue4715 : + 'JUMP_IF_FALSE_OR_POP', 'JUMP_IF_TRUE_OR_POP', 'POP_JUMP_IF_FALSE', + 'POP_JUMP_IF_TRUE', 'SETUP_EXCEPT', 'END_FINALLY', + 'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'UNPACK_SEQUENCE', + 'LOAD_GLOBAL', # Only allows access to restricted globals +] if x in opmap)) + +_logger = logging.getLogger(__name__) + +def _get_opcodes(codeobj): + """_get_opcodes(codeobj) -> [opcodes] + + Extract the actual opcodes as a list from a code object + + >>> c = compile("[1 + 2, (1,2)]", "", "eval") + >>> _get_opcodes(c) + [100, 100, 23, 100, 100, 102, 103, 83] + """ + i = 0 + byte_codes = codeobj.co_code + while i < len(byte_codes): + code = ord(byte_codes[i]) + yield code + + if code >= HAVE_ARGUMENT: + i += 3 + else: + i += 1 + +def assert_no_dunder_name(code_obj, expr): + """ assert_no_dunder_name(code_obj, expr) -> None + + Asserts that the code object does not refer to any "dunder name" + (__$name__), so that safe_eval prevents access to any internal-ish Python + attribute or method (both are loaded via LOAD_ATTR which uses a name, not a + const or a var). + + Checks that no such name exists in the provided code object (co_names). + + :param code_obj: code object to name-validate + :type code_obj: CodeType + :param str expr: expression corresponding to the code object, for debugging + purposes + :raises NameError: in case a forbidden name (containing two underscores) + is found in ``code_obj`` + + .. note:: actually forbids every name containing 2 underscores + """ + for name in code_obj.co_names: + if "__" in name or name in _UNSAFE_ATTRIBUTES: + raise NameError('Access to forbidden name %r (%r)' % (name, expr)) + +def assert_valid_codeobj(allowed_codes, code_obj, expr): + """ Asserts that the provided code object validates against the bytecode + and name constraints. + + Recursively validates the code objects stored in its co_consts in case + lambdas are being created/used (lambdas generate their own separated code + objects and don't live in the root one) + + :param allowed_codes: list of permissible bytecode instructions + :type allowed_codes: set(int) + :param code_obj: code object to name-validate + :type code_obj: CodeType + :param str expr: expression corresponding to the code object, for debugging + purposes + :raises ValueError: in case of forbidden bytecode in ``code_obj`` + :raises NameError: in case a forbidden name (containing two underscores) + is found in ``code_obj`` + """ + assert_no_dunder_name(code_obj, expr) + + # almost twice as fast as a manual iteration + condition when loading + # /web according to line_profiler + if set(_get_opcodes(code_obj)) - allowed_codes: + raise ValueError("forbidden opcode(s) in %r" % expr) + + for const in code_obj.co_consts: + if isinstance(const, CodeType): + assert_valid_codeobj(allowed_codes, const, 'lambda') + +def test_expr(expr, allowed_codes, mode="eval"): + """test_expr(expression, allowed_codes[, mode]) -> code_object + + Test that the expression contains only the allowed opcodes. + If the expression is valid and contains only allowed codes, + return the compiled code object. + Otherwise raise a ValueError, a Syntax Error or TypeError accordingly. + """ + try: + if mode == 'eval': + # eval() does not like leading/trailing whitespace + expr = expr.strip() + code_obj = compile(expr, "", mode) + except (SyntaxError, TypeError, ValueError): + raise + except Exception, e: + import sys + exc_info = sys.exc_info() + raise ValueError, '"%s" while compiling\n%r' % (ustr(e), expr), exc_info[2] + assert_valid_codeobj(allowed_codes, code_obj, expr) + return code_obj + + +def const_eval(expr): + """const_eval(expression) -> value + + Safe Python constant evaluation + + Evaluates a string that contains an expression describing + a Python constant. Strings that are not valid Python expressions + or that contain other code besides the constant raise ValueError. + + >>> const_eval("10") + 10 + >>> const_eval("[1,2, (3,4), {'foo':'bar'}]") + [1, 2, (3, 4), {'foo': 'bar'}] + >>> const_eval("1+2") + Traceback (most recent call last): + ... + ValueError: opcode BINARY_ADD not allowed + """ + c = test_expr(expr, _CONST_OPCODES) + return unsafe_eval(c) + +def expr_eval(expr): + """expr_eval(expression) -> value + + Restricted Python expression evaluation + + Evaluates a string that contains an expression that only + uses Python constants. This can be used to e.g. evaluate + a numerical expression from an untrusted source. + + >>> expr_eval("1+2") + 3 + >>> expr_eval("[1,2]*2") + [1, 2, 1, 2] + >>> expr_eval("__import__('sys').modules") + Traceback (most recent call last): + ... + ValueError: opcode LOAD_NAME not allowed + """ + c = test_expr(expr, _EXPR_OPCODES) + return unsafe_eval(c) + +def _import(name, globals=None, locals=None, fromlist=None, level=-1): + if globals is None: + globals = {} + if locals is None: + locals = {} + if fromlist is None: + fromlist = [] + if name in _ALLOWED_MODULES: + return __import__(name, globals, locals, level) + raise ImportError(name) +_BUILTINS = { + '__import__': _import, + 'True': True, + 'False': False, + 'None': None, + 'str': str, + 'unicode': unicode, + 'bool': bool, + 'int': int, + 'float': float, + 'long': long, + 'enumerate': enumerate, + 'dict': dict, + 'list': list, + 'tuple': tuple, + 'map': map, + 'abs': abs, + 'min': min, + 'max': max, + 'sum': sum, + 'reduce': reduce, + 'filter': filter, + 'round': round, + 'len': len, + 'repr': repr, + 'set': set, + 'all': all, + 'any': any, + 'ord': ord, + 'chr': chr, + 'cmp': cmp, + 'divmod': divmod, + 'isinstance': isinstance, + 'range': range, + 'xrange': xrange, + 'zip': zip, + 'Exception': Exception, +} +def safe_eval(expr, globals_dict=None, locals_dict=None, mode="eval", nocopy=False, locals_builtins=False): + """safe_eval(expression[, globals[, locals[, mode[, nocopy]]]]) -> result + + System-restricted Python expression evaluation + + Evaluates a string that contains an expression that mostly + uses Python constants, arithmetic expressions and the + objects directly provided in context. + + This can be used to e.g. evaluate + an OpenERP domain expression from an untrusted source. + + :throws TypeError: If the expression provided is a code object + :throws SyntaxError: If the expression provided is not valid Python + :throws NameError: If the expression provided accesses forbidden names + :throws ValueError: If the expression provided uses forbidden bytecode + """ + if type(expr) is CodeType: + raise TypeError("safe_eval does not allow direct evaluation of code objects.") + + # prevent altering the globals/locals from within the sandbox + # by taking a copy. + if not nocopy: + # isinstance() does not work below, we want *exactly* the dict class + if (globals_dict is not None and type(globals_dict) is not dict) \ + or (locals_dict is not None and type(locals_dict) is not dict): + _logger.warning( + "Looks like you are trying to pass a dynamic environment, " + "you should probably pass nocopy=True to safe_eval().") + if globals_dict is not None: + globals_dict = dict(globals_dict) + if locals_dict is not None: + locals_dict = dict(locals_dict) + + if globals_dict is None: + globals_dict = {} + + globals_dict['__builtins__'] = _BUILTINS + if locals_builtins: + if locals_dict is None: + locals_dict = {} + locals_dict.update(_BUILTINS) + c = test_expr(expr, _SAFE_OPCODES, mode=mode) + try: + return unsafe_eval(c, globals_dict, locals_dict) + except openerp.exceptions.except_orm: + raise + except openerp.exceptions.Warning: + raise + except openerp.exceptions.RedirectWarning: + raise + except openerp.exceptions.AccessDenied: + raise + except openerp.exceptions.AccessError: + raise + except werkzeug.exceptions.HTTPException: + raise + except openerp.http.AuthenticationError: + raise + except OperationalError: + # Do not hide PostgreSQL low-level exceptions, to let the auto-replay + # of serialized transactions work its magic + raise + except openerp.exceptions.MissingError: + raise + except Exception, e: + import sys + exc_info = sys.exc_info() + raise ValueError, '%s: "%s" while evaluating\n%r' % (ustr(type(e)), ustr(e), expr), exc_info[2] +def test_python_expr(expr, mode="eval"): + try: + test_expr(expr, _SAFE_OPCODES, mode=mode) + except (SyntaxError, TypeError, ValueError) as err: + if len(err.args) >= 2 and len(err.args[1]) >= 4: + error = { + 'message': err.args[0], + 'filename': err.args[1][0], + 'lineno': err.args[1][1], + 'offset': err.args[1][2], + 'error_line': err.args[1][3], + } + msg = "%s : %s at line %d\n%s" % (type(err).__name__, error['message'], error['lineno'], error['error_line']) + else: + msg = ustr(err) + return msg + return False diff --git a/odoo/tools/sql.py b/odoo/tools/sql.py new file mode 100644 index 00000000000..43c54c1a28a --- /dev/null +++ b/odoo/tools/sql.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +def drop_view_if_exists(cr, viewname): + cr.execute("DROP view IF EXISTS %s CASCADE" % (viewname,)) + cr.commit() + +def escape_psql(to_escape): + return to_escape.replace('\\', r'\\').replace('%', '\%').replace('_', '\_') + +def pg_varchar(size=0): + """ Returns the VARCHAR declaration for the provided size: + + * If no size (or an empty or negative size is provided) return an + 'infinite' VARCHAR + * Otherwise return a VARCHAR(n) + + :type int size: varchar size, optional + :rtype: str + """ + if size: + if not isinstance(size, int): + raise ValueError("VARCHAR parameter should be an int, got %s" % type(size)) + if size > 0: + return 'VARCHAR(%d)' % size + return 'VARCHAR' diff --git a/odoo/tools/test-config-values b/odoo/tools/test-config-values new file mode 100755 index 00000000000..43991e8a21d --- /dev/null +++ b/odoo/tools/test-config-values @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import logging +import os +import signal +import sys +import threading +import traceback +import time + +import openerp + +openerp.tools.config.parse_config(sys.argv[1:]) +config = openerp.tools.config + +for name in [ + 'db_name', + 'addons_path', + 'demo', + 'osv_memory_count_limit', + 'osv_memory_age_limit', + ]: + print "%s: %s - %s" % (name, config[name], type(config[name])) + diff --git a/odoo/tools/test-config-values-00.conf b/odoo/tools/test-config-values-00.conf new file mode 100644 index 00000000000..1d0bd7419fe --- /dev/null +++ b/odoo/tools/test-config-values-00.conf @@ -0,0 +1,3 @@ +[options] +osv_memory_count_limit = 5 +osv_memory_age_limit = 3.4 diff --git a/odoo/tools/test_config.py b/odoo/tools/test_config.py new file mode 100644 index 00000000000..256e7ba4c32 --- /dev/null +++ b/odoo/tools/test_config.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +""" Tests for the configuration file/command-line arguments. """ + +# This test should be run from its directory. + +# TODO A configmanager object cannot parse multiple times a config file +# and/or the command line, preventing to 'reload' a configuration. + +import os + +import config + +config_file_00 = os.path.join(os.path.dirname(__file__),'test-config-values-00.conf') + +# 1. No config file, no command-line arguments (a.k.a. default values) + +conf = config.configmanager() +conf.parse_config() + +assert conf['osv_memory_age_limit'] == 1.0 +assert os.path.join(conf['root_path'], 'addons') == conf['addons_path'] + +# 2. No config file, some command-line arguments + +conf = config.configmanager() +# mess with the optparse.Option definition to allow an invalid path +conf.casts['addons_path'].action = 'store' +conf.parse_config(['--addons-path=/xyz/dont-exist', '--osv-memory-age-limit=2.3']) + +assert conf['osv_memory_age_limit'] == 2.3 +assert conf['addons_path'] == '/xyz/dont-exist' + +# 3. Config file, no command-line arguments + +conf = config.configmanager() +conf.parse_config(['-c', config_file_00]) + +assert conf['osv_memory_age_limit'] == 3.4 + +# 4. Config file, and command-line arguments + +conf = config.configmanager() +conf.parse_config(['-c', config_file_00, '--osv-memory-age-limit=2.3']) + +assert conf['osv_memory_age_limit'] == 2.3 diff --git a/odoo/tools/test_reports.py b/odoo/tools/test_reports.py new file mode 100644 index 00000000000..ab924b5b4d1 --- /dev/null +++ b/odoo/tools/test_reports.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +""" Helper functions for reports testing. + + Please /do not/ import this file by default, but only explicitly call it + through the code of yaml tests. +""" + +import openerp +import openerp.report +import openerp.tools as tools +import logging +from openerp.tools.safe_eval import safe_eval +from subprocess import Popen, PIPE +import os +import tempfile + +_logger = logging.getLogger(__name__) +_test_logger = logging.getLogger('openerp.tests') + + +def try_report(cr, uid, rname, ids, data=None, context=None, our_module=None, report_type=None): + """ Try to render a report <rname> with contents of ids + + This function should also check for common pitfalls of reports. + """ + if data is None: + data = {} + if context is None: + context = {} + if rname.startswith('report.'): + rname_s = rname[7:] + else: + rname_s = rname + _test_logger.info(" - Trying %s.create(%r)", rname, ids) + + res = openerp.report.render_report(cr, uid, ids, rname_s, data, context=context) + if not isinstance(res, tuple): + raise RuntimeError("Result of %s.create() should be a (data,format) tuple, now it is a %s" % \ + (rname, type(res))) + (res_data, res_format) = res + + if not res_data: + raise ValueError("Report %s produced an empty result!" % rname) + + if tools.config['test_report_directory']: + file(os.path.join(tools.config['test_report_directory'], rname+ '.'+res_format), 'wb+').write(res_data) + + _logger.debug("Have a %s report for %s, will examine it", res_format, rname) + if res_format == 'pdf': + if res_data[:5] != '%PDF-': + raise ValueError("Report %s produced a non-pdf header, %r" % (rname, res_data[:10])) + + res_text = False + try: + fd, rfname = tempfile.mkstemp(suffix=res_format) + os.write(fd, res_data) + os.close(fd) + + proc = Popen(['pdftotext', '-enc', 'UTF-8', '-nopgbrk', rfname, '-'], shell=False, stdout=PIPE) + stdout, stderr = proc.communicate() + res_text = tools.ustr(stdout) + os.unlink(rfname) + except Exception: + _logger.debug("Unable to parse PDF report: install pdftotext to perform automated tests.") + + if res_text is not False: + for line in res_text.split('\n'): + if ('[[' in line) or ('[ [' in line): + _logger.error("Report %s may have bad expression near: \"%s\".", rname, line[80:]) + # TODO more checks, what else can be a sign of a faulty report? + elif res_format == 'html': + pass + else: + _logger.warning("Report %s produced a \"%s\" chunk, cannot examine it", rname, res_format) + return False + + _test_logger.info(" + Report %s produced correctly.", rname) + return True + +def try_report_action(cr, uid, action_id, active_model=None, active_ids=None, + wiz_data=None, wiz_buttons=None, + context=None, our_module=None): + """Take an ir.action.act_window and follow it until a report is produced + + :param action_id: the integer id of an action, or a reference to xml id + of the act_window (can search [our_module.]+xml_id + :param active_model, active_ids: call the action as if it had been launched + from that model+ids (tree/form view action) + :param wiz_data: a dictionary of values to use in the wizard, if needed. + They will override (or complete) the default values of the + wizard form. + :param wiz_buttons: a list of button names, or button icon strings, which + should be preferred to press during the wizard. + Eg. 'OK' or 'fa-print' + :param our_module: the name of the calling module (string), like 'account' + """ + if not our_module and isinstance(action_id, basestring): + if '.' in action_id: + our_module = action_id.split('.', 1)[0] + + context = dict(context or {}) + # TODO context fill-up + + env = openerp.api.Environment(cr, uid, context) + + def log_test(msg, *args): + _test_logger.info(" - " + msg, *args) + + datas = {} + if active_model: + datas['model'] = active_model + if active_ids: + datas['ids'] = active_ids + + if not wiz_buttons: + wiz_buttons = [] + + if isinstance(action_id, basestring): + if '.' in action_id: + _, act_xmlid = action_id.split('.', 1) + else: + if not our_module: + raise ValueError('You cannot only specify action_id "%s" without a module name' % action_id) + act_xmlid = action_id + action_id = '%s.%s' % (our_module, action_id) + action = env.ref(action_id) + act_model, act_id = action._name, action.id + else: + assert isinstance(action_id, (long, int)) + act_model = 'ir.action.act_window' # assume that + act_id = action_id + act_xmlid = '<%s>' % act_id + + def _exec_action(action, datas, env): + # taken from client/modules/action/main.py:84 _exec_action() + if isinstance(action, bool) or 'type' not in action: + return + # Updating the context : Adding the context of action in order to use it on Views called from buttons + context = dict(env.context) + if datas.get('id',False): + context.update( {'active_id': datas.get('id',False), 'active_ids': datas.get('ids',[]), 'active_model': datas.get('model',False)}) + context1 = action.get('context', {}) + if isinstance(context1, basestring): + context1 = safe_eval(context1, dict(context)) + context.update(context1) + env = env(context=context) + if action['type'] in ['ir.actions.act_window', 'ir.actions.submenu']: + for key in ('res_id', 'res_model', 'view_type', 'view_mode', + 'limit', 'search_view', 'auto_search', 'search_view_id'): + datas[key] = action.get(key, datas.get(key, None)) + + view_id = False + if action.get('views', []): + if isinstance(action['views'],list): + view_id = action['views'][0][0] + datas['view_mode']= action['views'][0][1] + else: + if action.get('view_id', False): + view_id = action['view_id'][0] + elif action.get('view_id', False): + view_id = action['view_id'][0] + + assert datas['res_model'], "Cannot use the view without a model" + # Here, we have a view that we need to emulate + log_test("will emulate a %s view: %s#%s", + action['view_type'], datas['res_model'], view_id or '?') + + view_res = env[datas['res_model']].fields_view_get(view_id, action['view_type']) + assert view_res and view_res.get('arch'), "Did not return any arch for the view" + view_data = {} + if view_res.get('fields'): + view_data = env[datas['res_model']].default_get(list(view_res['fields'])) + if datas.get('form'): + view_data.update(datas.get('form')) + if wiz_data: + view_data.update(wiz_data) + _logger.debug("View data is: %r", view_data) + + for fk, field in view_res.get('fields',{}).items(): + # Default fields returns list of int, while at create() + # we need to send a [(6,0,[int,..])] + if field['type'] in ('one2many', 'many2many') \ + and view_data.get(fk, False) \ + and isinstance(view_data[fk], list) \ + and not isinstance(view_data[fk][0], tuple) : + view_data[fk] = [(6, 0, view_data[fk])] + + action_name = action.get('name') + try: + from xml.dom import minidom + cancel_found = False + buttons = [] + dom_doc = minidom.parseString(view_res['arch']) + if not action_name: + action_name = dom_doc.documentElement.getAttribute('name') + + for button in dom_doc.getElementsByTagName('button'): + button_weight = 0 + if button.getAttribute('special') == 'cancel': + cancel_found = True + continue + if button.getAttribute('icon') == 'fa-times-circle': + cancel_found = True + continue + if button.getAttribute('default_focus') == '1': + button_weight += 20 + if button.getAttribute('string') in wiz_buttons: + button_weight += 30 + elif button.getAttribute('icon') in wiz_buttons: + button_weight += 10 + string = button.getAttribute('string') or '?%s' % len(buttons) + + buttons.append({ + 'name': button.getAttribute('name'), + 'string': string, + 'type': button.getAttribute('type'), + 'weight': button_weight, + }) + except Exception, e: + _logger.warning("Cannot resolve the view arch and locate the buttons!", exc_info=True) + raise AssertionError(e.args[0]) + + if not datas['res_id']: + # it is probably an orm_memory object, we need to create + # an instance + datas['res_id'] = env[datas['res_model']].create(view_data).id + + if not buttons: + raise AssertionError("view form doesn't have any buttons to press!") + + buttons.sort(key=lambda b: b['weight']) + _logger.debug('Buttons are: %s', ', '.join([ '%s: %d' % (b['string'], b['weight']) for b in buttons])) + + res = None + while buttons and not res: + b = buttons.pop() + log_test("in the \"%s\" form, I will press the \"%s\" button.", action_name, b['string']) + if not b['type']: + log_test("the \"%s\" button has no type, cannot use it", b['string']) + continue + if b['type'] == 'object': + #there we are! press the button! + rec = env[datas['res_model']].browse(datas['res_id']) + func = getattr(rec, b['name'], None) + if not func: + _logger.error("The %s model doesn't have a %s attribute!", datas['res_model'], b['name']) + continue + res = func() + break + else: + _logger.warning("in the \"%s\" form, the \"%s\" button has unknown type %s", + action_name, b['string'], b['type']) + return res + + elif action['type']=='ir.actions.report.xml': + if 'window' in datas: + del datas['window'] + if not datas: + datas = action.get('datas') + if not datas: + datas = action.get('data') + datas = datas.copy() + ids = datas.get('ids') + if 'ids' in datas: + del datas['ids'] + res = try_report(cr, uid, 'report.'+action['report_name'], ids, datas, context, our_module=our_module) + return res + else: + raise Exception("Cannot handle action of type %s" % act_model) + + log_test("will be using %s action %s #%d", act_model, act_xmlid, act_id) + action = env[act_model].browse(act_id).read()[0] + assert action, "Could not read action %s[%s]" % (act_model, act_id) + loop = 0 + while action: + loop += 1 + # This part tries to emulate the loop of the Gtk client + if loop > 100: + _logger.info("Passed %d loops, giving up", loop) + raise Exception("Too many loops at action") + log_test("it is an %s action at loop #%d", action.get('type', 'unknown'), loop) + result = _exec_action(action, datas, env) + if not isinstance(result, dict): + break + datas = result.get('datas', {}) + if datas: + del result['datas'] + action = result + + return True diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py new file mode 100644 index 00000000000..07531254cf1 --- /dev/null +++ b/odoo/tools/translate.py @@ -0,0 +1,1197 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import codecs +import csv +import fnmatch +import inspect +import locale +import logging +import os +import re +import tarfile +import tempfile +import threading +from collections import defaultdict +from datetime import datetime +from os.path import join +from xml.sax.saxutils import escape + +from babel.messages import extract +from lxml import etree + +import openerp +from openerp.tools import config +from openerp.tools.misc import file_open, get_iso_codes, SKIPPED_ELEMENT_TYPES +from openerp.tools.osutil import walksymlinks +from openerp import sql_db, SUPERUSER_ID + +_logger = logging.getLogger(__name__) + +# used to notify web client that these translations should be loaded in the UI +WEB_TRANSLATION_COMMENT = "openerp-web" + +SKIPPED_ELEMENTS = ('script', 'style', 'title') + +_LOCALE2WIN32 = { + 'af_ZA': 'Afrikaans_South Africa', + 'sq_AL': 'Albanian_Albania', + 'ar_SA': 'Arabic_Saudi Arabia', + 'eu_ES': 'Basque_Spain', + 'be_BY': 'Belarusian_Belarus', + 'bs_BA': 'Bosnian_Bosnia and Herzegovina', + 'bg_BG': 'Bulgarian_Bulgaria', + 'ca_ES': 'Catalan_Spain', + 'hr_HR': 'Croatian_Croatia', + 'zh_CN': 'Chinese_China', + 'zh_TW': 'Chinese_Taiwan', + 'cs_CZ': 'Czech_Czech Republic', + 'da_DK': 'Danish_Denmark', + 'nl_NL': 'Dutch_Netherlands', + 'et_EE': 'Estonian_Estonia', + 'fa_IR': 'Farsi_Iran', + 'ph_PH': 'Filipino_Philippines', + 'fi_FI': 'Finnish_Finland', + 'fr_FR': 'French_France', + 'fr_BE': 'French_France', + 'fr_CH': 'French_France', + 'fr_CA': 'French_France', + 'ga': 'Scottish Gaelic', + 'gl_ES': 'Galician_Spain', + 'ka_GE': 'Georgian_Georgia', + 'de_DE': 'German_Germany', + 'el_GR': 'Greek_Greece', + 'gu': 'Gujarati_India', + 'he_IL': 'Hebrew_Israel', + 'hi_IN': 'Hindi', + 'hu': 'Hungarian_Hungary', + 'is_IS': 'Icelandic_Iceland', + 'id_ID': 'Indonesian_indonesia', + 'it_IT': 'Italian_Italy', + 'ja_JP': 'Japanese_Japan', + 'kn_IN': 'Kannada', + 'km_KH': 'Khmer', + 'ko_KR': 'Korean_Korea', + 'lo_LA': 'Lao_Laos', + 'lt_LT': 'Lithuanian_Lithuania', + 'lat': 'Latvian_Latvia', + 'ml_IN': 'Malayalam_India', + 'mi_NZ': 'Maori', + 'mn': 'Cyrillic_Mongolian', + 'no_NO': 'Norwegian_Norway', + 'nn_NO': 'Norwegian-Nynorsk_Norway', + 'pl': 'Polish_Poland', + 'pt_PT': 'Portuguese_Portugal', + 'pt_BR': 'Portuguese_Brazil', + 'ro_RO': 'Romanian_Romania', + 'ru_RU': 'Russian_Russia', + 'sr_CS': 'Serbian (Cyrillic)_Serbia and Montenegro', + 'sk_SK': 'Slovak_Slovakia', + 'sl_SI': 'Slovenian_Slovenia', + #should find more specific locales for spanish countries, + #but better than nothing + 'es_AR': 'Spanish_Spain', + 'es_BO': 'Spanish_Spain', + 'es_CL': 'Spanish_Spain', + 'es_CO': 'Spanish_Spain', + 'es_CR': 'Spanish_Spain', + 'es_DO': 'Spanish_Spain', + 'es_EC': 'Spanish_Spain', + 'es_ES': 'Spanish_Spain', + 'es_GT': 'Spanish_Spain', + 'es_HN': 'Spanish_Spain', + 'es_MX': 'Spanish_Spain', + 'es_NI': 'Spanish_Spain', + 'es_PA': 'Spanish_Spain', + 'es_PE': 'Spanish_Spain', + 'es_PR': 'Spanish_Spain', + 'es_PY': 'Spanish_Spain', + 'es_SV': 'Spanish_Spain', + 'es_UY': 'Spanish_Spain', + 'es_VE': 'Spanish_Spain', + 'sv_SE': 'Swedish_Sweden', + 'ta_IN': 'English_Australia', + 'th_TH': 'Thai_Thailand', + 'tr_TR': 'Turkish_Turkey', + 'uk_UA': 'Ukrainian_Ukraine', + 'vi_VN': 'Vietnamese_Viet Nam', + 'tlh_TLH': 'Klingon', + +} + +# These are not all english small words, just those that could potentially be isolated within views +ENGLISH_SMALL_WORDS = set("as at by do go if in me no of ok on or to up us we".split()) + + +class UNIX_LINE_TERMINATOR(csv.excel): + lineterminator = '\n' + +csv.register_dialect("UNIX", UNIX_LINE_TERMINATOR) + + +# +# Helper functions for translating fields +# +def encode(s): + if isinstance(s, unicode): + return s.encode('utf8') + return s + +# which elements are translated inline +TRANSLATED_ELEMENTS = { + 'abbr', 'b', 'bdi', 'bdo', 'br', 'cite', 'code', 'data', 'del', 'dfn', 'em', + 'font', 'i', 'ins', 'kbd', 'keygen', 'mark', 'math', 'meter', 'output', + 'progress', 'q', 'ruby', 's', 'samp', 'small', 'span', 'strong', 'sub', + 'sup', 'time', 'u', 'var', 'wbr', 'text', +} + +# which attributes must be translated +TRANSLATED_ATTRS = { + 'string', 'help', 'sum', 'avg', 'confirm', 'placeholder', 'alt', 'title', +} + +avoid_pattern = re.compile(r"[\s\n]*<!DOCTYPE", re.IGNORECASE) + +class XMLTranslator(object): + """ A sequence of serialized XML/HTML items, with some of them to translate + (todo) and others already translated (done). The purpose of this object + is to simplify the handling of phrasing elements (like <b>) that must be + translated together with their surrounding text. + + For instance, the content of the "div" element below will be translated + as a whole (without surrounding spaces): + + <div> + Lorem ipsum dolor sit amet, consectetur adipiscing elit, + <b>sed</b> do eiusmod tempor incididunt ut labore et dolore + magna aliqua. <span class="more">Ut enim ad minim veniam, + <em>quis nostrud exercitation</em> ullamco laboris nisi ut + aliquip ex ea commodo consequat.</span> + </div> + + """ + def __init__(self, callback, method, parser=None): + self.callback = callback # callback function to translate terms + self.method = method # serialization method ('xml' or 'html') + self.parser = parser # parser for validating translations + self._done = [] # translated strings + self._todo = [] # todo strings that come after _done + self.needs_trans = False # whether todo needs translation + + def todo(self, text, needs_trans=True): + self._todo.append(text) + if needs_trans and text.strip(): + self.needs_trans = True + + def all_todo(self): + return not self._done + + def get_todo(self): + return "".join(self._todo) + + def flush(self): + if self._todo: + todo = "".join(self._todo) + done = self.process_text(todo) if self.needs_trans else todo + self._done.append(done) + del self._todo[:] + self.needs_trans = False + + def done(self, text): + self.flush() + self._done.append(text) + + def get_done(self): + """ Complete the translations and return the result. """ + self.flush() + return "".join(self._done) + + def process_text(self, text): + """ Translate text.strip(), but keep the surrounding spaces from text. """ + term = text.strip() + trans = term and self.callback(term) + if trans: + try: + # parse the translation to validate it + etree.fromstring("<div>%s</div>" % encode(trans), parser=self.parser) + except etree.ParseError: + # fallback: escape the translation + trans = escape(trans) + text = text.replace(term, trans) + return text + + def process_attr(self, attr): + """ Translate the given node attribute value. """ + term = attr.strip() + trans = term and self.callback(term) + return attr.replace(term, trans) if trans else attr + + def process(self, node): + """ Process the given xml `node`: collect `todo` and `done` items. """ + if ( + isinstance(node, SKIPPED_ELEMENT_TYPES) or + node.tag in SKIPPED_ELEMENTS or + node.get("t-translation", "").strip() == "off" or + node.tag == "attribute" and node.get("name") not in TRANSLATED_ATTRS or + node.getparent() is None and node.text and '<!DOCTYPE' in node.text + ): + # do not translate the contents of the node + tail, node.tail = node.tail, None + self.done(etree.tostring(node, method=self.method)) + self.todo(escape(tail or "")) + return + + # process children nodes locally in child_trans + child_trans = XMLTranslator(self.callback, self.method, parser=self.parser) + if node.text: + if avoid_pattern.match(node.text): + child_trans.done(escape(node.text)) # do not translate <!DOCTYPE... + else: + child_trans.todo(escape(node.text)) + for child in node: + child_trans.process(child) + + if (child_trans.all_todo() and + node.tag in TRANSLATED_ELEMENTS and + not any(attr.startswith("t-") for attr in node.attrib)): + # serialize the node element as todo + self.todo(self.serialize(node.tag, node.attrib, child_trans.get_todo()), + child_trans.needs_trans) + else: + # complete translations and serialize result as done + for attr in TRANSLATED_ATTRS: + if node.get(attr): + node.set(attr, self.process_attr(node.get(attr))) + self.done(self.serialize(node.tag, node.attrib, child_trans.get_done())) + + # add node tail as todo + self.todo(escape(node.tail or "")) + + def serialize(self, tag, attrib, content): + """ Return a serialized element with the given `tag`, attributes + `attrib`, and already-serialized `content`. + """ + if content: + elem = etree.tostring(etree.Element(tag, attrib), method='xml') + assert elem.endswith("/>") + return "%s>%s</%s>" % (elem[:-2], content, tag) + else: + return etree.tostring(etree.Element(tag, attrib), method=self.method) + + +def xml_translate(callback, value): + """ Translate an XML value (string), using `callback` for translating text + appearing in `value`. + """ + if not value: + return value + + trans = XMLTranslator(callback, 'xml') + try: + root = etree.fromstring(encode(value)) + trans.process(root) + return trans.get_done() + except etree.ParseError: + # fallback for translated terms: use an HTML parser and wrap the term + wrapped = "<div>%s</div>" % encode(value) + root = etree.fromstring(wrapped, etree.HTMLParser(encoding='utf-8')) + trans.process(root[0][0]) # html > body > div + return trans.get_done()[5:-6] # remove tags <div> and </div> + +def html_translate(callback, value): + """ Translate an HTML value (string), using `callback` for translating text + appearing in `value`. + """ + if not value: + return value + + parser = etree.HTMLParser(encoding='utf-8') + trans = XMLTranslator(callback, 'html', parser) + wrapped = "<div>%s</div>" % encode(value) + root = etree.fromstring(wrapped, parser) + trans.process(root[0][0]) # html > body > div + return trans.get_done()[5:-6] # remove tags <div> and </div> + + +# +# Warning: better use self.env['ir.translation']._get_source if you can +# +def translate(cr, name, source_type, lang, source=None): + if source and name: + cr.execute('select value from ir_translation where lang=%s and type=%s and name=%s and src=%s and md5(src)=md5(%s)', (lang, source_type, str(name), source, source)) + elif name: + cr.execute('select value from ir_translation where lang=%s and type=%s and name=%s', (lang, source_type, str(name))) + elif source: + cr.execute('select value from ir_translation where lang=%s and type=%s and src=%s and md5(src)=md5(%s)', (lang, source_type, source, source)) + res_trans = cr.fetchone() + res = res_trans and res_trans[0] or False + return res + +class GettextAlias(object): + + def _get_db(self): + # find current DB based on thread/worker db name (see netsvc) + db_name = getattr(threading.currentThread(), 'dbname', None) + if db_name: + return sql_db.db_connect(db_name) + + def _get_cr(self, frame, allow_create=True): + # try, in order: cr, cursor, self.env.cr, self.cr, + # request.env.cr + if 'cr' in frame.f_locals: + return frame.f_locals['cr'], False + if 'cursor' in frame.f_locals: + return frame.f_locals['cursor'], False + s = frame.f_locals.get('self') + if hasattr(s, 'env'): + return s.env.cr, False + if hasattr(s, 'cr'): + return s.cr, False + try: + from openerp.http import request + return request.env.cr, False + except RuntimeError: + pass + if allow_create: + # create a new cursor + db = self._get_db() + if db is not None: + return db.cursor(), True + return None, False + + def _get_uid(self, frame): + # try, in order: uid, user, self.env.uid + if 'uid' in frame.f_locals: + return frame.f_locals['uid'] + if 'user' in frame.f_locals: + return int(frame.f_locals['user']) # user may be a record + s = frame.f_locals.get('self') + return s.env.uid + + def _get_lang(self, frame): + # try, in order: context.get('lang'), kwargs['context'].get('lang'), + # self.env.lang, self.localcontext.get('lang'), request.env.lang + lang = None + if frame.f_locals.get('context'): + lang = frame.f_locals['context'].get('lang') + if not lang: + kwargs = frame.f_locals.get('kwargs', {}) + if kwargs.get('context'): + lang = kwargs['context'].get('lang') + if not lang: + s = frame.f_locals.get('self') + if hasattr(s, 'env'): + lang = s.env.lang + if not lang: + if hasattr(s, 'localcontext'): + lang = s.localcontext.get('lang') + if not lang: + try: + from openerp.http import request + lang = request.env.lang + except RuntimeError: + pass + if not lang: + # Last resort: attempt to guess the language of the user + # Pitfall: some operations are performed in sudo mode, and we + # don't know the originial uid, so the language may + # be wrong when the admin language differs. + (cr, dummy) = self._get_cr(frame, allow_create=False) + uid = self._get_uid(frame) + if cr and uid: + env = openerp.api.Environment(cr, uid, {}) + lang = env['res.users'].context_get()['lang'] + return lang + + def __call__(self, source): + res = source + cr = None + is_new_cr = False + try: + frame = inspect.currentframe() + if frame is None: + return source + frame = frame.f_back + if not frame: + return source + lang = self._get_lang(frame) + if lang: + cr, is_new_cr = self._get_cr(frame) + if cr: + # Try to use ir.translation to benefit from global cache if possible + env = openerp.api.Environment(cr, SUPERUSER_ID, {}) + res = env['ir.translation']._get_source(None, ('code','sql_constraint'), lang, source) + else: + _logger.debug('no context cursor detected, skipping translation for "%r"', source) + else: + _logger.debug('no translation language detected, skipping translation for "%r" ', source) + except Exception: + _logger.debug('translation went wrong for "%r", skipped', source) + # if so, double-check the root/base translations filenames + finally: + if cr and is_new_cr: + cr.close() + return res + +_ = GettextAlias() + + +def quote(s): + """Returns quoted PO term string, with special PO characters escaped""" + assert r"\n" not in s, "Translation terms may not include escaped newlines ('\\n'), please use only literal newlines! (in '%s')" % s + return '"%s"' % s.replace('\\','\\\\') \ + .replace('"','\\"') \ + .replace('\n', '\\n"\n"') + +re_escaped_char = re.compile(r"(\\.)") +re_escaped_replacements = {'n': '\n', } + +def _sub_replacement(match_obj): + return re_escaped_replacements.get(match_obj.group(1)[1], match_obj.group(1)[1]) + +def unquote(str): + """Returns unquoted PO term string, with special PO characters unescaped""" + return re_escaped_char.sub(_sub_replacement, str[1:-1]) + +# class to handle po files +class PoFile(object): + def __init__(self, buffer): + self.buffer = buffer + + def __iter__(self): + self.buffer.seek(0) + self.lines = self._get_lines() + self.lines_count = len(self.lines) + + self.first = True + self.extra_lines= [] + return self + + def _get_lines(self): + lines = self.buffer.readlines() + # remove the BOM (Byte Order Mark): + if len(lines): + lines[0] = unicode(lines[0], 'utf8').lstrip(unicode( codecs.BOM_UTF8, "utf8")) + + lines.append('') # ensure that the file ends with at least an empty line + return lines + + def cur_line(self): + return self.lines_count - len(self.lines) + + def next(self): + trans_type = name = res_id = source = trad = None + if self.extra_lines: + trans_type, name, res_id, source, trad, comments = self.extra_lines.pop(0) + if not res_id: + res_id = '0' + else: + comments = [] + targets = [] + line = None + fuzzy = False + while not line: + if 0 == len(self.lines): + raise StopIteration() + line = self.lines.pop(0).strip() + while line.startswith('#'): + if line.startswith('#~ '): + break + if line.startswith('#.'): + line = line[2:].strip() + if not line.startswith('module:'): + comments.append(line) + elif line.startswith('#:'): + # Process the `reference` comments. Each line can specify + # multiple targets (e.g. model, view, code, selection, + # ...). For each target, we will return an additional + # entry. + for lpart in line[2:].strip().split(' '): + trans_info = lpart.strip().split(':',2) + if trans_info and len(trans_info) == 2: + # looks like the translation trans_type is missing, which is not + # unexpected because it is not a GetText standard. Default: 'code' + trans_info[:0] = ['code'] + if trans_info and len(trans_info) == 3: + # this is a ref line holding the destination info (model, field, record) + targets.append(trans_info) + elif line.startswith('#,') and (line[2:].strip() == 'fuzzy'): + fuzzy = True + line = self.lines.pop(0).strip() + if not self.lines: + raise StopIteration() + while not line: + # allow empty lines between comments and msgid + line = self.lines.pop(0).strip() + if line.startswith('#~ '): + while line.startswith('#~ ') or not line.strip(): + if 0 == len(self.lines): + raise StopIteration() + line = self.lines.pop(0) + # This has been a deprecated entry, don't return anything + return self.next() + + if not line.startswith('msgid'): + raise Exception("malformed file: bad line: %s" % line) + source = unquote(line[6:]) + line = self.lines.pop(0).strip() + if not source and self.first: + self.first = False + # if the source is "" and it's the first msgid, it's the special + # msgstr with the informations about the traduction and the + # traductor; we skip it + self.extra_lines = [] + while line: + line = self.lines.pop(0).strip() + return self.next() + + while not line.startswith('msgstr'): + if not line: + raise Exception('malformed file at %d'% self.cur_line()) + source += unquote(line) + line = self.lines.pop(0).strip() + + trad = unquote(line[7:]) + line = self.lines.pop(0).strip() + while line: + trad += unquote(line) + line = self.lines.pop(0).strip() + + if targets and not fuzzy: + # Use the first target for the current entry (returned at the + # end of this next() call), and keep the others to generate + # additional entries (returned the next next() calls). + trans_type, name, res_id = targets.pop(0) + for t, n, r in targets: + if t == trans_type == 'code': continue + self.extra_lines.append((t, n, r, source, trad, comments)) + + if name is None: + if not fuzzy: + _logger.warning('Missing "#:" formated comment at line %d for the following source:\n\t%s', + self.cur_line(), source[:30]) + return self.next() + return trans_type, name, res_id, source, trad, '\n'.join(comments) + + def write_infos(self, modules): + import openerp.release as release + self.buffer.write("# Translation of %(project)s.\n" \ + "# This file contains the translation of the following modules:\n" \ + "%(modules)s" \ + "#\n" \ + "msgid \"\"\n" \ + "msgstr \"\"\n" \ + '''"Project-Id-Version: %(project)s %(version)s\\n"\n''' \ + '''"Report-Msgid-Bugs-To: \\n"\n''' \ + '''"POT-Creation-Date: %(now)s\\n"\n''' \ + '''"PO-Revision-Date: %(now)s\\n"\n''' \ + '''"Last-Translator: <>\\n"\n''' \ + '''"Language-Team: \\n"\n''' \ + '''"MIME-Version: 1.0\\n"\n''' \ + '''"Content-Type: text/plain; charset=UTF-8\\n"\n''' \ + '''"Content-Transfer-Encoding: \\n"\n''' \ + '''"Plural-Forms: \\n"\n''' \ + "\n" + + % { 'project': release.description, + 'version': release.version, + 'modules': reduce(lambda s, m: s + "#\t* %s\n" % m, modules, ""), + 'now': datetime.utcnow().strftime('%Y-%m-%d %H:%M')+"+0000", + } + ) + + def write(self, modules, tnrs, source, trad, comments=None): + + plurial = len(modules) > 1 and 's' or '' + self.buffer.write("#. module%s: %s\n" % (plurial, ', '.join(modules))) + + if comments: + self.buffer.write(''.join(('#. %s\n' % c for c in comments))) + + code = False + for typy, name, res_id in tnrs: + self.buffer.write("#: %s:%s:%s\n" % (typy, name, res_id)) + if typy == 'code': + code = True + + if code: + # only strings in python code are python formated + self.buffer.write("#, python-format\n") + + if not isinstance(trad, unicode): + trad = unicode(trad, 'utf8') + if not isinstance(source, unicode): + source = unicode(source, 'utf8') + + msg = "msgid %s\n" \ + "msgstr %s\n\n" \ + % (quote(source), quote(trad)) + self.buffer.write(msg.encode('utf8')) + + +# Methods to export the translation file + +def trans_export(lang, modules, buffer, format, cr): + + def _process(format, modules, rows, buffer, lang): + if format == 'csv': + writer = csv.writer(buffer, 'UNIX') + # write header first + writer.writerow(("module","type","name","res_id","src","value","comments")) + for module, type, name, res_id, src, trad, comments in rows: + comments = '\n'.join(comments) + writer.writerow((module, type, name, res_id, src, trad, comments)) + + elif format == 'po': + writer = PoFile(buffer) + writer.write_infos(modules) + + # we now group the translations by source. That means one translation per source. + grouped_rows = {} + for module, type, name, res_id, src, trad, comments in rows: + row = grouped_rows.setdefault(src, {}) + row.setdefault('modules', set()).add(module) + if not row.get('translation') and trad != src: + row['translation'] = trad + row.setdefault('tnrs', []).append((type, name, res_id)) + row.setdefault('comments', set()).update(comments) + + for src, row in sorted(grouped_rows.items()): + if not lang: + # translation template, so no translation value + row['translation'] = '' + elif not row.get('translation'): + row['translation'] = src + writer.write(row['modules'], row['tnrs'], src, row['translation'], row['comments']) + + elif format == 'tgz': + rows_by_module = {} + for row in rows: + module = row[0] + rows_by_module.setdefault(module, []).append(row) + tmpdir = tempfile.mkdtemp() + for mod, modrows in rows_by_module.items(): + tmpmoddir = join(tmpdir, mod, 'i18n') + os.makedirs(tmpmoddir) + pofilename = (lang if lang else mod) + ".po" + ('t' if not lang else '') + buf = file(join(tmpmoddir, pofilename), 'w') + _process('po', [mod], modrows, buf, lang) + buf.close() + + tar = tarfile.open(fileobj=buffer, mode='w|gz') + tar.add(tmpdir, '') + tar.close() + + else: + raise Exception(_('Unrecognized extension: must be one of ' + '.csv, .po, or .tgz (received .%s).') % format) + + translations = trans_generate(lang, modules, cr) + modules = set(t[0] for t in translations) + _process(format, modules, translations, buffer, lang) + del translations + + +def trans_parse_rml(de): + res = [] + for n in de: + for m in n: + if isinstance(m, SKIPPED_ELEMENT_TYPES) or not m.text: + continue + string_list = [s.replace('\n', ' ').strip() for s in re.split('\[\[.+?\]\]', m.text)] + for s in string_list: + if s: + res.append(s.encode("utf8")) + res.extend(trans_parse_rml(n)) + return res + + +def _push(callback, term, source_line): + """ Sanity check before pushing translation terms """ + term = (term or "").strip().encode('utf8') + # Avoid non-char tokens like ':' '...' '.00' etc. + if len(term) > 8 or any(x.isalpha() for x in term): + callback(term, source_line) + + +# tests whether an object is in a list of modules +def in_modules(object_name, modules): + if 'all' in modules: + return True + + module_dict = { + 'ir': 'base', + 'res': 'base', + 'workflow': 'base', + } + module = object_name.split('.')[0] + module = module_dict.get(module, module) + return module in modules + + +def _extract_translatable_qweb_terms(element, callback): + """ Helper method to walk an etree document representing + a QWeb template, and call ``callback(term)`` for each + translatable term that is found in the document. + + :param etree._Element element: root of etree document to extract terms from + :param Callable callback: a callable in the form ``f(term, source_line)``, + that will be called for each extracted term. + """ + # not using elementTree.iterparse because we need to skip sub-trees in case + # the ancestor element had a reason to be skipped + for el in element: + if isinstance(el, SKIPPED_ELEMENT_TYPES): continue + if (el.tag.lower() not in SKIPPED_ELEMENTS + and "t-js" not in el.attrib + and not ("t-jquery" in el.attrib and "t-operation" not in el.attrib) + and el.get("t-translation", '').strip() != "off"): + _push(callback, el.text, el.sourceline) + for att in ('title', 'alt', 'label', 'placeholder'): + if att in el.attrib: + _push(callback, el.attrib[att], el.sourceline) + _extract_translatable_qweb_terms(el, callback) + _push(callback, el.tail, el.sourceline) + + +def babel_extract_qweb(fileobj, keywords, comment_tags, options): + """Babel message extractor for qweb template files. + + :param fileobj: the file-like object the messages should be extracted from + :param keywords: a list of keywords (i.e. function names) that should + be recognized as translation functions + :param comment_tags: a list of translator tags to search for and + include in the results + :param options: a dictionary of additional options (optional) + :return: an iterator over ``(lineno, funcname, message, comments)`` + tuples + :rtype: Iterable + """ + result = [] + def handle_text(text, lineno): + result.append((lineno, None, text, [])) + tree = etree.parse(fileobj) + _extract_translatable_qweb_terms(tree.getroot(), handle_text) + return result + + +def trans_generate(lang, modules, cr): + env = openerp.api.Environment(cr, SUPERUSER_ID, {}) + to_translate = set() + + def push_translation(module, type, name, id, source, comments=None): + # empty and one-letter terms are ignored, they probably are not meant to be + # translated, and would be very hard to translate anyway. + sanitized_term = (source or '').strip() + try: + # verify the minimal size without eventual xml tags + # wrap to make sure html content like '<a>b</a><c>d</c>' is accepted by lxml + wrapped = "<div>%s</div>" % sanitized_term + node = etree.fromstring(wrapped) + sanitized_term = etree.tostring(node, encoding='UTF-8', method='text') + except etree.ParseError: + pass + # remove non-alphanumeric chars + sanitized_term = re.sub(r'\W+', '', sanitized_term) + if not sanitized_term or len(sanitized_term) <= 1: + return + + tnx = (module, source, name, id, type, tuple(comments or ())) + to_translate.add(tnx) + + query = 'SELECT name, model, res_id, module FROM ir_model_data' + query_models = """SELECT m.id, m.model, imd.module + FROM ir_model AS m, ir_model_data AS imd + WHERE m.id = imd.res_id AND imd.model = 'ir.model'""" + + if 'all_installed' in modules: + query += ' WHERE module IN ( SELECT name FROM ir_module_module WHERE state = \'installed\') ' + query_models += " AND imd.module in ( SELECT name FROM ir_module_module WHERE state = 'installed') " + + if 'all' not in modules: + query += ' WHERE module IN %s' + query_models += ' AND imd.module IN %s' + query_param = (tuple(modules),) + else: + query += ' WHERE module != %s' + query_models += ' AND imd.module != %s' + query_param = ('__export__',) + + query += ' ORDER BY module, model, name' + query_models += ' ORDER BY module, model' + + cr.execute(query, query_param) + + for (xml_name, model, res_id, module) in cr.fetchall(): + module = encode(module) + model = encode(model) + xml_name = "%s.%s" % (module, encode(xml_name)) + + if model not in env: + _logger.error("Unable to find object %r", model) + continue + + record = env[model].browse(res_id) + if not record._translate: + # explicitly disabled + continue + + if not record.exists(): + _logger.warning("Unable to find object %r with id %d", model, res_id) + continue + + if model=='ir.model.fields': + try: + field_name = encode(record.name) + except AttributeError, exc: + _logger.error("name error in %s: %s", xml_name, str(exc)) + continue + field_model = env.get(record.model) + if (field_model is None or not field_model._translate or + field_name not in field_model._fields): + continue + field = field_model._fields[field_name] + + if isinstance(getattr(field, 'selection', None), (list, tuple)): + name = "%s,%s" % (encode(record.model), field_name) + for dummy, val in field.selection: + push_translation(module, 'selection', name, 0, encode(val)) + + elif model=='ir.actions.report.xml': + name = encode(record.report_name) + fname = "" + if record.report_rml: + fname = record.report_rml + parse_func = trans_parse_rml + report_type = "report" + elif record.report_xsl: + continue + if fname and record.report_type in ('pdf', 'xsl'): + try: + with file_open(fname) as report_file: + d = etree.parse(report_file) + for t in parse_func(d.iter()): + push_translation(module, report_type, name, 0, t) + except (IOError, etree.XMLSyntaxError): + _logger.exception("couldn't export translation for report %s %s %s", name, report_type, fname) + + for field_name, field in record._fields.iteritems(): + if field.translate: + name = model + "," + field_name + try: + value = record[field_name] or '' + except Exception: + continue + for term in set(field.get_trans_terms(value)): + push_translation(module, 'model', name, xml_name, encode(term)) + + # End of data for ir.model.data query results + + def push_constraint_msg(module, term_type, model, msg): + if not callable(msg): + push_translation(encode(module), term_type, encode(model), 0, encode(msg)) + + def push_local_constraints(module, model, cons_type='sql_constraints'): + """ Climb up the class hierarchy and ignore inherited constraints from other modules. """ + term_type = 'sql_constraint' if cons_type == 'sql_constraints' else 'constraint' + msg_pos = 2 if cons_type == 'sql_constraints' else 1 + for cls in model.__class__.__mro__: + if getattr(cls, '_module', None) != module: + continue + constraints = getattr(cls, '_local_' + cons_type, []) + for constraint in constraints: + push_constraint_msg(module, term_type, model._name, constraint[msg_pos]) + + cr.execute(query_models, query_param) + + for (_, model, module) in cr.fetchall(): + if model not in env: + _logger.error("Unable to find object %r", model) + continue + Model = env[model] + if Model._constraints: + push_local_constraints(module, Model, 'constraints') + if Model._sql_constraints: + push_local_constraints(module, Model, 'sql_constraints') + + installed_modules = [ + m['name'] + for m in env['ir.module.module'].search_read([('state', '=', 'installed')], fields=['name']) + ] + + path_list = [(path, True) for path in openerp.modules.module.ad_paths] + # Also scan these non-addon paths + for bin_path in ['osv', 'report', 'modules', 'service', 'tools']: + path_list.append((os.path.join(config['root_path'], bin_path), True)) + # non-recursive scan for individual files in root directory but without + # scanning subdirectories that may contain addons + path_list.append((config['root_path'], False)) + _logger.debug("Scanning modules at paths: %s", path_list) + + def get_module_from_path(path): + for (mp, rec) in path_list: + mp = os.path.join(mp, '') + if rec and path.startswith(mp) and os.path.dirname(path) != mp: + path = path[len(mp):] + return path.split(os.path.sep)[0] + return 'base' # files that are not in a module are considered as being in 'base' module + + def verified_module_filepaths(fname, path, root): + fabsolutepath = join(root, fname) + frelativepath = fabsolutepath[len(path):] + display_path = "addons%s" % frelativepath + module = get_module_from_path(fabsolutepath) + if ('all' in modules or module in modules) and module in installed_modules: + if os.path.sep != '/': + display_path = display_path.replace(os.path.sep, '/') + return module, fabsolutepath, frelativepath, display_path + return None, None, None, None + + def babel_extract_terms(fname, path, root, extract_method="python", trans_type='code', + extra_comments=None, extract_keywords={'_': None}): + module, fabsolutepath, _, display_path = verified_module_filepaths(fname, path, root) + extra_comments = extra_comments or [] + if not module: return + src_file = open(fabsolutepath, 'r') + try: + for extracted in extract.extract(extract_method, src_file, keywords=extract_keywords): + # Babel 0.9.6 yields lineno, message, comments + # Babel 1.3 yields lineno, message, comments, context + lineno, message, comments = extracted[:3] + push_translation(module, trans_type, display_path, lineno, + encode(message), comments + extra_comments) + except Exception: + _logger.exception("Failed to extract terms from %s", fabsolutepath) + finally: + src_file.close() + + for (path, recursive) in path_list: + _logger.debug("Scanning files of modules at %s", path) + for root, dummy, files in walksymlinks(path): + for fname in fnmatch.filter(files, '*.py'): + babel_extract_terms(fname, path, root) + # mako provides a babel extractor: http://docs.makotemplates.org/en/latest/usage.html#babel + for fname in fnmatch.filter(files, '*.mako'): + babel_extract_terms(fname, path, root, 'mako', trans_type='report') + # Javascript source files in the static/src/js directory, rest is ignored (libs) + if fnmatch.fnmatch(root, '*/static/src/js*'): + for fname in fnmatch.filter(files, '*.js'): + babel_extract_terms(fname, path, root, 'javascript', + extra_comments=[WEB_TRANSLATION_COMMENT], + extract_keywords={'_t': None, '_lt': None}) + # QWeb template files + if fnmatch.fnmatch(root, '*/static/src/xml*'): + for fname in fnmatch.filter(files, '*.xml'): + babel_extract_terms(fname, path, root, 'openerp.tools.translate:babel_extract_qweb', + extra_comments=[WEB_TRANSLATION_COMMENT]) + if not recursive: + # due to topdown, first iteration is in first level + break + + out = [] + # translate strings marked as to be translated + Translation = env['ir.translation'] + for module, source, name, id, type, comments in sorted(to_translate): + trans = Translation._get_source(name, type, lang, source) if lang else "" + out.append((module, type, name, id, source, encode(trans) or '', comments)) + return out + + +def trans_load(cr, filename, lang, verbose=True, module_name=None, context=None): + try: + with file_open(filename) as fileobj: + _logger.info("loading %s", filename) + fileformat = os.path.splitext(filename)[-1][1:].lower() + result = trans_load_data(cr, fileobj, fileformat, lang, verbose=verbose, module_name=module_name, context=context) + return result + except IOError: + if verbose: + _logger.error("couldn't read translation file %s", filename) + return None + + +def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, module_name=None, context=None): + """Populates the ir_translation table.""" + if verbose: + _logger.info('loading translation file for language %s', lang) + + env = openerp.api.Environment(cr, SUPERUSER_ID, context or {}) + Lang = env['res.lang'] + Translation = env['ir.translation'] + + try: + if not Lang.search_count([('code', '=', lang)]): + # lets create the language with locale information + Lang.load_lang(lang=lang, lang_name=lang_name) + + # Parse also the POT: it will possibly provide additional targets. + # (Because the POT comments are correct on Launchpad but not the + # PO comments due to a Launchpad limitation. See LP bug 933496.) + pot_reader = [] + + # now, the serious things: we read the language file + fileobj.seek(0) + if fileformat == 'csv': + reader = csv.reader(fileobj, quotechar='"', delimiter=',') + # read the first line of the file (it contains columns titles) + for row in reader: + fields = row + break + + elif fileformat == 'po': + reader = PoFile(fileobj) + fields = ['type', 'name', 'res_id', 'src', 'value', 'comments'] + + # Make a reader for the POT file and be somewhat defensive for the + # stable branch. + if fileobj.name.endswith('.po'): + try: + # Normally the path looks like /path/to/xxx/i18n/lang.po + # and we try to find the corresponding + # /path/to/xxx/i18n/xxx.pot file. + # (Sometimes we have 'i18n_extra' instead of just 'i18n') + addons_module_i18n, _ignored = os.path.split(fileobj.name) + addons_module, i18n_dir = os.path.split(addons_module_i18n) + addons, module = os.path.split(addons_module) + pot_handle = file_open(os.path.join( + addons, module, i18n_dir, module + '.pot')) + pot_reader = PoFile(pot_handle) + except: + pass + + else: + _logger.info('Bad file format: %s', fileformat) + raise Exception(_('Bad file format: %s') % fileformat) + + # Read the POT references, and keep them indexed by source string. + class Target(object): + def __init__(self): + self.value = None + self.targets = set() # set of (type, name, res_id) + self.comments = None + + pot_targets = defaultdict(Target) + for type, name, res_id, src, _ignored, comments in pot_reader: + if type is not None: + target = pot_targets[src] + target.targets.add((type, name, res_id)) + target.comments = comments + + # read the rest of the file + irt_cursor = Translation._get_import_cursor() + + def process_row(row): + """Process a single PO (or POT) entry.""" + # dictionary which holds values for this line of the csv file + # {'lang': ..., 'type': ..., 'name': ..., 'res_id': ..., + # 'src': ..., 'value': ..., 'module':...} + dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value', + 'comments', 'imd_model', 'imd_name', 'module')) + dic['lang'] = lang + dic.update(zip(fields, row)) + + # discard the target from the POT targets. + src = dic['src'] + if src in pot_targets: + target = pot_targets[src] + target.value = dic['value'] + target.targets.discard((dic['type'], dic['name'], dic['res_id'])) + + # This would skip terms that fail to specify a res_id + res_id = dic['res_id'] + if not res_id: + return + + if isinstance(res_id, (int, long)) or \ + (isinstance(res_id, basestring) and res_id.isdigit()): + dic['res_id'] = int(res_id) + if module_name: + dic['module'] = module_name + else: + # res_id is an xml id + dic['res_id'] = None + dic['imd_model'] = dic['name'].split(',')[0] + if '.' in res_id: + dic['module'], dic['imd_name'] = res_id.split('.', 1) + else: + dic['module'], dic['imd_name'] = module_name, res_id + + irt_cursor.push(dic) + + # First process the entries from the PO file (doing so also fills/removes + # the entries from the POT file). + for row in reader: + process_row(row) + + # Then process the entries implied by the POT file (which is more + # correct w.r.t. the targets) if some of them remain. + pot_rows = [] + for src, target in pot_targets.iteritems(): + if target.value: + for type, name, res_id in target.targets: + pot_rows.append((type, name, res_id, src, target.value, target.comments)) + pot_targets.clear() + for row in pot_rows: + process_row(row) + + irt_cursor.finish() + Translation.clear_caches() + if verbose: + _logger.info("translation file loaded succesfully") + + except IOError: + iso_lang = get_iso_codes(lang) + filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat) + _logger.exception("couldn't read translation file %s", filename) + + +def get_locales(lang=None): + if lang is None: + lang = locale.getdefaultlocale()[0] + + if os.name == 'nt': + lang = _LOCALE2WIN32.get(lang, lang) + + def process(enc): + ln = locale._build_localename((lang, enc)) + yield ln + nln = locale.normalize(ln) + if nln != ln: + yield nln + + for x in process('utf8'): yield x + + prefenc = locale.getpreferredencoding() + if prefenc: + for x in process(prefenc): yield x + + prefenc = { + 'latin1': 'latin9', + 'iso-8859-1': 'iso8859-15', + 'cp1252': '1252', + }.get(prefenc.lower()) + if prefenc: + for x in process(prefenc): yield x + + yield lang + + +def resetlocale(): + # locale.resetlocale is bugged with some locales. + for ln in get_locales(): + try: + return locale.setlocale(locale.LC_ALL, ln) + except locale.Error: + continue + + +def load_language(cr, lang): + """ Loads a translation terms for a language. + Used mainly to automate language loading at db initialization. + + :param lang: language ISO code with optional _underscore_ and l10n flavor (ex: 'fr', 'fr_BE', but not 'fr-BE') + :type lang: str + """ + env = openerp.api.Environment(cr, SUPERUSER_ID, {}) + installer = env['base.language.install'].create({'lang': lang}) + installer.lang_install() diff --git a/odoo/tools/view_validation.py b/odoo/tools/view_validation.py new file mode 100644 index 00000000000..40a85f18ccd --- /dev/null +++ b/odoo/tools/view_validation.py @@ -0,0 +1,84 @@ +""" View validation code (using assertions, not the RNG schema). """ + +import collections +import logging + +_logger = logging.getLogger(__name__) + + +_validators = collections.defaultdict(list) +def valid_view(arch): + for pred in _validators[arch.tag]: + if not pred(arch): + _logger.error("Invalid XML: %s", pred.__doc__) + return False + return True + + +def validate(*view_types): + """ Registers a view-validation function for the specific view types + """ + def decorator(fn): + for arch in view_types: + _validators[arch].append(fn) + return fn + return decorator + + +@validate('form') +def valid_page_in_book(arch): + """A `page` node must be below a `notebook` node.""" + return not arch.xpath('//page[not(ancestor::notebook)]') + + +@validate('graph') +def valid_field_in_graph(arch): + """ Children of ``graph`` can only be ``field`` """ + return all( + child.tag == 'field' + for child in arch.xpath('/graph/*') + ) + + +@validate('tree') +def valid_field_in_tree(arch): + """ Children of ``tree`` view must be ``field`` or ``button``.""" + return all( + child.tag in ('field', 'button') + for child in arch.xpath('/tree/*') + ) + + +@validate('form', 'graph', 'tree') +def valid_att_in_field(arch): + """ ``field`` nodes must all have a ``@name`` """ + return not arch.xpath('//field[not(@name)]') + + +@validate('form') +def valid_att_in_label(arch): + """ ``label`` nodes must have a ``@for`` or a ``@string`` """ + return not arch.xpath('//label[not(@for or @string)]') + + +@validate('form') +def valid_att_in_form(arch): + return True + + +@validate('form') +def valid_type_in_colspan(arch): + """A `colspan` attribute must be an `integer` type.""" + return all( + attrib.isdigit() + for attrib in arch.xpath('//@colspan') + ) + + +@validate('form') +def valid_type_in_col(arch): + """A `col` attribute must be an `integer` type.""" + return all( + attrib.isdigit() + for attrib in arch.xpath('//@col') + ) diff --git a/odoo/tools/which.py b/odoo/tools/which.py new file mode 100755 index 00000000000..548b836247c --- /dev/null +++ b/odoo/tools/which.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +""" Which - locate a command + + * adapted from Brian Curtin's http://bugs.python.org/file15381/shutil_which.patch + * see http://bugs.python.org/issue444582 + * uses ``PATHEXT`` on Windows + * searches current directory before ``PATH`` on Windows, + but not before an explicitly passed path + * accepts both string or iterable for an explicitly passed path, or pathext + * accepts an explicitly passed empty path, or pathext (either '' or []) + * does not search ``PATH`` for files that have a path specified in their name already + * moved defpath and defpathext lists initialization to module level, + instead of initializing them on each function call + * changed interface: which_files() returns generator, which() returns first match, + or raises IOError(errno.ENOENT) + + .. function:: which_files(file [, mode=os.F_OK | os.X_OK[, path=None[, pathext=None]]]) + + Return a generator which yields full paths in which the *file* name exists + in a directory that is part of the file name, or on *path*, + and has the given *mode*. + By default, *mode* matches an inclusive OR of os.F_OK and os.X_OK - an + existing executable file. + The *path* is, by default, the ``PATH`` variable on the platform, + or the string/iterable passed in as *path*. + In the event that a ``PATH`` variable is not found, :const:`os.defpath` is used. + On Windows, a current directory is searched before using the ``PATH`` variable, + but not before an explicitly passed *path*. + The *pathext* is only used on Windows to match files with given extensions appended as well. + It defaults to the ``PATHEXT`` variable, or the string/iterable passed in as *pathext*. + In the event that a ``PATHEXT`` variable is not found, + default value for Windows XP/Vista is used. + The command is always searched without extension first, + even when *pathext* is explicitly passed. + + .. function:: which(file [, mode=os.F_OK | os.X_OK[, path=None[, pathext=None]]]) + Return first match generated by which_files(file, mode, path, pathext), + or raise IOError(errno.ENOENT). + +""" +__docformat__ = 'restructuredtext en' +__all__ = 'which which_files pathsep defpath defpathext F_OK R_OK W_OK X_OK'.split() + +import sys +from os import access, defpath, pathsep, environ, F_OK, R_OK, W_OK, X_OK +from os.path import exists, dirname, split, join + +windows = sys.platform.startswith('win') + +defpath = environ.get('PATH', defpath).split(pathsep) + +if windows: + defpath.insert(0, '.') # can insert without checking, when duplicates are removed + # given the quite usual mess in PATH on Windows, let's rather remove duplicates + seen = set() + defpath = [dir for dir in defpath if dir.lower() not in seen and not seen.add(dir.lower())] + del seen + + defpathext = [''] + environ.get('PATHEXT', + '.COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC').lower().split(pathsep) +else: + defpathext = [''] + +def which_files(file, mode=F_OK | X_OK, path=None, pathext=None): + """ Locate a file in a path supplied as a part of the file name, + or the user's path, or a supplied path. + The function yields full paths (not necessarily absolute paths), + in which the given file name matches an existing file in a directory on the path. + + >>> def test_which(expected, *args, **argd): + ... result = list(which_files(*args, **argd)) + ... assert result == expected, 'which_files: %s != %s' % (result, expected) + ... + ... try: + ... result = [ which(*args, **argd) ] + ... except IOError: + ... result = [] + ... assert result[:1] == expected[:1], 'which: %s != %s' % (result[:1], expected[:1]) + + >>> if windows: cmd = environ['COMSPEC'] + >>> if windows: test_which([cmd], 'cmd') + >>> if windows: test_which([cmd], 'cmd.exe') + >>> if windows: test_which([cmd], 'cmd', path=dirname(cmd)) + >>> if windows: test_which([cmd], 'cmd', pathext='.exe') + >>> if windows: test_which([cmd], cmd) + >>> if windows: test_which([cmd], cmd, path='<nonexistent>') + >>> if windows: test_which([cmd], cmd, pathext='<nonexistent>') + >>> if windows: test_which([cmd], cmd[:-4]) + >>> if windows: test_which([cmd], cmd[:-4], path='<nonexistent>') + + >>> if windows: test_which([], 'cmd', path='<nonexistent>') + >>> if windows: test_which([], 'cmd', pathext='<nonexistent>') + >>> if windows: test_which([], '<nonexistent>/cmd') + >>> if windows: test_which([], cmd[:-4], pathext='<nonexistent>') + + >>> if not windows: sh = '/bin/sh' + >>> if not windows: test_which([sh], 'sh') + >>> if not windows: test_which([sh], 'sh', path=dirname(sh)) + >>> if not windows: test_which([sh], 'sh', pathext='<nonexistent>') + >>> if not windows: test_which([sh], sh) + >>> if not windows: test_which([sh], sh, path='<nonexistent>') + >>> if not windows: test_which([sh], sh, pathext='<nonexistent>') + + >>> if not windows: test_which([], 'sh', mode=W_OK) # not running as root, are you? + >>> if not windows: test_which([], 'sh', path='<nonexistent>') + >>> if not windows: test_which([], '<nonexistent>/sh') + """ + filepath, file = split(file) + + if filepath: + path = (filepath,) + elif path is None: + path = defpath + elif isinstance(path, str): + path = path.split(pathsep) + + if pathext is None: + pathext = defpathext + elif isinstance(pathext, str): + pathext = pathext.split(pathsep) + + if not '' in pathext: + pathext.insert(0, '') # always check command without extension, even for custom pathext + + for dir in path: + basepath = join(dir, file) + for ext in pathext: + fullpath = basepath + ext + if exists(fullpath) and access(fullpath, mode): + yield fullpath + +def which(file, mode=F_OK | X_OK, path=None, pathext=None): + """ Locate a file in a path supplied as a part of the file name, + or the user's path, or a supplied path. + The function returns full path (not necessarily absolute path), + in which the given file name matches an existing file in a directory on the path, + or raises IOError(errno.ENOENT). + + >>> # for doctest see which_files() + """ + try: + return iter(which_files(file, mode, path, pathext)).next() + except StopIteration: + try: + from errno import ENOENT + except ImportError: + ENOENT = 2 + raise IOError(ENOENT, '%s not found' % (mode & X_OK and 'command' or 'file'), file) + + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/odoo/tools/win32.py b/odoo/tools/win32.py new file mode 100644 index 00000000000..9445ba857f5 --- /dev/null +++ b/odoo/tools/win32.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import locale +import time +import datetime + +if not hasattr(locale, 'D_FMT'): + locale.D_FMT = 1 + +if not hasattr(locale, 'T_FMT'): + locale.T_FMT = 2 + +if not hasattr(locale, 'nl_langinfo'): + def nl_langinfo(param): + if param == locale.D_FMT: + val = time.strptime('30/12/2004', '%d/%m/%Y') + dt = datetime.datetime(*val[:-2]) + format_date = dt.strftime('%x') + for x, y in [('30', '%d'),('12', '%m'),('2004','%Y'),('04', '%Y')]: + format_date = format_date.replace(x, y) + return format_date + if param == locale.T_FMT: + val = time.strptime('13:24:56', '%H:%M:%S') + dt = datetime.datetime(*val[:-2]) + format_time = dt.strftime('%X') + for x, y in [('13', '%H'),('24', '%M'),('56','%S')]: + format_time = format_time.replace(x, y) + return format_time + locale.nl_langinfo = nl_langinfo diff --git a/odoo/tools/yaml_import.py b/odoo/tools/yaml_import.py new file mode 100644 index 00000000000..225c0388f8c --- /dev/null +++ b/odoo/tools/yaml_import.py @@ -0,0 +1,949 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from collections import OrderedDict +from datetime import datetime, timedelta +import logging +import re +import time # used to eval time.strftime expressions +import types + +from lxml import etree +import yaml + +import openerp +from . import assertion_report +from . import yaml_tag +from .config import config +from .misc import file_open, DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT +from openerp import SUPERUSER_ID + +# YAML import needs both safe and unsafe eval, but let's +# default to /safe/. +unsafe_eval = eval +from .safe_eval import safe_eval + +_logger = logging.getLogger(__name__) + +def encode(s): + return s.encode('utf8') if isinstance(s, unicode) else s + +class YamlImportException(Exception): + pass + +class YamlImportAbortion(Exception): + pass + +def _is_yaml_mapping(node, tag_constructor): + value = isinstance(node, types.DictionaryType) \ + and len(node.keys()) == 1 \ + and isinstance(node.keys()[0], tag_constructor) + return value + +def is_comment(node): + return isinstance(node, types.StringTypes) + +def is_assert(node): + return isinstance(node, yaml_tag.Assert) \ + or _is_yaml_mapping(node, yaml_tag.Assert) + +def is_record(node): + return _is_yaml_mapping(node, yaml_tag.Record) + +def is_python(node): + return _is_yaml_mapping(node, yaml_tag.Python) + +def is_menuitem(node): + return isinstance(node, yaml_tag.Menuitem) \ + or _is_yaml_mapping(node, yaml_tag.Menuitem) + +def is_function(node): + return isinstance(node, yaml_tag.Function) \ + or _is_yaml_mapping(node, yaml_tag.Function) + +def is_report(node): + return isinstance(node, yaml_tag.Report) + +def is_workflow(node): + return isinstance(node, yaml_tag.Workflow) + +def is_act_window(node): + return isinstance(node, yaml_tag.ActWindow) + +def is_delete(node): + return isinstance(node, yaml_tag.Delete) + +def is_context(node): + return isinstance(node, yaml_tag.Context) + +def is_url(node): + return isinstance(node, yaml_tag.Url) + +def is_eval(node): + return isinstance(node, yaml_tag.Eval) + +def is_ref(node): + return isinstance(node, yaml_tag.Ref) \ + or _is_yaml_mapping(node, yaml_tag.Ref) + +def is_ir_set(node): + return _is_yaml_mapping(node, yaml_tag.IrSet) + +def is_string(node): + return isinstance(node, basestring) + +class RecordDictWrapper(dict): + """ + Used to pass a record as locals in eval: + records do not strictly behave like dict, so we force them to. + """ + def __init__(self, record): + self.record = record + def __getitem__(self, key): + if key in self.record: + return self.record[key] + return dict.__getitem__(self, key) + +class YamlInterpreter(object): + def __init__(self, cr, module, id_map, mode, filename, report=None, noupdate=False, loglevel=logging.DEBUG): + self.cr = cr + self.module = module + self.id_map = id_map + self.mode = mode + self.filename = filename + if report is None: + report = assertion_report.assertion_report() + self.assertion_report = report + self.noupdate = noupdate + self.loglevel = loglevel + self.uid = SUPERUSER_ID + self.context = {} # opererp context + self.eval_context = {'ref': self.get_id, + '_ref': self.get_id, # added '_ref' so that record['ref'] is possible + 'time': time, + 'datetime': datetime, + 'timedelta': timedelta} + self.env = openerp.api.Environment(self.cr, self.uid, self.context) + self.sudo_env = self.env + + def _log(self, *args, **kwargs): + _logger.log(self.loglevel, *args, **kwargs) + + def validate_xml_id(self, xml_id): + id = xml_id + if '.' in xml_id: + module, id = xml_id.split('.', 1) + assert '.' not in id, "The ID reference '%s' must contain at most one dot.\n" \ + "It is used to refer to other modules ID, in the form: module.record_id" \ + % (xml_id,) + if module != self.module: + module_count = self.env['ir.module.module'].search_count([('name', '=', module), ('state', '=', 'installed')]) + assert module_count == 1, 'The ID "%s" refers to an uninstalled module.' % (xml_id,) + if len(id) > 64: # TODO where does 64 come from (DB is 128)? should be a constant or loaded form DB + _logger.error('id: %s is to long (max: 64)', id) + + def get_id(self, xml_id): + if xml_id is False or xml_id is None: + return False + #if not xml_id: + # raise YamlImportException("The xml_id should be a non empty string.") + elif isinstance(xml_id, types.IntType): + id = xml_id + elif xml_id in self.id_map: + id = self.id_map[xml_id] + else: + full_xml_id = xml_id + if '.' not in full_xml_id: + full_xml_id = self.module + '.' + full_xml_id + try: + id = self.env.ref(full_xml_id).id + self.id_map[xml_id] = id + except ValueError: + raise ValueError("""%r not found when processing %s. + This Yaml file appears to depend on missing data. This often happens for + tests that belong to a module's test suite and depend on each other.""" % (xml_id, self.filename)) + + return id + + def get_record(self, xml_id): + if '.' not in xml_id: + xml_id = "%s.%s" % (self.module, xml_id) + return self.env.ref(xml_id) + + def get_context(self, node, eval_dict): + context = self.context.copy() + if node.context: + context.update(safe_eval(node.context, eval_dict)) + return context + + def isnoupdate(self, node): + return self.noupdate or node.noupdate or False + + def _get_first_result(self, results, default=False): + if len(results): + value = results[0] + if isinstance(value, types.TupleType): + value = value[0] + else: + value = default + return value + + def process_comment(self, node): + return node + + def _log_assert_failure(self, msg, *args): + self.assertion_report.record_failure() + _logger.error(msg, *args) + + def _get_assertion_id(self, assertion): + if assertion.id: + ids = [self.get_id(assertion.id)] + elif assertion.search: + q = safe_eval(assertion.search, self.eval_context) + ids = self.env(context=assertion.context)[assertion.model].search(q) + else: + raise YamlImportException('Nothing to assert: you must give either an id or a search criteria.') + return ids + + def process_assert(self, node): + if isinstance(node, dict): + assertion, expressions = node.items()[0] + else: + assertion, expressions = node, [] + + if self.isnoupdate(assertion) and self.mode != 'init': + _logger.warning('This assertion was not evaluated ("%s").', assertion.string) + return + model = self.env[assertion.model] + ids = self._get_assertion_id(assertion) + if assertion.count is not None and len(ids) != assertion.count: + msg = 'assertion "%s" failed!\n' \ + ' Incorrect search count:\n' \ + ' expected count: %d\n' \ + ' obtained count: %d\n' + args = (assertion.string, assertion.count, len(ids)) + self._log_assert_failure(msg, *args) + else: + context = self.get_context(assertion, self.eval_context) + records = model.with_context(context).browse(ids) + for record in records: + for test in expressions: + try: + success = unsafe_eval(test, self.eval_context, RecordDictWrapper(record)) + except Exception, e: + _logger.debug('Exception during evaluation of !assert block in yaml_file %s.', self.filename, exc_info=True) + raise YamlImportAbortion(e) + if not success: + msg = 'Assertion "%s" FAILED\ntest: %s\n' + args = (assertion.string, test) + for aop in ('==', '!=', '<>', 'in', 'not in', '>=', '<=', '>', '<'): + if aop in test: + left, right = test.split(aop,1) + lmsg = '' + rmsg = '' + try: + lmsg = unsafe_eval(left, self.eval_context, RecordDictWrapper(record)) + except Exception, e: + lmsg = '<exc>' + + try: + rmsg = unsafe_eval(right, self.eval_context, RecordDictWrapper(record)) + except Exception, e: + rmsg = '<exc>' + + msg += 'values: ! %s %s %s' + args += ( lmsg, aop, rmsg ) + break + + self._log_assert_failure(msg, *args) + return + else: # all tests were successful for this assertion tag (no break) + self.assertion_report.record_success() + + def _coerce_bool(self, value, default=False): + if isinstance(value, types.BooleanType): + b = value + if isinstance(value, types.StringTypes): + b = value.strip().lower() not in ('0', 'false', 'off', 'no') + elif isinstance(value, types.IntType): + b = bool(value) + else: + b = default + return b + + def create_osv_memory_record(self, record, fields): + model = self.env[record.model] + context = self.get_context(record, self.eval_context) + record_dict = self._create_record(model, fields, context=context) + id_new = model.with_context(context).create(record_dict).id + self.id_map[record.id] = int(id_new) + return record_dict + + def process_record(self, node): + record, fields = node.items()[0] + model = self.env[record.model] + view_id = record.view + if view_id and (view_id is not True) and isinstance(view_id, basestring): + if '.' not in view_id: + view_id = self.module + '.' + view_id + view_id = self.env.ref(view_id).id + + if model.is_transient(): + record_dict=self.create_osv_memory_record(record, fields) + else: + self.validate_xml_id(record.id) + module = self.module + record_id = record.id + if '.' in record_id: + module, record_id = record_id.split('.',1) + try: + self.sudo_env['ir.model.data']._get_id(module, record_id) + default = False + except ValueError: + default = True + + if self.isnoupdate(record) and self.mode != 'init': + id = self.sudo_env['ir.model.data']._update_dummy(record.model, module, record_id) + # check if the resource already existed at the last update + if id: + self.id_map[record] = int(id) + return None + else: + if not self._coerce_bool(record.forcecreate): + return None + + #context = self.get_context(record, self.eval_context) + # FIXME: record.context like {'withoutemployee':True} should pass from self.eval_context. example: test_project.yml in project module + # TODO: cleaner way to avoid resetting password in auth_signup (makes user creation costly) + context = dict(record.context or {}, no_reset_password=True) + env = self.env(user=SUPERUSER_ID, context=context) + view_info = False + if view_id: + varg = view_id + if view_id is True: varg = False + view_info = model.with_env(env).fields_view_get(varg, 'form') + + record_dict = self._create_record(model, fields, view_info, default=default, context=context) + id = env['ir.model.data']._update(record.model, \ + module, record_dict, record_id, noupdate=self.isnoupdate(record), mode=self.mode) + self.id_map[record.id] = int(id) + if config.get('import_partial'): + self.cr.commit() + + def _create_record(self, model, fields, view_info=None, parent={}, default=True, context=None): + """This function processes the !record tag in yaml files. It simulates the record creation through an xml + view (either specified on the !record tag or the default one for this object), including the calls to + on_change() functions, and sending only values for fields that aren't set as readonly. + :param model: model instance (new API) + :param fields: dictonary mapping the field names and their values + :param view_info: result of fields_view_get() called on the object + :param parent: dictionary containing the values already computed for the parent, in case of one2many fields + :param default: if True, the default values must be processed too or not + :return: dictionary mapping the field names and their values, ready to use when calling the create() function + :rtype: dict + """ + readonly_re = re.compile(r"""("readonly"|'readonly'): *true""") + + class dotdict(object): + """ Dictionary class that allow to access a dictionary value by using '.'. + This is needed to eval correctly statements like 'parent.fieldname' in context. + """ + def __init__(self, d): + self._dict = d + def __getattr__(self, attr): + return self._dict.get(attr, False) + + def get_field_elems(view): + """ return the field elements from a view as an OrderedDict """ + def traverse(node, elems): + if node.tag == 'field': + elems[node.get('name')] = node + else: + for child in node: + traverse(child, elems) + + elems = OrderedDict() + traverse(etree.fromstring(encode(view['arch'])), elems) + return elems + + def is_readonly(field_elem): + """ return whether a given field is readonly """ + # TODO: currently we only support if readonly is True in modifiers. + # Some improvement may be done in order to support modifiers like + # {"readonly": [["state", "not in", ["draft", "confirm"]]]} + return readonly_re.search(field_elem.get('modifiers', '{}')) + + def get_2many_view(fg, field_name, view_type): + """ return a view of the given type for the given field's comodel """ + fdesc = fg[field_name] + return fdesc['views'].get(view_type) or \ + self.sudo_env[fdesc['relation']].fields_view_get(False, view_type) + + def process_vals(fg, vals): + """ sanitize the given field values """ + result = {} + for field_name, field_value in vals.iteritems(): + if field_name not in fg: + continue + if fg[field_name]['type'] == 'many2one' and isinstance(field_value, (tuple, list)): + field_value = field_value[0] + elif fg[field_name]['type'] in ('one2many', 'many2many'): + # 2many fields: sanitize field values of sub-records + sub_fg = get_2many_view(fg, field_name, 'form')['fields'] + def process(command): + if isinstance(command, (tuple, list)) and command[0] in (0, 1): + return (command[0], command[1], process_vals(sub_fg, command[2])) + elif isinstance(command, dict): + return process_vals(sub_fg, command) + return command + field_value = map(process, field_value or []) + result[field_name] = field_value + return result + + def post_process(fg, elems, vals): + """ filter out readonly fields from vals """ + result = {} + for field_name, field_value in vals.iteritems(): + if is_readonly(elems[field_name]): + continue + if fg[field_name]['type'] in ('one2many', 'many2many'): + # 2many fields: filter field values of sub-records + sub_view = get_2many_view(fg, field_name, 'form') + sub_fg = sub_view['fields'] + sub_elems = get_field_elems(sub_view) + def process(command): + if isinstance(command, (tuple, list)) and command[0] in (0, 1): + return (command[0], command[1], post_process(sub_fg, sub_elems, command[2])) + elif isinstance(command, dict): + return (0, 0, post_process(sub_fg, sub_elems, command)) + return command + field_value = map(process, field_value or []) + result[field_name] = field_value + return result + + context = context or {} + fields = fields or {} + parent_values = {context['field_parent']: parent} if context.get('field_parent') else {} + + if view_info: + fg = view_info['fields'] + elems = get_field_elems(view_info) + recs = model.sudo().with_context(**context) + onchange_spec = recs._onchange_spec(view_info) + record_dict = {} + + if default: + # gather the default values on the object. (Can't use `fields´ as parameter instead of {} because we may + # have references like `base.main_company´ in the yaml file and it's not compatible with the function) + defaults = recs.sudo(self.uid)._add_missing_default_values({}) + + # copy the default values in record_dict, only if they are in the view (because that's what the client does) + # the other default values will be added later on by the create(). The other fields in the view that haven't any + # default value are set to False because we may have references to them in other field's context + record_dict = dict.fromkeys(fg, False) + record_dict.update(process_vals(fg, defaults)) + + # execute onchange on default values first + default_names = [name for name in elems if name in record_dict] + result = recs.onchange(dict(record_dict, **parent_values), default_names, onchange_spec) + record_dict.update(process_vals(fg, result.get('value', {}))) + + # fill in fields, and execute onchange where necessary + for field_name, field_elem in elems.iteritems(): + assert field_name in fg, "The field '%s' is defined in the form view but not on the object '%s'!" % (field_name, model._name) + if is_readonly(field_elem): + # skip readonly fields + continue + + if field_name not in fields: + continue + + ctx = dict(context) + form_view = view_info + if fg[field_name]['type'] == 'one2many': + # evaluate one2many fields using the inline form view defined in the parent + form_view = get_2many_view(fg, field_name, 'form') + ctx['field_parent'] = fg[field_name]['relation_field'] + if default and field_elem.get('context'): + ctx.update(safe_eval(field_elem.get('context'), + globals_dict={'parent': dotdict(parent)}, + locals_dict=record_dict)) + + field_value = self._eval_field(model, field_name, fields[field_name], form_view, parent=record_dict, default=default, context=ctx) + record_dict.update(process_vals(fg, {field_name: field_value})) + + # if field_name is given or has a default value, we evaluate its onchanges + if not field_elem.attrib.get('on_change', False): + continue + + result = recs.onchange(dict(record_dict, **parent_values), field_name, onchange_spec) + record_dict.update(process_vals(fg, { + key: val + for key, val in result.get('value', {}).iteritems() + if key not in fields # do not shadow values explicitly set in yaml + })) + + record_dict = post_process(fg, elems, record_dict) + + else: + record_dict = {} + + for field_name, expression in fields.iteritems(): + if record_dict.get(field_name): + continue + field_value = self._eval_field(model, field_name, expression, parent=record_dict, default=False, context=context) + record_dict[field_name] = field_value + + # filter returned values; indeed the last modification in the import process have added a default + # value for all fields in the view; however some fields present in the view are not stored and + # should not be sent to create. This bug appears with not stored function fields in the new API. + return { + key: val + for key, val in record_dict.iteritems() + for field in [model._fields[key].base_field] + if field.store or field.inverse + } + + def process_ref(self, node, field=None): + assert node.search or node.id, '!ref node should have a `search` attribute or `id` attribute' + if node.search: + if node.model: + model_name = node.model + elif field: + model_name = field.comodel_name + else: + raise YamlImportException('You need to give a model for the search, or a field to infer it.') + model = self.env[model_name] + q = safe_eval(node.search, self.eval_context) + instances = model.search(q) + if node.use: + value = [inst[node.use] for inst in instances] + else: + value = instances.ids + elif node.id: + if field and field.type == 'reference': + record = self.get_record(node.id) + value = "%s,%s" % (record._name, record.id) + else: + value = self.get_id(node.id) + else: + value = None + return value + + def process_eval(self, node): + return safe_eval(node.expression, self.eval_context) + + def _eval_field(self, model, field_name, expression, view_info=False, parent={}, default=True, context=None): + # TODO this should be refactored as something like model.get_field() in bin/osv + if field_name not in model._fields: + raise KeyError("Object '%s' does not contain field '%s'" % (model, field_name)) + field = model._fields[field_name] + + if is_ref(expression): + elements = self.process_ref(expression, field) + if field.type in ("many2many", "one2many"): + value = [(6, 0, elements)] + else: # many2one or reference + if isinstance(elements, (list,tuple)): + value = self._get_first_result(elements) + else: + value = elements + elif field.type == "many2one": + value = self.get_id(expression) + elif field.type == "one2many": + comodel = self.env[field.comodel_name] + value = [(0, 0, self._create_record(comodel, fields, view_info, parent=parent, default=default, context=context)) for fields in expression] + elif field.type == "many2many": + ids = [self.get_id(xml_id) for xml_id in expression] + value = [(6, 0, ids)] + elif field.type == "date" and is_string(expression): + # enforce ISO format for string date values, to be locale-agnostic during tests + time.strptime(expression, DEFAULT_SERVER_DATE_FORMAT) + value = expression + elif field.type == "datetime" and is_string(expression): + # enforce ISO format for string datetime values, to be locale-agnostic during tests + time.strptime(expression, DEFAULT_SERVER_DATETIME_FORMAT) + value = expression + elif field.type == "reference": + record = self.get_record(expression) + value = "%s,%s" % (record._name, record.id) + else: # scalar field + if is_eval(expression): + value = self.process_eval(expression) + else: + value = expression + # raise YamlImportException('Unsupported field "%s" or value %s:%s' % (field_name, type(expression), expression)) + return value + + def process_context(self, node): + self.context = node.__dict__ + if node.uid: + self.uid = self.get_id(node.uid) + if node.noupdate: + self.noupdate = node.noupdate + self.env = openerp.api.Environment(self.cr, self.uid, self.context) + self.sudo_env = self.env(user=SUPERUSER_ID) + + def process_python(self, node): + python, statements = node.items()[0] + assert python.model or python.id, "!python node must have attribute `model` or `id`" + if python.id is None: + record = self.env[python.model] + elif isinstance(python.id, basestring): + record = self.get_record(python.id) + else: + record = self.env[python.model].browse(python.id) + if python.model: + assert record._name == python.model, "`id` is not consistent with `model`" + statements = "\n" * python.first_line + statements.replace("\r\n", "\n") + code_context = { + 'self': record, + 'model': record, + 'cr': self.cr, + 'uid': self.uid, + 'log': self._log, + 'context': self.context, + 'openerp': openerp, + } + try: + code_obj = compile(statements, self.filename, 'exec') + unsafe_eval(code_obj, {'ref': self.get_id}, code_context) + except AssertionError, e: + self._log_assert_failure('AssertionError in Python code %s (line %d): %s', + python.name, python.first_line, e) + return + except Exception, e: + _logger.debug('Exception during evaluation of !python block in yaml_file %s.', self.filename, exc_info=True) + raise + else: + self.assertion_report.record_success() + + def process_workflow(self, node): + workflow, values = node.items()[0] + if self.isnoupdate(workflow) and self.mode != 'init': + return + if workflow.ref: + id = self.get_id(workflow.ref) + else: + if not values: + raise YamlImportException('You must define a child node if you do not give a ref.') + if not len(values) == 1: + raise YamlImportException('Only one child node is accepted (%d given).' % len(values)) + value = values[0] + if not 'model' in value and (not 'eval' in value or not 'search' in value): + raise YamlImportException('You must provide a "model" and an "eval" or "search" to evaluate.') + value_model = self.env[value['model']] + local_context = {'obj': value_model.browse} + local_context.update(self.id_map) + id = safe_eval(value['eval'], self.eval_context, local_context) + + if workflow.uid is not None: + uid = workflow.uid + else: + uid = self.uid + self.cr.execute('select distinct signal, sequence, id from wkf_transition ORDER BY sequence,id') + signals=[x['signal'] for x in self.cr.dictfetchall()] + if workflow.action not in signals: + raise YamlImportException('Incorrect action %s. No such action defined' % workflow.action) + record = self.env(user=uid)[workflow.model].browse(id) + record.signal_workflow(workflow.action) + + def _eval_params(self, model, params): + args = [] + for i, param in enumerate(params): + if isinstance(param, types.ListType): + value = self._eval_params(model, param) + elif is_ref(param): + value = self.process_ref(param) + elif is_eval(param): + value = self.process_eval(param) + elif isinstance(param, types.DictionaryType): # supports XML syntax + param_model = self.env[param.get('model', model)] + if 'search' in param: + q = safe_eval(param['search'], self.eval_context) + ids = param_model.search(q).ids + value = self._get_first_result(ids) + elif 'eval' in param: + local_context = {'obj': param_model.browse} + local_context.update(self.id_map) + value = safe_eval(param['eval'], self.eval_context, local_context) + else: + raise YamlImportException('You must provide either a !ref or at least a "eval" or a "search" to function parameter #%d.' % i) + else: + value = param # scalar value + args.append(value) + return args + + def process_function(self, node): + function, params = node.items()[0] + if self.isnoupdate(function) and self.mode != 'init': + return + model = self.env[function.model] + if function.eval: + args = self.process_eval(function.eval) + else: + args = self._eval_params(function.model, params) + # this one still depends on the old API + return openerp.api.call_kw(model, function.name, args, {}) + + def _set_group_values(self, node, values): + if node.groups: + group_names = node.groups.split(',') + groups_value = [] + for group in group_names: + if group.startswith('-'): + group_id = self.get_id(group[1:]) + groups_value.append((3, group_id)) + else: + group_id = self.get_id(group) + groups_value.append((4, group_id)) + values['groups_id'] = groups_value + + def process_menuitem(self, node): + self.validate_xml_id(node.id) + + if not node.parent: + parent_id = False + self.cr.execute('select id from ir_ui_menu where parent_id is null and name=%s', (node.name,)) + res = self.cr.fetchone() + values = {'parent_id': parent_id, 'name': node.name} + else: + parent_id = self.get_id(node.parent) + values = {'parent_id': parent_id} + if node.name: + values['name'] = node.name + try: + res = [ self.get_id(node.id) ] + except: # which exception ? + res = None + + if node.action: + action = self.get_record(node.action) + values['action'] = '%s,%s' % (action._name, action.id) + if not values.get('name'): + values['name'] = action.name + + if node.sequence: + values['sequence'] = node.sequence + + self._set_group_values(node, values) + + pid = self.sudo_env['ir.model.data']._update('ir.ui.menu', self.module, values, node.id, \ + mode=self.mode, noupdate=self.isnoupdate(node), res_id=res and res[0] or False) + + if node.id and pid: + self.id_map[node.id] = int(pid) + + def process_act_window(self, node): + assert getattr(node, 'id'), "Attribute %s of act_window is empty !" % ('id',) + assert getattr(node, 'name'), "Attribute %s of act_window is empty !" % ('name',) + assert getattr(node, 'res_model'), "Attribute %s of act_window is empty !" % ('res_model',) + self.validate_xml_id(node.id) + view_id = False + if node.view: + view_id = self.get_id(node.view) + if not node.context: + node.context={} + context = safe_eval(str(node.context), self.eval_context) + values = { + 'name': node.name, + 'type': node.type or 'ir.actions.act_window', + 'view_id': view_id, + 'domain': node.domain, + 'context': context, + 'res_model': node.res_model, + 'src_model': node.src_model, + 'view_type': node.view_type or 'form', + 'view_mode': node.view_mode or 'tree,form', + 'usage': node.usage, + 'limit': node.limit, + 'multi': getattr(node, 'multi', False), + } + + self._set_group_values(node, values) + + if node.target: + values['target'] = node.target + id = self.sudo_env['ir.model.data']._update('ir.actions.act_window', self.module, values, node.id, mode=self.mode) + self.id_map[node.id] = int(id) + + if node.src_model: + keyword = 'client_action_relate' + value = 'ir.actions.act_window,%s' % id + res_id = False + model = node.src_model + if isinstance(model, (list, tuple)): + model, res_id = model + self.env['ir.values'].sudo().set_action(node.id, action_slot=keyword, model=model, action=value, res_id=res_id) + # TODO add remove ir.model.data + + def process_delete(self, node): + assert getattr(node, 'model'), "Attribute %s of delete tag is empty !" % ('model',) + if node.model in self.env: + if node.search: + records = self.env[node.model].search(safe_eval(node.search, self.eval_context)) + else: + records = self.env[node.model].browse(self.get_id(node.id)) + if records: + records.unlink() + else: + self._log("Record not deleted.") + + def process_url(self, node): + self.validate_xml_id(node.id) + + res = {'name': node.name, 'url': node.url, 'target': node.target} + + id = self.sudo_env['ir.model.data']._update("ir.actions.act_url", self.module, res, node.id, mode=self.mode) + self.id_map[node.id] = int(id) + # ir_set + if (not node.menu or safe_eval(node.menu)) and id: + keyword = node.keyword or 'client_action_multi' + value = 'ir.actions.act_url,%s' % id + self.env['ir.values'].sudo().set_action(node.url, action_slot=keyword, model="ir.actions.act_url", action=value, res_id=False) + + def process_ir_set(self, node): + if not self.mode == 'init': + return False + _, fields = node.items()[0] + res = {} + for fieldname, expression in fields.items(): + if is_eval(expression): + value = safe_eval(expression.expression, self.eval_context) + else: + value = expression + res[fieldname] = value + ir_values = self.env['ir.values'] + for model in res['models']: + res_id = False + if isinstance(model, (list, tuple)): + model, res_id = model + if res['key'] == 'default': + ir_values.sudo().set_default(model, field_name=res['name'], value=res['value'], condition=res['key2']) + elif res['key'] == 'action': + ir_values.sudo().set_action(res['name'], action_slot=res['key2'], model=model, action=res['value'], res_id=res_id) + + def process_report(self, node): + values = {} + for dest, f in (('name','string'), ('model','model'), ('report_name','name')): + values[dest] = getattr(node, f) + assert values[dest], "Attribute %s of report is empty !" % (f,) + for field,dest in (('rml','report_rml'),('file','report_rml'),('xml','report_xml'),('xsl','report_xsl'),('attachment','attachment'),('attachment_use','attachment_use')): + if getattr(node, field): + values[dest] = getattr(node, field) + if node.auto: + values['auto'] = safe_eval(node.auto) + if node.sxw: + sxw_file = file_open(node.sxw) + try: + sxw_content = sxw_file.read() + values['report_sxw_content'] = sxw_content + finally: + sxw_file.close() + if node.header: + values['header'] = safe_eval(node.header) + values['multi'] = node.multi and safe_eval(node.multi) + xml_id = node.id + self.validate_xml_id(xml_id) + + self._set_group_values(node, values) + + id = self.sudo_env['ir.model.data']._update("ir.actions.report.xml", \ + self.module, values, xml_id, noupdate=self.isnoupdate(node), mode=self.mode) + self.id_map[xml_id] = int(id) + + if not node.menu or safe_eval(node.menu): + keyword = node.keyword or 'client_print_multi' + value = 'ir.actions.report.xml,%s' % id + ir_values = self.env['ir.values'] + res_id = False + model = values['model'] + if isinstance(model, (list, tuple)): + model, res_id = model + ir_values.sudo().set_action(values['name'], action_slot=keyword, model=model, action=value, res_id=res_id) + + def process_none(self): + """ + Empty node or commented node should not pass silently. + """ + self._log_assert_failure("You have an empty block in your tests.") + + + def process(self, yaml_string): + """ + Processes a Yaml string. Custom tags are interpreted by 'process_' instance methods. + """ + yaml_tag.add_constructors() + + is_preceded_by_comment = False + for node in yaml.load(yaml_string): + is_preceded_by_comment = self._log_node(node, is_preceded_by_comment) + try: + self._process_node(node) + except Exception, e: + _logger.exception(e) + raise + + def _process_node(self, node): + if is_comment(node): + self.process_comment(node) + elif is_assert(node): + self.process_assert(node) + elif is_record(node): + self.process_record(node) + elif is_python(node): + self.process_python(node) + elif is_menuitem(node): + self.process_menuitem(node) + elif is_delete(node): + self.process_delete(node) + elif is_url(node): + self.process_url(node) + elif is_context(node): + self.process_context(node) + elif is_ir_set(node): + self.process_ir_set(node) + elif is_act_window(node): + self.process_act_window(node) + elif is_report(node): + self.process_report(node) + elif is_workflow(node): + if isinstance(node, types.DictionaryType): + self.process_workflow(node) + else: + self.process_workflow({node: []}) + elif is_function(node): + if isinstance(node, types.DictionaryType): + self.process_function(node) + else: + self.process_function({node: []}) + elif node is None: + self.process_none() + else: + raise YamlImportException("Can not process YAML block: %s" % node) + + def _log_node(self, node, is_preceded_by_comment): + if is_comment(node): + is_preceded_by_comment = True + self._log(node) + elif not is_preceded_by_comment: + if isinstance(node, types.DictionaryType): + msg = "Creating %s\n with %s" + args = node.items()[0] + self._log(msg, *args) + else: + self._log(node) + else: + is_preceded_by_comment = False + return is_preceded_by_comment + +def yaml_import(cr, module, yamlfile, kind, idref=None, mode='init', noupdate=False, report=None): + if idref is None: + idref = {} + loglevel = logging.DEBUG + yaml_string = yamlfile.read() + yaml_interpreter = YamlInterpreter(cr, module, idref, mode, filename=yamlfile.name, report=report, noupdate=noupdate, loglevel=loglevel) + yaml_interpreter.process(yaml_string) + +# keeps convention of convert.py +convert_yaml_import = yaml_import diff --git a/odoo/tools/yaml_tag.py b/odoo/tools/yaml_tag.py new file mode 100644 index 00000000000..863394e317c --- /dev/null +++ b/odoo/tools/yaml_tag.py @@ -0,0 +1,186 @@ +import yaml +import logging + +class YamlTag(object): + """ + Superclass for constructors of custom tags defined in yaml file. + __str__ is overriden in subclass and used for serialization in module recorder. + """ + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + def __getitem__(self, key): + return getattr(self, key) + def __getattr__(self, attr): + return None + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, sorted(self.__dict__.items())) + +class Assert(YamlTag): + def __init__(self, model, id=None, severity=logging.WARNING, string="NONAME", **kwargs): + self.model = model + self.id = id + self.severity = severity + self.string = string + super(Assert, self).__init__(**kwargs) + +class Record(YamlTag): + def __init__(self, model, id, use='id', view=True, **kwargs): + self.model = model + self.id = id + self.view = view + super(Record, self).__init__(**kwargs) + def __str__(self): + return '!record {model: %s, id: %s}:' % (str(self.model,), str(self.id,)) + +class Python(YamlTag): + def __init__(self, model, severity=logging.ERROR, name="", **kwargs): + self.model= model + self.severity = severity + self.name = name + super(Python, self).__init__(**kwargs) + def __str__(self): + return '!python {model: %s}: |' % (str(self.model), ) + +class Menuitem(YamlTag): + def __init__(self, id, name, **kwargs): + self.id = id + self.name = name + super(Menuitem, self).__init__(**kwargs) + +class Workflow(YamlTag): + def __init__(self, model, action, ref=None, **kwargs): + self.model = model + self.action = action + self.ref = ref + super(Workflow, self).__init__(**kwargs) + def __str__(self): + return '!workflow {model: %s, action: %s, ref: %s}' % (str(self.model,), str(self.action,), str(self.ref,)) + +class ActWindow(YamlTag): + def __init__(self, **kwargs): + super(ActWindow, self).__init__(**kwargs) + +class Function(YamlTag): + def __init__(self, model, name, **kwargs): + self.model = model + self.name = name + super(Function, self).__init__(**kwargs) + +class Report(YamlTag): + def __init__(self, model, name, string, **kwargs): + self.model = model + self.name = name + self.string = string + super(Report, self).__init__(**kwargs) + +class Delete(YamlTag): + def __init__(self, **kwargs): + super(Delete, self).__init__(**kwargs) + +class Context(YamlTag): + def __init__(self, **kwargs): + super(Context, self).__init__(**kwargs) + +class Url(YamlTag): + def __init__(self, **kwargs): + super(Url, self).__init__(**kwargs) + +class Eval(YamlTag): + def __init__(self, expression): + self.expression = expression + super(Eval, self).__init__() + def __str__(self): + return '!eval %s' % str(self.expression) + +class Ref(YamlTag): + def __init__(self, expr="False", *args, **kwargs): + self.expr = expr + super(Ref, self).__init__(*args, **kwargs) + def __str__(self): + return 'ref(%s)' % repr(self.expr) + +class IrSet(YamlTag): + def __init__(self): + super(IrSet, self).__init__() + +def assert_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Assert(**kwargs) + +def record_constructor(loader, node): + kwargs = loader.construct_mapping(node) + assert "model" in kwargs, "'model' argument is required for !record" + assert "id" in kwargs, "'id' argument is required for !record" + return Record(**kwargs) + +def python_constructor(loader, node): + kwargs = loader.construct_mapping(node) + kwargs['first_line'] = node.start_mark.line + 1 + return Python(**kwargs) + +def menuitem_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Menuitem(**kwargs) + +def workflow_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Workflow(**kwargs) + +def act_window_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return ActWindow(**kwargs) + +def function_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Function(**kwargs) + +def report_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Report(**kwargs) + +def delete_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Delete(**kwargs) + +def context_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Context(**kwargs) + +def url_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return Url(**kwargs) + +def eval_constructor(loader, node): + expression = loader.construct_scalar(node) + return Eval(expression) + +def ref_constructor(loader, tag_suffix, node): + if tag_suffix == "id": + kwargs = {"id": loader.construct_scalar(node)} + else: + kwargs = loader.construct_mapping(node) + return Ref(**kwargs) + +def ir_set_constructor(loader, node): + kwargs = loader.construct_mapping(node) + return IrSet(**kwargs) + +# Registers constructors for custom tags. +# Constructors are actually defined globally: do not redefined them in another +# class/file/package. This means that module recorder need import this file. +def add_constructors(): + yaml.add_constructor(u"!assert", assert_constructor) + yaml.add_constructor(u"!record", record_constructor) + yaml.add_constructor(u"!python", python_constructor) + yaml.add_constructor(u"!menuitem", menuitem_constructor) + yaml.add_constructor(u"!workflow", workflow_constructor) + yaml.add_constructor(u"!act_window", act_window_constructor) + yaml.add_constructor(u"!function", function_constructor) + yaml.add_constructor(u"!report", report_constructor) + yaml.add_constructor(u"!context", context_constructor) + yaml.add_constructor(u"!delete", delete_constructor) + yaml.add_constructor(u"!url", url_constructor) + yaml.add_constructor(u"!eval", eval_constructor) + yaml.add_multi_constructor(u"!ref", ref_constructor) + yaml.add_constructor(u"!ir_set", ir_set_constructor) +add_constructors() |