[MERGE] from trunk

bzr revid: rvalyi@gmail.com-20100201130334-mgd708bztw2uxya9

3 files changed, 230 insertions, 19 deletions

diff --git a/bin/tools/config.py b/bin/tools/config.py
index 4cdee8736c2..68fd732090f 100644
--- a/bin/tools/config.py
+++ b/bin/tools/config.py
@@ -77,10 +77,11 @@ class configmanager(object):
             'secure' : False,
             'syslog' : False,
             'log_level': logging.INFO,
-            'assert_exit_level': logging.WARNING, # level above which a failed assert will be raise
+            'assert_exit_level': logging.WARNING, # level above which a failed assert will be raised
             'cache_timeout': 100000,
             'login_message': False,
             'list_db' : True,
+            'timezone' : False, # to override the default TZ
         }
     
         self.misc = {}
@@ -111,6 +112,7 @@ class configmanager(object):
                           help="update a module (use \"all\" for all modules)")
         parser.add_option("--cache-timeout", dest="cache_timeout",
                           help="set the timeout for the cache system", default=100000, type="int")
+        parser.add_option("-t", "--timezone", dest="timezone", help="specify reference timezone for the server (e.g. Europe/Brussels")
 
         # stops the server from launching after initialization
         parser.add_option("--stop-after-init", action="store_true", dest="stop_after_init", default=False,
@@ -119,6 +121,8 @@ class configmanager(object):
         parser.add_option("--assert-exit-level", dest='assert_exit_level', type="choice", choices=self._LOGLEVELS.keys(),
                           help="specify the level at which a failed assertion will stop the server. Accepted values: %s" % (self._LOGLEVELS.keys(),))
         parser.add_option('--price_accuracy', dest='price_accuracy', default='2', help='specify the price accuracy')
+        parser.add_option('--no-database-list', action="store_false", dest='list_db', default=True, help="disable the ability to return the list of databases")
+
         if hasSSL:
             group = optparse.OptionGroup(parser, "SSL Configuration")
             group.add_option("-S", "--secure", dest="secure",
@@ -162,7 +166,6 @@ class configmanager(object):
         group.add_option("--db_port", dest="db_port", help="specify the database port", type="int")
         group.add_option("--db_maxconn", dest="db_maxconn", default='64',
                          help="specify the the maximum number of physical connections to posgresql")
-        group.add_option("--list_db", dest="list_db", default=False, help="This option hides Database list for security purpose:\n \'False\' value disables listing of Databases")
         group.add_option("-P", "--import-partial", dest="import_partial",
                          help="Use this for big data importation, if it crashes you will be able to continue at the current state. Provide a filename to store intermediate importation states.", default=False)
         parser.add_option_group(group)
@@ -227,7 +230,7 @@ class configmanager(object):
                 'db_port', 'list_db', 'logfile', 'pidfile', 'smtp_port', 'cache_timeout',
                 'email_from', 'smtp_server', 'smtp_user', 'smtp_password', 'price_accuracy',
                 'netinterface', 'netport', 'db_maxconn', 'import_partial', 'addons_path', 
-                'netrpc', 'xmlrpc', 'syslog', 'without_demo']
+                'netrpc', 'xmlrpc', 'syslog', 'without_demo', 'timezone',]
 
         if hasSSL:
             keys.extend(['smtp_ssl', 'secure_cert_file', 'secure_pkey_file'])
@@ -238,7 +241,10 @@ class configmanager(object):
                 self.options[arg] = getattr(opt, arg)
 
         keys = ['language', 'translate_out', 'translate_in', 'debug_mode',
-                'stop_after_init', 'logrotate']
+                'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog', 'list_db']
+
+        if hasSSL and not  self.options['secure']:
+            keys.append('secure')
 
         for arg in keys:
             if getattr(opt, arg) is not None:
@@ -266,6 +272,17 @@ class configmanager(object):
         self.options['translate_modules'] = opt.translate_modules and map(lambda m: m.strip(), opt.translate_modules.split(',')) or ['all']
         self.options['translate_modules'].sort()
 
+        if self.options['timezone']:
+            # If an explicit TZ was provided in the config, make sure it is known  
+            try:
+                import pytz
+                tz = pytz.timezone(self.options['timezone'])
+            except pytz.UnknownTimeZoneError:
+                die(True, "The specified timezone (%s) is invalid" % self.options['timezone'])
+            except:
+                # If pytz is missing, don't check the provided TZ, it will be ignored anyway.
+                pass
+
         if opt.pg_path:
             self.options['pg_path'] = opt.pg_path
 
@@ -330,6 +347,18 @@ class configmanager(object):
         res = os.path.abspath(os.path.expanduser(value))
         if not os.path.exists(res):
             raise optparse.OptionValueError("option %s: no such directory: %r" % (opt, value))
+        
+        contains_addons = False
+        for f in os.listdir(res):
+            modpath = os.path.join(res, f)
+            if os.path.isdir(modpath) and os.path.exists(os.path.join(modpath, '__init__.py')) and \
+            os.path.exists(os.path.join(modpath, '__terp__.py')):
+                contains_addons = True
+                break
+
+        if not contains_addons:
+            raise optparse.OptionValueError("option %s: The addons-path %r does not seem to a be a valid Addons Directory!" % (opt, value))
+        
         setattr(parser.values, option.dest, res)
 
     def load(self):
diff --git a/bin/tools/misc.py b/bin/tools/misc.py
index c3394c2ed4f..2984fb09186 100644
--- a/bin/tools/misc.py
+++ b/bin/tools/misc.py
@@ -31,6 +31,7 @@ from config import config
 import zipfile
 import release
 import socket
+import re
 
 if sys.version_info[:2] < (2, 4):
     from threadinglocal import local
@@ -302,8 +303,103 @@ def reverse_enumerate(l):
 #----------------------------------------------------------
 # Emails
 #----------------------------------------------------------
+email_re = re.compile(r"""
+    ([a-zA-Z][\w\.-]*[a-zA-Z0-9]     # username part
+    @                                # mandatory @ sign
+    [a-zA-Z0-9][\w\.-]*              # domain must start with a letter ... Ged> why do we include a 0-9 then?
+     \.
+     [a-z]{2,3}                      # TLD
+    )
+    """, re.VERBOSE)
+res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE)
+command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE)
+reference_re = re.compile("<.*-openobject-(\\d+)@(.*)>", re.UNICODE)
+
+priorities = {
+        '1': '1 (Highest)',
+        '2': '2 (High)',
+        '3': '3 (Normal)',
+        '4': '4 (Low)',
+        '5': '5 (Lowest)',
+    }
+
+def html2plaintext(html, body_id=None, encoding='utf-8'):
+    ## (c) Fry-IT, www.fry-it.com, 2007
+    ## <peter@fry-it.com>
+    ## download here: http://www.peterbe.com/plog/html2plaintext
+    
+    
+    """ from an HTML text, convert the HTML to plain text.
+    If @body_id is provided then this is the tag where the 
+    body (not necessarily <body>) starts.
+    """
+    try:
+        from BeautifulSoup import BeautifulSoup, SoupStrainer, Comment
+    except:
+        return html
+            
+    urls = []
+    if body_id is not None:
+        strainer = SoupStrainer(id=body_id)
+    else:
+        strainer = SoupStrainer('body')
+    
+    soup = BeautifulSoup(html, parseOnlyThese=strainer, fromEncoding=encoding)
+    for link in soup.findAll('a'):
+        title = link.renderContents()
+        for url in [x[1] for x in link.attrs if x[0]=='href']:
+            urls.append(dict(url=url, tag=str(link), title=title))
+
+    html = soup.__str__()
+            
+    url_index = []
+    i = 0
+    for d in urls:
+        if d['title'] == d['url'] or 'http://'+d['title'] == d['url']:
+            html = html.replace(d['tag'], d['url'])
+        else:
+            i += 1
+            html = html.replace(d['tag'], '%s [%s]' % (d['title'], i))
+            url_index.append(d['url'])
+
+    html = html.replace('<strong>','*').replace('</strong>','*')
+    html = html.replace('<b>','*').replace('</b>','*')
+    html = html.replace('<h3>','*').replace('</h3>','*')
+    html = html.replace('<h2>','**').replace('</h2>','**')
+    html = html.replace('<h1>','**').replace('</h1>','**')
+    html = html.replace('<em>','/').replace('</em>','/')
+    
+
+    # the only line breaks we respect is those of ending tags and 
+    # breaks
+    
+    html = html.replace('\n',' ')
+    html = html.replace('<br>', '\n')
+    html = html.replace('<tr>', '\n')
+    html = html.replace('</p>', '\n\n')
+    html = re.sub('<br\s*/>', '\n', html)
+    html = html.replace(' ' * 2, ' ')
+
+
+    # for all other tags we failed to clean up, just remove then and 
+    # complain about them on the stderr
+    def desperate_fixer(g):
+        #print >>sys.stderr, "failed to clean up %s" % str(g.group())
+        return ' '
+
+    html = re.sub('<.*?>', desperate_fixer, html)
+
+    # lstrip all lines
+    html = '\n'.join([x.lstrip() for x in html.splitlines()])
+
+    for i, url in enumerate(url_index):
+        if i == 0:
+            html += '\n\n'
+        html += '[%s] %s\n' % (i+1, url)       
+    return html
+
 def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False,
-               attach=None, tinycrm=False, ssl=False, debug=False, subtype='plain', x_headers=None):
+               attach=None, openobject_id=False, ssl=False, debug=False, subtype='plain', x_headers=None, priority='3'):
 
     """Send an email."""
     import smtplib
@@ -314,7 +410,7 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non
     from email.Utils import formatdate, COMMASPACE
     from email.Utils import formatdate, COMMASPACE
     from email import Encoders
-    import netsvc
+    import netsvc    
 
     if x_headers is None:
         x_headers = {}
@@ -325,6 +421,9 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non
     if not email_from and not config['email_from']:
         raise Exception("No Email sender by default, see config file")
 
+    if not email_from:
+        email_from = config.get('email_from', False)
+
     if not email_cc:
         email_cc = []
     if not email_bcc:
@@ -356,13 +455,15 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non
     msg['X-Generated-By'] = 'OpenERP (http://www.openerp.com)'
     msg['X-OpenERP-Server-Host'] = socket.gethostname()
     msg['X-OpenERP-Server-Version'] = release.version
+    if priority:
+        msg['X-Priority'] = priorities.get(priority, '3 (Normal)')
 
     # Add dynamic X Header
     for key, value in x_headers.items():
         msg['X-OpenERP-%s' % key] = str(value)
 
-    if tinycrm:
-        msg['Message-Id'] = "<%s-tinycrm-%s@%s>" % (time.time(), tinycrm, socket.gethostname())
+    if openobject_id:
+        msg['Message-Id'] = "<%s-openobject-%s@%s>" % (time.time(), openobject_id, socket.gethostname())
 
     if attach:
         try:
@@ -393,12 +494,11 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non
             return True
         except Exception,e:
             netsvc.Logger().notifyChannel('email_send (maildir)', netsvc.LOG_ERROR, e)
-            return False
+            return False    
     
     try:
         oldstderr = smtplib.stderr
         s = smtplib.SMTP()
-
         try:
             # in case of debug, the messages are printed to stderr.
             if debug:
@@ -412,13 +512,11 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non
                 s.ehlo()
 
             if config['smtp_user'] or config['smtp_password']:
-                s.login(config['smtp_user'], config['smtp_password'])
-
+                s.login(config['smtp_user'], config['smtp_password'])            
             s.sendmail(email_from,
                        flatten([email_to, email_cc, email_bcc]),
                        msg.as_string()
                       )
-
         finally:
             s.quit()
             if debug:
@@ -538,9 +636,6 @@ class UpdateableDict(local):
     def __ge__(self, y):
         return self.dict.__ge__(y)
 
-    def __getitem__(self, y):
-        return self.dict.__getitem__(y)
-
     def __gt__(self, y):
         return self.dict.__gt__(y)
 
@@ -1076,6 +1171,89 @@ def detect_ip_addr():
         ip_addr = 'localhost'
     return ip_addr
 
+# RATIONALE BEHIND TIMESTAMP CALCULATIONS AND TIMEZONE MANAGEMENT:
+#  The server side never does any timestamp calculation, always
+#  sends them in a naive (timezone agnostic) format supposed to be
+#  expressed within the server timezone, and expects the clients to 
+#  provide timestamps in the server timezone as well.
+#  It stores all timestamps in the database in naive format as well, 
+#  which also expresses the time in the server timezone.
+#  For this reason the server makes its timezone name available via the
+#  common/timezone_get() rpc method, which clients need to read
+#  to know the appropriate time offset to use when reading/writing
+#  times.
+def get_win32_timezone():
+    """Attempt to return the "standard name" of the current timezone on a win32 system.
+       @return: the standard name of the current win32 timezone, or False if it cannot be found.
+    """
+    res = False
+    if (sys.platform == "win32"):
+        try:
+            import _winreg
+            hklm = _winreg.ConnectRegistry(None,_winreg.HKEY_LOCAL_MACHINE)
+            current_tz_key = _winreg.OpenKey(hklm, r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation", 0,_winreg.KEY_ALL_ACCESS)
+            res = str(_winreg.QueryValueEx(current_tz_key,"StandardName")[0])  # [0] is value, [1] is type code
+            _winreg.CloseKey(current_tz_key)
+            _winreg.CloseKey(hklm)
+        except:
+            pass
+    return res
+
+def detect_server_timezone():
+    """Attempt to detect the timezone to use on the server side.
+       Defaults to UTC if no working timezone can be found.
+       @return: the timezone identifier as expected by pytz.timezone.
+    """
+    import time
+    import netsvc
+    try:
+        import pytz
+    except:
+        netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING,
+            "Python pytz module is not available. Timezone will be set to UTC by default.")
+        return 'UTC'
+
+    # Option 1: the configuration option (did not exist before, so no backwards compatibility issue)
+    # Option 2: to be backwards compatible with 5.0 or earlier, the value from time.tzname[0], but only if it is known to pytz
+    # Option 3: the environment variable TZ
+    sources = [ (config['timezone'], 'OpenERP configuration'),
+                (time.tzname[0], 'time.tzname'),
+                (os.environ.get('TZ',False),'TZ environment variable'), ]
+    # Option 4: OS-specific: /etc/timezone on Unix
+    if (os.path.exists("/etc/timezone")):
+        tz_value = False
+        try:
+            f = open("/etc/timezone")
+            tz_value = f.read(128).strip()
+        except:
+            pass
+        finally:
+            f.close()
+        sources.append((tz_value,"/etc/timezone file"))
+    # Option 5: timezone info from registry on Win32
+    if (sys.platform == "win32"):
+        # Timezone info is stored in windows registry.
+        # However this is not likely to work very well as the standard name
+        # of timezones in windows is rarely something that is known to pytz.
+        # But that's ok, it is always possible to use a config option to set
+        # it explicitly.
+        sources.append((get_win32_timezone(),"Windows Registry"))
+
+    for (value,source) in sources:
+        if value:
+            try:
+                tz = pytz.timezone(value)
+                netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_INFO,
+                    "Using timezone %s obtained from %s." % (tz.zone,source))
+                return value
+            except pytz.UnknownTimeZoneError:
+                netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING,
+                    "The timezone specified in %s (%s) is invalid, ignoring it." % (source,value))
+
+    netsvc.Logger().notifyChannel("detect_server_timezone", netsvc.LOG_WARNING,
+        "No valid timezone could be detected, using default UTC timezone. You can specify it explicitly with option 'timezone' in the server configuration.")
+    return 'UTC'
+
 
 if __name__ == '__main__':
     import doctest
diff --git a/bin/tools/translate.py b/bin/tools/translate.py
index da718f0d9d1..7e2401462fe 100644
--- a/bin/tools/translate.py
+++ b/bin/tools/translate.py
@@ -155,8 +155,12 @@ _ = GettextAlias()
 # class to handle po files
 class TinyPoFile(object):
     def __init__(self, buffer):
+        self.logger = netsvc.Logger()
         self.buffer = buffer
 
+    def warn(self, msg):
+        self.logger.notifyChannel("i18n", netsvc.LOG_WARNING, msg)
+
     def __iter__(self):
         self.buffer.seek(0)
         self.lines = self._get_lines()
@@ -219,7 +223,6 @@ class TinyPoFile(object):
                 # This has been a deprecated entry, don't return anything
                 return self.next()
 
-
             if not line.startswith('msgid'):
                 raise Exception("malformed file: bad line: %s" % line)
             source = unquote(line[6:])
@@ -251,8 +254,9 @@ class TinyPoFile(object):
                     self.tnrs.append((t, n, r, source, trad))
 
         self.first = False
-    
-        if name == None:
+
+        if name is None:
+            self.warn('Missing "#:" formated comment for the following source:\n\t%s' % (source,))
             return self.next()
         return type, name, res_id, source, trad