[MERGE] sitemap backport, merge trunk from 9438 til 9440

merge from 9438 revid:fp@tinyerp.com-20140511115231-g8ke14r9iepyypv4 til 9440 revid:fp@tinyerp.com-20140511143314-nzn1239l00arbxjq

6 files changed, 107 insertions, 102 deletions

diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py
index b611496ef14..5068d7d0e74 100644
--- a/addons/website/controllers/main.py
+++ b/addons/website/controllers/main.py
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 # Completely arbitrary limits
 MAX_IMAGE_WIDTH, MAX_IMAGE_HEIGHT = IMAGE_LIMITS = (1024, 768)
-
+LOC_PER_SITEMAP = 45000
 
 class Website(openerp.addons.web.controllers.main.Home):
     #------------------------------------------------------
@@ -49,7 +49,7 @@ class Website(openerp.addons.web.controllers.main.Home):
         # TODO: can't we just put auth=public, ... in web client ?
         return super(Website, self).web_login(*args, **kw)
 
-    @http.route('/page/<page:page>', type='http', auth="public", website=True, multilang=True)
+    @http.route('/page/<path:page>', type='http', auth="public", website=True, multilang=True)
     def page(self, page, **opt):
         values = {
             'path': page,
@@ -69,20 +69,34 @@ class Website(openerp.addons.web.controllers.main.Home):
 
         return request.render(page, values)
 
-    @http.route(['/robots.txt'], type='http', auth="public", website=True)
+    @http.route(['/robots.txt'], type='http', auth="public")
     def robots(self):
         return request.render('website.robots', {'url_root': request.httprequest.url_root}, mimetype='text/plain')
 
-    @http.route('/sitemap', type='http', auth='public', website=True, multilang=True)
-    def sitemap(self):
-        return request.render('website.sitemap', {
-            'pages': request.website.enumerate_pages()
-        })
-
     @http.route('/sitemap.xml', type='http', auth="public", website=True)
-    def sitemap_xml(self):
+    def sitemap_xml_index(self):
+        pages = list(request.website.enumerate_pages())
+        if len(pages)<=LOC_PER_SITEMAP:
+            return self.__sitemap_xml(pages, 0)
+        # Sitemaps must be split in several smaller files with a sitemap index
+        values = {
+            'pages': range(len(pages)/LOC_PER_SITEMAP+1),
+            'url_root': request.httprequest.url_root
+        }
+        headers = {
+            'Content-Type': 'application/xml;charset=utf-8',
+        }
+        return request.render('website.sitemap_index_xml', values, headers=headers)
+
+    @http.route('/sitemap-<int:page>.xml', type='http', auth="public", website=True)
+    def sitemap_xml(self, page):
+        pages = list(request.website.enumerate_pages())
+        return self.__sitemap_xml(pages, page)
+
+    def __sitemap_xml(self, pages, index=0):
         values = {
-            'pages': request.website.enumerate_pages()
+            'pages': pages[index*LOC_PER_SITEMAP:(index+1)*LOC_PER_SITEMAP],
+            'url_root': request.httprequest.url_root.rstrip('/')
         }
         headers = {
             'Content-Type': 'application/xml;charset=utf-8',
@@ -450,4 +464,3 @@ class Website(openerp.addons.web.controllers.main.Home):
             return res
         return request.redirect('/')
 
-# vim:et:
diff --git a/addons/website/data/website_data.xml b/addons/website/data/website_data.xml
index 4eff1b8a60d..c5254d65c0f 100644
--- a/addons/website/data/website_data.xml
+++ b/addons/website/data/website_data.xml
@@ -15,7 +15,7 @@
 
         <record id="menu_homepage" model="website.menu">
             <field name="name">Home</field>
-            <field name="url">/</field>
+            <field name="url">/page/homepage</field>
             <field name="parent_id" ref="website.main_menu"/>
             <field name="sequence" type="int">10</field>
         </record>
diff --git a/addons/website/models/ir_http.py b/addons/website/models/ir_http.py
index acceed1e718..91b998c76c9 100644
--- a/addons/website/models/ir_http.py
+++ b/addons/website/models/ir_http.py
@@ -171,8 +171,9 @@ class ir_http(orm.AbstractModel):
         return super(ir_http, self)._handle_exception(exception)
 
 class ModelConverter(ir.ir_http.ModelConverter):
-    def __init__(self, url_map, model=False):
+    def __init__(self, url_map, model=False, domain='[]'):
         super(ModelConverter, self).__init__(url_map, model)
+        self.domain = domain
         self.regex = r'(?:[A-Za-z0-9-_]+?-)?(\d+)(?=$|/)'
 
     def to_url(self, value):
@@ -184,24 +185,31 @@ class ModelConverter(ir.ir_http.ModelConverter):
         return request.registry[self.model].browse(
             request.cr, _uid, int(m.group(1)), context=request.context)
 
-    def generate(self, cr, uid, query=None, context=None):
-        return request.registry[self.model].name_search(
-            cr, uid, name=query or '', context=context)
+    def generate(self, cr, uid, query=None, args=None, context=None):
+        obj = request.registry[self.model]
+        domain = eval( self.domain, (args or {}).copy())
+        if query:
+            domain.append((obj._rec_name, 'ilike', '%'+query+'%'))
+        for record in obj.search_read(cr, uid, domain=domain, fields=['write_date',obj._rec_name], context=context):
+            if record.get(obj._rec_name, False):
+                yield {'loc': (record['id'], record[obj._rec_name])}
 
 class PageConverter(werkzeug.routing.PathConverter):
-    """ Only point of this converter is to bundle pages enumeration logic
-
-    Sads got: no way to get the view's human-readable name even if one exists
-    """
-    def generate(self, cr, uid, query=None, context=None):
+    """ Only point of this converter is to bundle pages enumeration logic """
+    def generate(self, cr, uid, query=None, args={}, context=None):
         View = request.registry['ir.ui.view']
-        views = View.search_read(
-            cr, uid, [['page', '=', True]],
-            fields=[], order='name', context=context)
-        xids = View.get_external_id(
-            cr, uid, [view['id'] for view in views], context=context)
-
+        views = View.search_read(cr, uid, [['page', '=', True]],
+            fields=['xml_id','priority','write_date'], order='name', context=context)
         for view in views:
-            xid = xids[view['id']]
-            if xid and (not query or query.lower() in xid.lower()):
-                yield xid
+            xid = view['xml_id'].startswith('website.') and view['xml_id'][8:] or view['xml_id']
+            # the 'page/homepage' url is indexed as '/', avoid aving the same page referenced twice
+            # when we will have an url mapping mechanism, replace this by a rule: page/homepage --> /
+            if xid=='homepage': continue
+            if query and query.lower() not in xid.lower():
+                continue
+            record = {'loc': xid}
+            if view['priority'] <> 16:
+                record['__priority'] = min(round(view['priority'] / 32.0,1), 1)
+            if view.get('write_date'):
+                record['__lastmod'] = view['write_date'][:10]
+            yield record
diff --git a/addons/website/models/website.py b/addons/website/models/website.py
index 51dda75f516..584dc4b6105 100644
--- a/addons/website/models/website.py
+++ b/addons/website/models/website.py
@@ -296,44 +296,23 @@ class website(osv.osv):
         endpoint = rule.endpoint
         methods = rule.methods or ['GET']
         converters = rule._converters.values()
-
-        return (
-            'GET' in methods
+        if not ('GET' in methods
             and endpoint.routing['type'] == 'http'
             and endpoint.routing['auth'] in ('none', 'public')
             and endpoint.routing.get('website', False)
-            # preclude combinatorial explosion by only allowing a single converter
-            and len(converters) <= 1
-            # ensure all converters on the rule are able to generate values for
-            # themselves
             and all(hasattr(converter, 'generate') for converter in converters)
-        ) and self.endpoint_is_enumerable(rule)
-
-    def endpoint_is_enumerable(self, rule):
-        """ Verifies that it's possible to generate a valid url for the rule's
-        endpoint
-
-        :type rule: werkzeug.routing.Rule
-        :rtype: bool
-        """
-        spec = inspect.getargspec(rule.endpoint.method)
-
-        # if *args bail the fuck out, only dragons can live there
-        if spec.varargs:
+            and endpoint.routing.get('website')):
             return False
 
-        # remove all arguments with a default value from the list
-        defaults_count = len(spec.defaults or []) # spec.defaults can be None
-        # a[:-0] ~ a[:0] ~ [] -> replace defaults_count == 0 by None to get
-        # a[:None] ~ a
-        args = spec.args[:(-defaults_count or None)]
+        # dont't list routes without argument having no default value or converter
+        spec = inspect.getargspec(endpoint.method.original_func)
+
+        # remove self and arguments having a default value
+        defaults_count = len(spec.defaults or [])
+        args = spec.args[1:(-defaults_count or None)]
 
-        # params with defaults were removed, leftover allowed are:
-        # * self (technically should be first-parameter-of-instance-method but whatever)
-        # * any parameter mapping to a converter
-        return all(
-            (arg == 'self' or arg in rule._converters)
-            for arg in args)
+        # check that all args have a converter
+        return all( (arg in rule._converters) for arg in args)
 
     def enumerate_pages(self, cr, uid, ids, query_string=None, context=None):
         """ Available pages in the website/CMS. This is mostly used for links
@@ -357,27 +336,34 @@ class website(osv.osv):
             if not self.rule_is_enumerable(rule):
                 continue
 
-            converters = rule._converters
-            filtered = bool(converters)
-            if converters:
-                # allow single converter as decided by fp, checked by
-                # rule_is_enumerable
-                [(name, converter)] = converters.items()
-                converter_values = converter.generate(
-                    request.cr, uid, query=query_string, context=context)
-                generated = ({k: v} for k, v in itertools.izip(
-                    itertools.repeat(name), converter_values))
-            else:
-                # force single iteration for literal urls
-                generated = [{}]
-
-            for values in generated:
-                domain_part, url = rule.build(values, append_unknown=False)
-                page = {'name': url, 'url': url}
+            converters = rule._converters or {}
+            values = [{}]
+            convitems = converters.items()
+            # converters with a domain are processed after the other ones
+            gd = lambda x: hasattr(x[1], 'domain') and (x[1].domain <> '[]')
+            convitems.sort(lambda x, y: cmp(gd(x), gd(y)))
+            for (name, converter) in convitems:
+                newval = []
+                for val in values:
+                    for v in converter.generate(request.cr, uid, query=query_string, args=val, context=context):
+                        newval.append( val.copy() )
+                        v[name] = v['loc']
+                        del v['loc']
+                        newval[-1].update(v)
+                values = newval
+
+            for value in values:
+                domain_part, url = rule.build(value, append_unknown=False)
+                page = {'loc': url}
+                for key,val in value.items():
+                    if key.startswith('__'):
+                        page[key[2:]] = val
+                if url in ('/sitemap.xml',):
+                    continue
                 if url in url_list:
                     continue
                 url_list.append(url)
-                if not filtered and query_string and not self.page_matches(cr, uid, page, query_string, context=context):
+                if query_string and not self.page_matches(cr, uid, page, query_string, context=context):
                     continue
                 yield page
 
diff --git a/addons/website/tests/test_requests.py b/addons/website/tests/test_requests.py
index e0309c9ab4b..2a1eea4c877 100644
--- a/addons/website/tests/test_requests.py
+++ b/addons/website/tests/test_requests.py
@@ -96,7 +96,7 @@ class CrawlSuite(unittest2.TestSuite):
             # switch registry to test mode, so that requests can be made
             registry.enter_test_mode()
 
-            paths = [URL('/'), URL('/sitemap')]
+            paths = [URL('/')]
             seen = set(paths)
 
             while paths:
diff --git a/addons/website/views/website_templates.xml b/addons/website/views/website_templates.xml
index de31f13bd66..b9f33ef1c6b 100644
--- a/addons/website/views/website_templates.xml
+++ b/addons/website/views/website_templates.xml
@@ -16,7 +16,7 @@
         ((submenu.url != '/' and request.httprequest.path.startswith(submenu.url)) or
          request.httprequest.path == submenu.url) and 'active'
         ">
-        <a t-att-href="submenu.url" t-ignore="true" t-att-target="'blank' if submenu.new_window else None">
+        <a t-att-href="(website.menu_id.child_id[0] == submenu) and '/' or submenu.url" t-ignore="true" t-att-target="'blank' if submenu.new_window else None">
             <span t-field="submenu.name"/>
         </a>
     </li>
@@ -458,7 +458,6 @@
             <div class="well mt32">
                 <p>This page does not exists, but you can create it as you are administrator of this site.</p>
                 <a class="btn btn-primary" t-attf-href="/website/add/#{ path }">Create Page</a>
-                <span class="text-muted">or</span> <a href="/sitemap">Search a Page</a>
             </div>
             <div class="text-center text-muted">Edit the content below this line to adapt the default "page not found" page.</div>
         </div>
@@ -694,34 +693,33 @@
 </template>
 
 <template id="robots">
-# robotstxt.org/
 User-agent: *
 Sitemap: <t t-esc="url_root"/>sitemap.xml
 </template>
 
-<template id="sitemap" name="Site Map">
-    <t t-call="website.layout">
-        <ul>
-            <li t-foreach="pages" t-as="page">
-                <a t-att-href="page['url']"><t t-esc="page['name']"/></a>
-            </li>
-        </ul>
-    </t>
+<template id="sitemap_xml">&lt;?xml version="1.0" encoding="UTF-8"?&gt;
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url t-foreach="pages" t-as="page">
+    <loc><t t-esc="url_root"/><t t-esc="page['loc']"/></loc><t t-if="page.get('lastmod', False)">
+    <lastmod t-esc="page['lastmod']"/></t><t t-if="page.get('priority', False)">
+    <priority t-esc="page['priority']"/></t><t t-if="page.get('changefreq', False)">
+    <changefreq t-esc="page['changefreq']"/></t>
+  </url>
+</urlset>
 </template>
 
-<template id="sitemap_xml">
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-        <t t-foreach="pages" t-as="page">
-            <url>
-                <loc><t t-esc="page['url']"/></loc>
-            </url>
-        </t>
-    </urlset>
+<template id="sitemap_index_xml">&lt;?xml version="1.0" encoding="UTF-8"?&gt;
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap t-foreach="pages" t-as="page">
+    <loc><t t-esc="url_root"/>sitemap-<t t-esc="page"/>.xml</loc>
+  </sitemap>
+</sitemapindex>
 </template>
 
+
 <!-- Actual pages -->
 
-<template id="homepage" name="Homepage" page="True">
+<template id="homepage" name="Homepage" page="True" priority="29">
     <t t-call="website.layout">
       <div id="wrap" class="oe_structure oe_empty"></div>
     </t>
@@ -729,7 +727,7 @@ Sitemap: <t t-esc="url_root"/>sitemap.xml
 
 <template id="company_description" name="Company Description">
     <address itemscope="itemscope" itemtype="http://schema.org/Organization">
-    	<!-- TODO widget contact must add itemprop attributes -->
+        <!-- TODO widget contact must add itemprop attributes -->
         <div t-field="res_company.partner_id" t-field-options='{
                 "widget": "contact",
                 "fields": ["name", "address", "phone", "mobile", "fax", "email"]}'/>