[FIX] server: limit concurrent http threads

Before this commit nothing prevented high concurrency on a threaded http
server to consume too much resources, ending up failing requests either
because the OS is unable to spawn that many threads
(`RuntimeError: can't start new thread`), either because the Odoo db
connection pool is full (`PoolError: The Connection Pool Is Full`).

This commit adds the ODOO_MAX_HTTP_THREADS environment variable which
allows to limit the amount of concurrent socket connections accepted by
a threaded server, implicitly limiting the amount of concurrent threads
running for http requests handling.

Note that if a value has been provided to ODOO_MAX_HTTP_THREADS that cannot
be parsed as an integer, a value will be automatically set to half the
db connection pool size (which defaults to 64). This dynamic value is
chosen because while most requests will borrow only one cursor
concurrently, there are some exceptions where some controllers might
allocate two or more cursors.

closes odoo/odoo#42327

X-original-commit: d42a951369ee87b50e834f73cd2af5cf031d43f7
Signed-off-by: Christophe Simonis <chs@odoo.com>

1 files changed, 25 insertions, 0 deletions

diff --git a/odoo/service/server.py b/odoo/service/server.py
index 4390a2f5cb5..95d96126c03 100644
--- a/odoo/service/server.py
+++ b/odoo/service/server.py
@@ -133,6 +133,19 @@ class ThreadedWSGIServerReloadable(LoggingBaseWSGIServerMixIn, werkzeug.serving.
     socket open when a reload happens.
     """
     def __init__(self, host, port, app):
+        # The ODOO_MAX_HTTP_THREADS environment variable allows to limit the amount of concurrent
+        # socket connections accepted by a threaded server, implicitly limiting the amount of
+        # concurrent threads running for http requests handling.
+        self.max_http_threads = os.environ.get("ODOO_MAX_HTTP_THREADS")
+        if self.max_http_threads:
+            try:
+                self.max_http_threads = int(self.max_http_threads)
+            except ValueError:
+                # If the value can't be parsed to an integer then it's computed in an automated way to
+                # half the size of db_maxconn because while most requests won't borrow cursors concurrently
+                # there are some exceptions where some controllers might allocate two or more cursors.
+                self.max_http_threads = config['db_maxconn'] // 2
+            self.http_threads_sem = threading.Semaphore(self.max_http_threads)
         super(ThreadedWSGIServerReloadable, self).__init__(host, port, app,
                                                            handler=RequestHandler)
 
@@ -185,8 +198,20 @@ class ThreadedWSGIServerReloadable(LoggingBaseWSGIServerMixIn, werkzeug.serving.
         """
         if self._BaseServer__shutdown_request:
             return
+        if self.max_http_threads and not self.http_threads_sem.acquire(timeout=0.1):
+            # If the semaphore is full we will return immediately to the upstream (most probably
+            # socketserver.BaseServer's serve_forever loop  which will retry immediately as the
+            # selector will find a pending connection to accept on the socket. There is a 100 ms
+            # penalty in such case in order to avoid cpu bound loop while waiting for the semaphore.
+            return
+        # upstream _handle_request_noblock will handle errors and call shutdown_request in any cases
         super(ThreadedWSGIServerReloadable, self)._handle_request_noblock()
 
+    def shutdown_request(self, request):
+        if self.max_http_threads:
+            # upstream is supposed to call this function no matter what happens during processing
+            self.http_threads_sem.release()
+        super().shutdown_request(request)
 
 #----------------------------------------------------------
 # FileSystem Watcher for autoreload and cache invalidation