From 08615bbbc13b7fc0957a0bbd1eacbd60b4cac6c3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 16 Dec 2008 00:06:06 +0200 Subject: If some SQL query fails, log the whole query. bzr revid: p_christ@hol.gr-20081215220606-l9mncs9qnyy0v7tv --- bin/sql_db.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'bin/sql_db.py') diff --git a/bin/sql_db.py b/bin/sql_db.py index 3f4e23cc392..ad3068a08e2 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -118,7 +118,12 @@ class Cursor(object): if self.sql_log: now = mdt.now() - res = self._obj.execute(query, p or None) + try: + res = self._obj.execute(query, p or None) + except psycopg2.ProgrammingError, pe: + logger= netsvc.Logger() + logger.notifyChannel('sql_db', netsvc.LOG_ERROR, "Programming error: %s, in query %s" % (pe, query)) + raise if self.sql_log: log("query: %s" % self._obj.query) -- cgit v1.2.3 From 3487f659ec80bda2855885c432af6d71586b3ff5 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 6 Jan 2009 15:17:46 +0200 Subject: Always print SQL counters. bzr revid: p_christ@hol.gr-20090106131746-mwdgfx2l1z9hd02b --- bin/sql_db.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'bin/sql_db.py') diff --git a/bin/sql_db.py b/bin/sql_db.py index 30b4df1e2b8..042c41940a3 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -59,6 +59,8 @@ def log(msg, lvl=netsvc.LOG_DEBUG): logger = netsvc.Logger() logger.notifyChannel('sql', lvl, msg) +sql_counter = 0 + class Cursor(object): IN_MAX = 1000 sql_from_log = {} @@ -72,7 +74,7 @@ class Cursor(object): @wraps(f) def wrapper(self, *args, **kwargs): if not hasattr(self, '_obj'): - raise psycopg2.ProgrammingError('Unable to use the cursor after having closing it') + raise psycopg2.ProgrammingError('Unable to use the cursor after having closed it') return f(self, *args, **kwargs) return wrapper @@ -104,7 +106,7 @@ class Cursor(object): self.count+=1 if '%d' in query or '%f' in query: log(query, netsvc.LOG_WARNING) - log("SQL queries mustn't containt %d or %f anymore. Use only %s", netsvc.LOG_WARNING) + log("SQL queries cannot contain %d or %f anymore. Use only %s", netsvc.LOG_WARNING) if params: query = query.replace('%d', '%s').replace('%f', '%s') @@ -134,19 +136,20 @@ class Cursor(object): return res def print_log(self): + global sql_counter + sql_counter += self.count def process(type): sqllogs = {'from':self.sql_from_log, 'into':self.sql_into_log} - if not sqllogs[type]: - return - sqllogitems = sqllogs[type].items() - sqllogitems.sort(key=lambda k: k[1][1]) sum = 0 - log("SQL LOG %s:" % (type,)) - for r in sqllogitems: - log("table: %s: %s/%s" %(r[0], str(r[1][1]), r[1][0])) - sum+= r[1][1] - log("SUM:%s/%d" % (sum, self.count)) - sqllogs[type].clear() + if sqllogs[type]: + sqllogitems = sqllogs[type].items() + sqllogitems.sort(key=lambda k: k[1][1]) + log("SQL LOG %s:" % (type,)) + for r in sqllogitems: + log("table: %s: %s/%s" %(r[0], str(r[1][1]), r[1][0])) + sum+= r[1][1] + sqllogs[type].clear() + log("SUM:%s/%d [%d]" % (sum, self.count,sql_counter)) process('from') process('into') self.count = 0 -- cgit v1.2.3 From 79d62886715d113d945a2f1c0b4469b811741fe0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Mon, 20 Apr 2009 20:20:16 +0300 Subject: Major fix for Python 2.6: fix db cursor speed like 2.5 In python 2.6, the hasattr() will undergo subclass checks. So, it is unwise to call it for the "object". That subclass check would overflow the recursion limit, on each cursor delete operation, which ends up in wasted CPU (and slow OpenObject responsiveness). Now, hopefully, OpenObject will run in 2.6 as fast as it ran in 2.5 . bzr revid: p_christ@hol.gr-20090420172016-1dgiys0cj90efh34 --- bin/sql_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/sql_db.py') diff --git a/bin/sql_db.py b/bin/sql_db.py index 8fa364b9410..93e95e82bfb 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -91,7 +91,7 @@ class Cursor(object): self.__caller = tuple(stack()[2][1:3]) def __del__(self): - if hasattr(self, '_obj'): + if hasattr(self.__dict__, '_obj'): if tools.config['log_level'] in (netsvc.LOG_DEBUG, netsvc.LOG_DEBUG_RPC): # Oops. 'self' has not been closed explicitly. # The cursor will be deleted by the garbage collector, -- cgit v1.2.3 From e0da6e16d1801fc16f8ac5cb2d82dd21ed5538d0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Mon, 20 Apr 2009 21:37:55 +0300 Subject: Rewrite previous patch, sane cursor close checks. bzr revid: p_christ@hol.gr-20090420183755-epdfdf6pgd6vc88m --- bin/sql_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/sql_db.py') diff --git a/bin/sql_db.py b/bin/sql_db.py index 93e95e82bfb..256df23a1e0 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -73,7 +73,7 @@ class Cursor(object): @wraps(f) def wrapper(self, *args, **kwargs): - if not hasattr(self, '_obj'): + if not '_obj' in self.__dict__: raise psycopg2.ProgrammingError('Unable to use the cursor after having closed it') return f(self, *args, **kwargs) return wrapper @@ -91,7 +91,7 @@ class Cursor(object): self.__caller = tuple(stack()[2][1:3]) def __del__(self): - if hasattr(self.__dict__, '_obj'): + if '_obj' in self.__dict__ : if tools.config['log_level'] in (netsvc.LOG_DEBUG, netsvc.LOG_DEBUG_RPC): # Oops. 'self' has not been closed explicitly. # The cursor will be deleted by the garbage collector, -- cgit v1.2.3 From 400ee151f82819230671f842544e6e7a2b21dd80 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 26 Jul 2009 10:55:04 +0300 Subject: Mark some points where SQL calls need optimize. In particular, calls to ir.translate are done in Python loops. This is slow. bzr revid: p_christ@hol.gr-20090726075504-v6qgdyyg9vyaazfr --- bin/sql_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/sql_db.py') diff --git a/bin/sql_db.py b/bin/sql_db.py index edf9a2aeaec..d7769945a84 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -158,7 +158,7 @@ class Cursor(object): log("table: %s: %s/%s" %(r[0], str(r[1][1]), r[1][0])) sum+= r[1][1] sqllogs[type].clear() - log("SUM:%s/%d [%d]" % (sum, self.count,sql_counter)) + log("SUM %s:%s/%d [%d]" % (type, sum, self.count,sql_counter)) process('from') process('into') self.count = 0 -- cgit v1.2.3 From e571324ab33939ce9f4fc2e02108d6ae36438315 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 14 Aug 2009 19:01:11 +0300 Subject: Introduce 'debug2' loglevel, use it for SQL Sometimes we want to debug SQL, not always though. bzr revid: p_christ@hol.gr-20090814160111-fhw1j4jmx94t2hjv --- bin/sql_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/sql_db.py') diff --git a/bin/sql_db.py b/bin/sql_db.py index edf9a2aeaec..85bd6fc88d9 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -55,7 +55,7 @@ from mx import DateTime as mdt re_from = re.compile('.* from "?([a-zA-Z_0-9]+)"? .*$'); re_into = re.compile('.* into "?([a-zA-Z_0-9]+)"? .*$'); -def log(msg, lvl=netsvc.LOG_DEBUG): +def log(msg, lvl=netsvc.LOG_DEBUG2): logger = netsvc.Logger() logger.notifyChannel('sql', lvl, msg) -- cgit v1.2.3