From 7f882e14cf2948a4fe4b2b2b34fa085e6dba4278 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: document: enhance API for iter() on node_descriptors Starting from WebDAV, the node_descriptors will need to act as stream objects, iterable. This way, we hope to minimize copies of the huge chunks of data each descriptor may contain (still, pywebdav will use a single large buffer). bzr revid: p_christ@hol.gr-20101103112534-go57aeaw946utfc5 --- addons/document/document_storage.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 0ecc01941b4..dd0247e52fb 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -48,7 +48,7 @@ DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get( We have to consider 3 cases of data /retrieval/: Given (context,path) we need to access the file (aka. node). given (directory, context), we need one of its children (for listings, views) - given (ir.attachment, context), we needs its data and metadata (node). + given (ir.attachment, context), we need its data and metadata (node). For data /storage/ we have the cases: Have (ir.attachment, context), we modify the file (save, update, rename etc). @@ -100,10 +100,17 @@ class nodefd_file(nodes.node_descriptor): if mode.endswith('b'): mode = mode[:-1] self.mode = mode + self._size = os.stat(path).st_size - for attr in ('closed', 'read', 'write', 'seek', 'tell'): + for attr in ('closed', 'read', 'write', 'seek', 'tell', 'next'): setattr(self,attr, getattr(self.__file, attr)) + def size(self): + return self._size + + def __iter__(self): + return self + def close(self): # TODO: locking in init, close() fname = self.__file.name @@ -165,6 +172,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): """ def __init__(self, parent, ira_browse, mode): nodes.node_descriptor.__init__(self, parent) + self._size = 0L if mode.endswith('b'): mode = mode[:-1] @@ -172,6 +180,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr = ira_browse._cr # reuse the cursor of the browse object, just now cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,)) data = cr.fetchone()[0] + self._size = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) @@ -184,6 +193,9 @@ class nodefd_db(StringIO, nodes.node_descriptor): raise IOError(errno.EINVAL, "Invalid file mode") self.mode = mode + def size(self): + return self._size + def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling @@ -241,11 +253,14 @@ class nodefd_db64(StringIO, nodes.node_descriptor): """ def __init__(self, parent, ira_browse, mode): nodes.node_descriptor.__init__(self, parent) + self._size = 0L if mode.endswith('b'): mode = mode[:-1] if mode in ('r', 'r+'): - StringIO.__init__(self, base64.decodestring(ira_browse.db_datas)) + data = base64.decodestring(ira_browse.db_datas) + self._size = len(data) + StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) # at write, we start at 0 (= overwrite), but have the original @@ -257,6 +272,9 @@ class nodefd_db64(StringIO, nodes.node_descriptor): raise IOError(errno.EINVAL, "Invalid file mode") self.mode = mode + def size(self): + return self._size + def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling -- cgit v1.2.3 From 0c13466702f84a36b46737b665837c4044cd0bf3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: document: fixes for lint checks bzr revid: p_christ@hol.gr-20101103112604-nh7xqcx5e17h3wjo --- addons/document/document_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index dd0247e52fb..18fec8fc18f 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -667,7 +667,7 @@ class document_storage(osv.osv): if ktype == 'file': try: os.unlink(fname) - except Exception, e: + except Exception: self._doclog.warning("Could not remove file %s, please remove manually.", fname, exc_info=True) else: self._doclog.warning("Unknown unlink key %s" % ktype) -- cgit v1.2.3 From 4959480bc1518d0c9bf838c8dcd700dd36e40eb8 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: document: fix realstore, implement move op. in it. bzr revid: p_christ@hol.gr-20101103112604-pd10spyv6hh3demi --- addons/document/document_storage.py | 51 ++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 21 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 18fec8fc18f..64b8b067bae 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -26,6 +26,7 @@ import tools import base64 import errno import logging +import shutil from StringIO import StringIO import psycopg2 @@ -470,11 +471,11 @@ class document_storage(osv.osv): elif boo.type == 'realstore': path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path, - do_create = (mode[1] in ('w','a')) ) + do_create = (mode[0] in ('w','a')) ) fpath = os.path.join(path, npath[-1]) - if (not os.path.exists(fpath)) and mode[1] == 'r': + if (not os.path.exists(fpath)) and mode[0] == 'r': raise IOError("File not found: %s" % fpath) - elif mode[1] in ('w', 'a') and not ira.store_fname: + elif mode[0] in ('w', 'a') and not ira.store_fname: store_fname = os.path.join(*npath) cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s', (store_fname, ira.id)) @@ -747,26 +748,34 @@ class document_storage(osv.osv): # nothing to do for a rename, allow to change the db field return { 'parent_id': ndir_bro.id } elif sbro.type == 'realstore': - raise NotImplementedError("Cannot move in realstore, yet") # TODO - fname = fil_bo.store_fname + ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) + + path, opath = self.__prepare_realpath(cr, file_node, ira, sbro.path, do_create=False) + fname = ira.store_fname + if not fname: - return ValueError("Tried to rename a non-stored file") - path = sbro.path - oldpath = os.path.join(path, fname) + self._doclog.warning("Trying to rename a non-stored file") + if fname != os.path.join(*opath): + self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(opath)) + + oldpath = os.path.join(path, opath[-1]) - for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'): - if ch in new_name: - raise ValueError("Invalid char %s in name %s" %(ch, new_name)) - - file_node.fix_ppath(cr, ira) - npath = file_node.full_path() or [] - dpath = [path,] - dpath.extend(npath[:-1]) - dpath.append(new_name) - newpath = os.path.join(*dpath) - # print "old, new paths:", oldpath, newpath - os.rename(oldpath, newpath) - return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name } + npath = [sbro.path,] + (ndir_bro.get_full_path() or []) + npath = filter(lambda x: x is not None, npath) + newdir = os.path.join(*npath) + if not os.path.isdir(newdir): + self._doclog.debug("Must create dir %s", newdir) + os.makedirs(newdir) + npath.append(opath[-1]) + newpath = os.path.join(*npath) + + self._doclog.debug("Going to move %s from %s to %s", opath[-1], oldpath, newpath) + shutil.move(oldpath, newpath) + + store_path = npath[1:] + [opath[-1],] + store_fname = os.path.join(*store_path) + + return { 'store_fname': store_fname } else: raise TypeError("No %s storage" % sbro.type) -- cgit v1.2.3 From a21dcd7e1c8623f4aaf3a3659e81247273f652e9 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:05 +0200 Subject: document: fix regressions at storage and node_descriptor Since the introduction of __len__, the descriptors were broken for the ftp module. Fix. bzr revid: p_christ@hol.gr-20101103112605-tu6n6529mbdc18ve --- addons/document/document_storage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 64b8b067bae..513da92bfcd 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -181,7 +181,8 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr = ira_browse._cr # reuse the cursor of the browse object, just now cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,)) data = cr.fetchone()[0] - self._size = len(data) + if data: + self._size = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) @@ -260,7 +261,8 @@ class nodefd_db64(StringIO, nodes.node_descriptor): if mode in ('r', 'r+'): data = base64.decodestring(ira_browse.db_datas) - self._size = len(data) + if data: + self._size = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) -- cgit v1.2.3 From 79ff8b4c24ae27b9d9e30b6044919af33e674298 Mon Sep 17 00:00:00 2001 From: "atp (Open ERP)" Date: Thu, 11 Nov 2010 12:17:07 +0530 Subject: [Fix]: Put file descriptors close() in try and finally statement for document module. bzr revid: atp@tinyerp.co.in-20101111064707-lltotiioe28zmke1 --- addons/document/document_storage.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 0ecc01941b4..467a1d7ff31 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -541,9 +541,11 @@ class document_storage(osv.osv): try: store_fname = self.__get_random_fname(path) fname = os.path.join(path, store_fname) - fp = file(fname, 'wb') - fp.write(data) - fp.close() + fp = open(fname, 'wb') + try: + fp.write(data) + finally: + fp.close() self._doclog.debug( "Saved data to %s" % fname) filesize = len(data) # os.stat(fname).st_size @@ -568,9 +570,11 @@ class document_storage(osv.osv): try: path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path, do_create=True) fname = os.path.join(path, npath[-1]) - fp = file(fname,'wb') - fp.write(data) - fp.close() + fp = open(fname,'wb') + try: + fp.write(data) + finally: + fp.close() self._doclog.debug("Saved data to %s", fname) filesize = len(data) # os.stat(fname).st_size store_fname = os.path.join(*npath) -- cgit v1.2.3