From f3cb7cd41539de507f800f10826f003017a40c8a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:30 +0300 Subject: Document storage: implement node_descriptors. bzr revid: p_christ@hol.gr-20100629140530-btusqmlz2lyadfbx --- addons/document/document_storage.py | 102 ++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 90be2bc47df..b2db4eb36a6 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -24,7 +24,9 @@ from osv import osv, fields import os import tools import base64 +import errno import logging +from StringIO import StringIO from tools.misc import ustr from tools.translate import _ @@ -34,6 +36,7 @@ from osv.orm import except_orm import random import string import netsvc +import nodes from content_index import cntIndex DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get('root_path'), 'filestore')) @@ -84,6 +87,64 @@ def create_directory(path): os.makedirs(path) return dir_name +class nodefd_file(nodes.node_descriptor): + """ A descriptor to a real file + + Inheriting directly from file doesn't work, since file exports + some read-only attributes (like 'name') that we don't like. + """ + def __init__(self, parent, path, mode): + nodes.node_descriptor.__init__(self, parent) + self.__file = open(path, mode) + + for attr in ('closed', 'read', 'write', 'seek', 'tell'): + setattr(self,attr, getattr(self.__file, attr)) + + def close(self): + # TODO: locking in init, close() + self.__file.close() + + +class nodefd_db(StringIO, nodes.node_descriptor): + """ A descriptor to db data + """ + def __init__(self, parent, ira_browse, mode): + nodes.node_descriptor.__init__(self, parent) + if mode.endswith('b'): + mode = mode[:-1] + + if mode == 'r': + StringIO.__init__(self, ira_browse.db_datas) + elif mode == 'w': + StringIO.__init__(self, ira_browse.db_datas) + # at write, we start at 0 (= overwrite), but have the original + # data available, in case of a seek() + elif mode == 'a': + StringIO.__init__(self, None) + else: + logging.getLogger('document.storage').error("Incorrect mode %s specified", mode) + raise IOError(errno.EINVAL, "Invalid file mode") + self.mode = mode + + def close(self): + # we now open a *separate* cursor, to update the data. + # FIXME: this may be improved, for concurrency handling + uid = self.__parent.context.uid + cr = pooler.get_db(self.__parent.context.dbname).cursor() + if mode == 'w': + out = self.getvalue() + cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', + (out, len(out), self.__parent.file_id)) + elif mode == 'a': + out = self.getvalue() + cr.execute("UPDATE ir_attachment " \ + "SET db_datas = COALESCE(db_datas,'') || %s, " \ + " file_size = COALESCE(file_size, 0) + %d " \ + " WHERE id = %s", + (out, len(out), self.__parent.file_id)) + cr.commit() + cr.close() + StringIO.close(self) class document_storage(osv.osv): """ The primary object for data storage. @@ -147,6 +208,47 @@ class document_storage(osv.osv): ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) return self.__get_data_3(cr, uid, boo, ira, context) + def get_file(self, cr, uid, id, file_node, mode, context=None): + if context is None: + context = {} + boo = self.browse(cr, uid, id, context) + if not boo.online: + raise RuntimeError('media offline') + + ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) + if boo.type == 'filestore': + if not ira.store_fname: + # On a migrated db, some files may have the wrong storage type + # try to fix their directory. + if ira.file_size: + self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id) + raise IOError(errno.ENOENT, 'No file can be located') + fpath = os.path.join(boo.path, ira.store_fname) + if self._debug: + self._doclog.debug("Trying to read \"%s\".."% fpath) + return nodefd_file(file_node, path=fpath, mode=mode) + + elif boo.type == 'db': + # TODO: we need a better api for large files + if self._debug: + self._doclog.debug("Trying to obtain db_datas for ir.attachment[%d]", ira.id) + return nodefd_db(file_node, ira_browse=ira, mode=mode) + + elif boo.type == 'realstore': + if not ira.store_fname: + # On a migrated db, some files may have the wrong storage type + # try to fix their directory. + if ira.file_size: + self._doclog.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id) + sfname = ira.name + fpath = os.path.join(boo.path,ira.store_fname or ira.name) + if not os.path.exists(fpath): + raise IOError("File not found: %s" % fpath) + return nodefd_file(file_node, path=fpath, mode=mode) + + else: + raise TypeError("No %s storage" % boo.type) + def __get_data_3(self, cr, uid, boo, ira, context): if not boo.online: raise RuntimeError('media offline') -- cgit v1.2.3 From 8307ff013916ffcaf8c7f0c8a796df77a64bfcd4 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: Document: workaround case when text file consists of non-unicode. The content indexer would be trying to process the garbage of the file and fail. bzr revid: p_christ@hol.gr-20100701175131-b83jg498xjuosmd9 --- addons/document/document_storage.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index b2db4eb36a6..26ba5dea772 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -378,16 +378,21 @@ class document_storage(osv.osv): self._doclog.debug('Cannot index file:', exc_info=True) pass + try: + icont_u = ustr(icont) + except UnicodeError: + icont_u = '' + # a hack: /assume/ that the calling write operation will not try # to write the fname and size, and update them in the db concurrently. # We cannot use a write() here, because we are already in one. cr.execute('UPDATE ir_attachment SET store_fname = %s, file_size = %s, index_content = %s, file_type = %s WHERE id = %s', - (store_fname, filesize, ustr(icont), mime, file_node.file_id)) + (store_fname, filesize, icont_u, mime, file_node.file_id)) file_node.content_length = filesize file_node.content_type = mime return True except Exception, e : - self._doclog.warning( "Couldn't save data:", exc_info=True) + self._doclog.warning("Couldn't save data:", exc_info=True) # should we really rollback once we have written the actual data? # at the db case (only), that rollback would be safe raise except_orm(_('Error at doc write!'), str(e)) -- cgit v1.2.3 From 4913a2f0dba67d082e225bafa97b4c576e4f8cb7 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: Document: fix nodefd_db.close() Some vars were missing/inaccessible, put the queries in a try block. bzr revid: p_christ@hol.gr-20100701175132-389l8frlqqg7hgr3 --- addons/document/document_storage.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 26ba5dea772..08d3d82893b 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -35,6 +35,7 @@ from osv.orm import except_orm import random import string +import pooler import netsvc import nodes from content_index import cntIndex @@ -129,21 +130,27 @@ class nodefd_db(StringIO, nodes.node_descriptor): def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling - uid = self.__parent.context.uid - cr = pooler.get_db(self.__parent.context.dbname).cursor() - if mode == 'w': - out = self.getvalue() - cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', - (out, len(out), self.__parent.file_id)) - elif mode == 'a': - out = self.getvalue() - cr.execute("UPDATE ir_attachment " \ - "SET db_datas = COALESCE(db_datas,'') || %s, " \ - " file_size = COALESCE(file_size, 0) + %d " \ - " WHERE id = %s", - (out, len(out), self.__parent.file_id)) - cr.commit() - cr.close() + par = self._get_parent() + uid = par.context.uid + cr = pooler.get_db(par.context.dbname).cursor() + try: + if self.mode == 'w': + out = self.getvalue() + cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', + (out, len(out), par.file_id)) + elif self.mode == 'a': + out = self.getvalue() + cr.execute("UPDATE ir_attachment " \ + "SET db_datas = COALESCE(db_datas,'') || %s, " \ + " file_size = COALESCE(file_size, 0) + %d " \ + " WHERE id = %s", + (out, len(out), par.file_id)) + cr.commit() + except Exception, e: + logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) + raise + finally: + cr.close() StringIO.close(self) class document_storage(osv.osv): -- cgit v1.2.3 From 0153d03c825974985ad5486f7ae9229cf0814853 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: Document storage: support more file open modes. We may be called with 'w+', 'r+' modes, support them. However, we still don't support the 'a+' mode. bzr revid: p_christ@hol.gr-20100701175132-v21yzk2h335emum1 --- addons/document/document_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 08d3d82893b..606600e17cb 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -114,10 +114,10 @@ class nodefd_db(StringIO, nodes.node_descriptor): if mode.endswith('b'): mode = mode[:-1] - if mode == 'r': - StringIO.__init__(self, ira_browse.db_datas) - elif mode == 'w': + if mode in ('r', 'r+'): StringIO.__init__(self, ira_browse.db_datas) + elif mode in ('w', 'w+'): + StringIO.__init__(self, None) # at write, we start at 0 (= overwrite), but have the original # data available, in case of a seek() elif mode == 'a': @@ -134,7 +134,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): uid = par.context.uid cr = pooler.get_db(par.context.dbname).cursor() try: - if self.mode == 'w': + if self.mode in ('w', 'w+', 'r+'): out = self.getvalue() cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', (out, len(out), par.file_id)) -- cgit v1.2.3 From 7c26eb67fb646e02848435e768fd9c08a63529c9 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: Document storage: the default db encoding is base64 in bytea The storage class has to consider that. Also, prepare a class for raw bytea storage. bzr revid: p_christ@hol.gr-20100701175133-xxgu3cdlff0u7086 --- addons/document/document_storage.py | 64 ++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 606600e17cb..290faac0229 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -115,7 +115,9 @@ class nodefd_db(StringIO, nodes.node_descriptor): mode = mode[:-1] if mode in ('r', 'r+'): - StringIO.__init__(self, ira_browse.db_datas) + cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', ira_browse.id) + data = cr.fetchone()[0] + StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) # at write, we start at 0 (= overwrite), but have the original @@ -136,13 +138,13 @@ class nodefd_db(StringIO, nodes.node_descriptor): try: if self.mode in ('w', 'w+', 'r+'): out = self.getvalue() - cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', + cr.execute("UPDATE ir_attachment SET db_datas = decode(%s,'escape'), file_size=%s WHERE id = %s", (out, len(out), par.file_id)) elif self.mode == 'a': out = self.getvalue() cr.execute("UPDATE ir_attachment " \ - "SET db_datas = COALESCE(db_datas,'') || %s, " \ - " file_size = COALESCE(file_size, 0) + %d " \ + "SET db_datas = COALESCE(db_datas,'') || decode(%s, 'escape'), " \ + " file_size = COALESCE(file_size, 0) + %s " \ " WHERE id = %s", (out, len(out), par.file_id)) cr.commit() @@ -153,6 +155,58 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr.close() StringIO.close(self) +class nodefd_db64(StringIO, nodes.node_descriptor): + """ A descriptor to db data, base64 (the old way) + + It stores the data in base64 encoding at the db. Not optimal, but + the transparent compression of Postgres will save the day. + """ + def __init__(self, parent, ira_browse, mode): + nodes.node_descriptor.__init__(self, parent) + if mode.endswith('b'): + mode = mode[:-1] + + if mode in ('r', 'r+'): + StringIO.__init__(self, base64.decodestring(ira_browse.db_datas)) + elif mode in ('w', 'w+'): + StringIO.__init__(self, None) + # at write, we start at 0 (= overwrite), but have the original + # data available, in case of a seek() + elif mode == 'a': + StringIO.__init__(self, None) + else: + logging.getLogger('document.storage').error("Incorrect mode %s specified", mode) + raise IOError(errno.EINVAL, "Invalid file mode") + self.mode = mode + + def close(self): + # we now open a *separate* cursor, to update the data. + # FIXME: this may be improved, for concurrency handling + par = self._get_parent() + uid = par.context.uid + cr = pooler.get_db(par.context.dbname).cursor() + try: + if self.mode in ('w', 'w+', 'r+'): + out = self.getvalue() + cr.execute('UPDATE ir_attachment SET db_datas = %s::bytea, file_size=%s WHERE id = %s', + (base64.encodestring(out), len(out), par.file_id)) + elif self.mode == 'a': + out = self.getvalue() + # Yes, we're obviously using the wrong representation for storing our + # data as base64-in-bytea + cr.execute("UPDATE ir_attachment " \ + "SET db_datas = encode( (COALESCE(decode(encode(db_datas,'escape'),'base64'),'') || decode(%s, 'base64')),'base64')::bytea , " \ + " file_size = COALESCE(file_size, 0) + %s " \ + " WHERE id = %s", + (base64.encodestring(out), len(out), par.file_id)) + cr.commit() + except Exception, e: + logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) + raise + finally: + cr.close() + StringIO.close(self) + class document_storage(osv.osv): """ The primary object for data storage. Each instance of this object is a storage media, in which our application @@ -239,7 +293,7 @@ class document_storage(osv.osv): # TODO: we need a better api for large files if self._debug: self._doclog.debug("Trying to obtain db_datas for ir.attachment[%d]", ira.id) - return nodefd_db(file_node, ira_browse=ira, mode=mode) + return nodefd_db64(file_node, ira_browse=ira, mode=mode) elif boo.type == 'realstore': if not ira.store_fname: -- cgit v1.2.3 From 49bf479c188533b93d74f1e0f873f8d71b6ebd75 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: Document storage: implementation of simple rename Conflicts: document/document_storage.py bzr revid: p_christ@hol.gr-20100701175133-ti0itn58w9az88w6 --- addons/document/document_storage.py | 38 ++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 290faac0229..aeea5f99011 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -480,7 +480,7 @@ class document_storage(osv.osv): path = storage_bo.path return ( storage_bo.id, 'file', os.path.join(path, fname)) else: - raise TypeError("No %s storage" % boo.type) + raise TypeError("No %s storage" % storage_bo.type) def do_unlink(self, cr, uid, unres): for id, ktype, fname in unres: @@ -494,6 +494,42 @@ class document_storage(osv.osv): return True + def simple_rename(self, cr, uid, file_node, new_name, context=None): + """ A preparation for a file rename. + It will not affect the database, but merely check and perhaps + rename the realstore file. + + @return the dict of values that can safely be be stored in the db. + """ + sbro = self.browse(cr, uid, file_node.storage_id, context=context) + assert sbro, "The file #%d didn't provide storage" % file_node.file_id + + if sbro.type in ('filestore', 'db'): + # nothing to do for a rename, allow to change the db field + return { 'name': new_name, 'datas_fname': new_name } + elif sbro.type == 'realstore': + fname = fil_bo.store_fname + if not fname: + return ValueError("Tried to rename a non-stored file") + path = storage_bo.path + oldpath = os.path.join(path, fname) + + for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'): + if ch in new_name: + raise ValueError("Invalid char %s in name %s" %(ch, new_name)) + + file_node.fix_ppath(cr, ira) + npath = file_node.full_path() or [] + dpath = [path,] + dpath.extend(npath[:-1]) + dpath.append(new_name) + newpath = os.path.join(*dpath) + # print "old, new paths:", oldpath, newpath + os.rename(oldpath, newpath) + return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name } + else: + raise TypeError("No %s storage" % boo.type) + document_storage() -- cgit v1.2.3 From 1927431e0bf256503fbdcf282008a7b45f449559 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 7 Jul 2010 02:27:00 +0300 Subject: Document: remove debug calls from trunk code. That code would only work in trunk-xrg. bzr revid: p_christ@hol.gr-20100706232700-x56f57gqhzupfphq --- addons/document/document_storage.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index aeea5f99011..809c3c83be8 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -285,14 +285,10 @@ class document_storage(osv.osv): self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id) raise IOError(errno.ENOENT, 'No file can be located') fpath = os.path.join(boo.path, ira.store_fname) - if self._debug: - self._doclog.debug("Trying to read \"%s\".."% fpath) return nodefd_file(file_node, path=fpath, mode=mode) elif boo.type == 'db': # TODO: we need a better api for large files - if self._debug: - self._doclog.debug("Trying to obtain db_datas for ir.attachment[%d]", ira.id) return nodefd_db64(file_node, ira_browse=ira, mode=mode) elif boo.type == 'realstore': -- cgit v1.2.3 From 03865bc917b6fa7ee3e3964411722ccdc75b52c2 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 9 Jul 2010 01:50:42 +0300 Subject: Document: Less dir, storage options, cleanup views. The 'virtual' storage is now hidden. An extra 'db64' type, hidden, is supported in the code, for v5-compatible databases (until we migrate the db_datas column of them). The 'at_record' directory type is gone. A module in extra-addons will permit the compatible behaviour, for those experts. bzr revid: p_christ@hol.gr-20100708225042-fp3zcwmw4zp3ee04 --- addons/document/document_storage.py | 39 ++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 809c3c83be8..b1b884a9d98 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -232,7 +232,7 @@ class document_storage(osv.osv): 'group_ids': fields.many2many('res.groups', 'document_storage_group_rel', 'item_id', 'group_id', 'Groups'), 'dir_ids': fields.one2many('document.directory', 'parent_id', 'Directories'), 'type': fields.selection([('db', 'Database'), ('filestore', 'Internal File storage'), - ('realstore', 'External file storage'), ('virtual', 'Virtual storage')], 'Type', required=True), + ('realstore','External file storage'),], 'Type', required=True), 'path': fields.char('Path', size=250, select=1, help="For file storage, the root path of the storage"), 'online': fields.boolean('Online', help="If not checked, media is currently offline and its contents not available", required=True), 'readonly': fields.boolean('Read Only', help="If set, media is for reading only"), @@ -270,6 +270,8 @@ class document_storage(osv.osv): return self.__get_data_3(cr, uid, boo, ira, context) def get_file(self, cr, uid, id, file_node, mode, context=None): + """ Return a file-like object for the contents of some node + """ if context is None: context = {} boo = self.browse(cr, uid, id, context) @@ -289,6 +291,9 @@ class document_storage(osv.osv): elif boo.type == 'db': # TODO: we need a better api for large files + return nodefd_db(file_node, ira_browse=ira, mode=mode) + + elif boo.type == 'db64': return nodefd_db64(file_node, ira_browse=ira, mode=mode) elif boo.type == 'realstore': @@ -303,6 +308,9 @@ class document_storage(osv.osv): raise IOError("File not found: %s" % fpath) return nodefd_file(file_node, path=fpath, mode=mode) + elif boo.type == 'virtual': + raise ValueError('Virtual storage does not support static files') + else: raise TypeError("No %s storage" % boo.type) @@ -318,13 +326,21 @@ class document_storage(osv.osv): return None fpath = os.path.join(boo.path, ira.store_fname) return file(fpath, 'rb').read() - elif boo.type == 'db': + elif boo.type == 'db64': # TODO: we need a better api for large files if ira.db_datas: out = base64.decodestring(ira.db_datas) else: out = '' return out + elif boo.type == 'db': + # We do an explicit query, to avoid type transformations. + cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', (ira.id,)) + res = cr.fetchone() + if res: + return res[0] + else: + return '' elif boo.type == 'realstore': if not ira.store_fname: # On a migrated db, some files may have the wrong storage type @@ -339,6 +355,10 @@ class document_storage(osv.osv): return None else: raise IOError("File not found: %s" % fpath) + + elif boo.type == 'virtual': + raise ValueError('Virtual storage does not support static files') + else: raise TypeError("No %s storage" % boo.type) @@ -387,7 +407,12 @@ class document_storage(osv.osv): raise except_orm(_('Error!'), str(e)) elif boo.type == 'db': filesize = len(data) - # will that work for huge data? TODO + # will that work for huge data? + cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s', + (data, file_node.file_id)) + elif boo.type == 'db64': + filesize = len(data) + # will that work for huge data? out = base64.encodestring(data) cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s', (out, file_node.file_id)) @@ -419,6 +444,10 @@ class document_storage(osv.osv): except Exception,e : self._doclog.warning("Couldn't save data:", exc_info=True) raise except_orm(_('Error!'), str(e)) + + elif boo.type == 'virtual': + raise ValueError('Virtual storage does not support static files') + else: raise TypeError("No %s storage" % boo.type) @@ -467,7 +496,7 @@ class document_storage(osv.osv): return None path = storage_bo.path return (storage_bo.id, 'file', os.path.join(path, fname)) - elif storage_bo.type == 'db': + elif storage_bo.type in ('db', 'db64'): return None elif storage_bo.type == 'realstore': fname = fil_bo.store_fname @@ -500,7 +529,7 @@ class document_storage(osv.osv): sbro = self.browse(cr, uid, file_node.storage_id, context=context) assert sbro, "The file #%d didn't provide storage" % file_node.file_id - if sbro.type in ('filestore', 'db'): + if sbro.type in ('filestore', 'db', 'db64'): # nothing to do for a rename, allow to change the db field return { 'name': new_name, 'datas_fname': new_name } elif sbro.type == 'realstore': -- cgit v1.2.3 From e2c068b7ba9fd46bd753140f781a74ec84fd1272 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 9 Jul 2010 11:23:41 +0300 Subject: Document storage: support move of files (for filestore, db) bzr revid: p_christ@hol.gr-20100709082341-x7bedn540ev34lzs --- addons/document/document_storage.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index b1b884a9d98..eedfc435e68 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -555,6 +555,56 @@ class document_storage(osv.osv): else: raise TypeError("No %s storage" % boo.type) + def simple_move(self, cr, uid, file_node, ndir_bro, context=None): + """ A preparation for a file move. + It will not affect the database, but merely check and perhaps + move the realstore file. + + @param ndir_bro a browse object of document.directory, where this + file should move to. + @return the dict of values that can safely be be stored in the db. + """ + sbro = self.browse(cr, uid, file_node.storage_id, context=context) + assert sbro, "The file #%d didn't provide storage" % file_node.file_id + + par = ndir_bro + psto = None + while par: + if par.storage_id: + psto = par.storage_id.id + break + par = par.parent_id + if file_node.storage_id != psto: + self._doclog.debug('Cannot move file %r from %r to %r', file_node, file_node.parent, ndir_bro.name) + raise NotImplementedError('Cannot move files between storage media') + + if sbro.type in ('filestore', 'db', 'db64'): + # nothing to do for a rename, allow to change the db field + return { 'parent_id': ndir_bro.id } + elif sbro.type == 'realstore': + raise NotImplementedError("Cannot move in realstore, yet") # TODO + fname = fil_bo.store_fname + if not fname: + return ValueError("Tried to rename a non-stored file") + path = storage_bo.path + oldpath = os.path.join(path, fname) + + for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'): + if ch in new_name: + raise ValueError("Invalid char %s in name %s" %(ch, new_name)) + + file_node.fix_ppath(cr, ira) + npath = file_node.full_path() or [] + dpath = [path,] + dpath.extend(npath[:-1]) + dpath.append(new_name) + newpath = os.path.join(*dpath) + # print "old, new paths:", oldpath, newpath + os.rename(oldpath, newpath) + return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name } + else: + raise TypeError("No %s storage" % boo.type) + document_storage() -- cgit v1.2.3 From 60727b0cb09a5065661df836fc01f83b5c407e95 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 9 Jul 2010 11:23:51 +0300 Subject: Document: fix db storage (the new, raw engine) The correct way to pass the raw file data into the db is through the psycopg2.Binary() pseydo-type. bzr revid: p_christ@hol.gr-20100709082351-rg1vle2d4mpzh056 --- addons/document/document_storage.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'addons/document/document_storage.py') diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index eedfc435e68..48f7ccf2f87 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -27,6 +27,7 @@ import base64 import errno import logging from StringIO import StringIO +import psycopg2 from tools.misc import ustr from tools.translate import _ @@ -115,7 +116,8 @@ class nodefd_db(StringIO, nodes.node_descriptor): mode = mode[:-1] if mode in ('r', 'r+'): - cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', ira_browse.id) + cr = ira_browse._cr # reuse the cursor of the browse object, just now + cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,)) data = cr.fetchone()[0] StringIO.__init__(self, data) elif mode in ('w', 'w+'): @@ -137,16 +139,18 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr = pooler.get_db(par.context.dbname).cursor() try: if self.mode in ('w', 'w+', 'r+'): - out = self.getvalue() - cr.execute("UPDATE ir_attachment SET db_datas = decode(%s,'escape'), file_size=%s WHERE id = %s", - (out, len(out), par.file_id)) + data = self.getvalue() + out = psycopg2.Binary(data) + cr.execute("UPDATE ir_attachment SET db_datas = %s, file_size=%s WHERE id = %s", + (out, len(data), par.file_id)) elif self.mode == 'a': - out = self.getvalue() + data = self.getvalue() + out = psycopg2.Binary(data) cr.execute("UPDATE ir_attachment " \ - "SET db_datas = COALESCE(db_datas,'') || decode(%s, 'escape'), " \ + "SET db_datas = COALESCE(db_datas,'') || %s, " \ " file_size = COALESCE(file_size, 0) + %s " \ " WHERE id = %s", - (out, len(out), par.file_id)) + (out, len(data), par.file_id)) cr.commit() except Exception, e: logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) @@ -408,8 +412,9 @@ class document_storage(osv.osv): elif boo.type == 'db': filesize = len(data) # will that work for huge data? + out = psycopg2.Binary(data) cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s', - (data, file_node.file_id)) + (out, file_node.file_id)) elif boo.type == 'db64': filesize = len(data) # will that work for huge data? -- cgit v1.2.3