diff options
Diffstat (limited to 'addons/document/document_storage.py')
| -rw-r--r-- | addons/document/document_storage.py | 182 |
1 files changed, 89 insertions, 93 deletions
diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index fe757e7876d..21cb4f9dcf4 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -29,18 +29,15 @@ import logging import shutil from StringIO import StringIO import psycopg2 - from tools.misc import ustr from tools.translate import _ - from osv.orm import except_orm - import random import string import pooler import nodes from content_index import cntIndex - +_logger = logging.getLogger(__name__) DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get('root_path'), 'filestore')) @@ -55,7 +52,7 @@ For data /storage/ we have the cases: Have (ir.attachment, context), we modify the file (save, update, rename etc). Have (directory, context), we create a file. Have (path, context), we create or modify a file. - + Note that in all above cases, we don't explicitly choose the storage media, but always require a context to be present. @@ -64,7 +61,7 @@ nodes, for once, won't. Their metadata will be computed by the parent storage media + directory. The algorithm says that in any of the above cases, our first goal is to locate -the node for any combination of search criteria. It would be wise NOT to +the node for any combination of search criteria. It would be wise NOT to represent each node in the path (like node[/] + node[/dir1] + node[/dir1/dir2]) but directly jump to the end node (like node[/dir1/dir2]) whenever possible. @@ -102,13 +99,13 @@ class nodefd_file(nodes.node_descriptor): mode = mode[:-1] self.mode = mode self._size = os.stat(path).st_size - + for attr in ('closed', 'read', 'write', 'seek', 'tell', 'next'): setattr(self,attr, getattr(self.__file, attr)) def size(self): return self._size - + def __iter__(self): return self @@ -125,12 +122,12 @@ class nodefd_file(nodes.node_descriptor): filename = par.path if isinstance(filename, (tuple, list)): filename = '/'.join(filename) - + try: mime, icont = cntIndex.doIndex(None, filename=filename, content_type=None, realfname=fname) except Exception: - logging.getLogger('document.storage').debug('Cannot index file:', exc_info=True) + _logger.debug('Cannot index file:', exc_info=True) pass try: @@ -150,7 +147,7 @@ class nodefd_file(nodes.node_descriptor): cr.commit() cr.close() except Exception: - logging.getLogger('document.storage').warning('Cannot save file indexed content:', exc_info=True) + _logger.warning('Cannot save file indexed content:', exc_info=True) elif self.mode in ('a', 'a+' ): try: @@ -164,7 +161,7 @@ class nodefd_file(nodes.node_descriptor): cr.commit() cr.close() except Exception: - logging.getLogger('document.storage').warning('Cannot save file appended content:', exc_info=True) + _logger.warning('Cannot save file appended content:', exc_info=True) @@ -176,7 +173,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): self._size = 0L if mode.endswith('b'): mode = mode[:-1] - + if mode in ('r', 'r+'): cr = ira_browse._cr # reuse the cursor of the browse object, just now cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,)) @@ -191,8 +188,8 @@ class nodefd_db(StringIO, nodes.node_descriptor): elif mode == 'a': StringIO.__init__(self, None) else: - logging.getLogger('document.storage').error("Incorrect mode %s specified", mode) - raise IOError(errno.EINVAL, "Invalid file mode") + _logger.error("Incorrect mode %s is specified.", mode) + raise IOError(errno.EINVAL, "Invalid file mode.") self.mode = mode def size(self): @@ -212,12 +209,12 @@ class nodefd_db(StringIO, nodes.node_descriptor): filename = par.path if isinstance(filename, (tuple, list)): filename = '/'.join(filename) - + try: mime, icont = cntIndex.doIndex(data, filename=filename, content_type=None, realfname=None) except Exception: - logging.getLogger('document.storage').debug('Cannot index file:', exc_info=True) + _logger.debug('Cannot index file:', exc_info=True) pass try: @@ -241,7 +238,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): (out, len(data), par.file_id)) cr.commit() except Exception: - logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) + _logger.exception('Cannot update db file #%d for close.', par.file_id) raise finally: cr.close() @@ -249,7 +246,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): class nodefd_db64(StringIO, nodes.node_descriptor): """ A descriptor to db data, base64 (the old way) - + It stores the data in base64 encoding at the db. Not optimal, but the transparent compression of Postgres will save the day. """ @@ -258,7 +255,7 @@ class nodefd_db64(StringIO, nodes.node_descriptor): self._size = 0L if mode.endswith('b'): mode = mode[:-1] - + if mode in ('r', 'r+'): data = base64.decodestring(ira_browse.db_datas) if data: @@ -271,8 +268,8 @@ class nodefd_db64(StringIO, nodes.node_descriptor): elif mode == 'a': StringIO.__init__(self, None) else: - logging.getLogger('document.storage').error("Incorrect mode %s specified", mode) - raise IOError(errno.EINVAL, "Invalid file mode") + _logger.error("Incorrect mode %s is specified.", mode) + raise IOError(errno.EINVAL, "Invalid file mode.") self.mode = mode def size(self): @@ -292,12 +289,12 @@ class nodefd_db64(StringIO, nodes.node_descriptor): filename = par.path if isinstance(filename, (tuple, list)): filename = '/'.join(filename) - + try: mime, icont = cntIndex.doIndex(data, filename=filename, content_type=None, realfname=None) except Exception: - logging.getLogger('document.storage').debug('Cannot index file:', exc_info=True) + self.logger.debug('Cannot index file:', exc_info=True) pass try: @@ -320,7 +317,7 @@ class nodefd_db64(StringIO, nodes.node_descriptor): (base64.encodestring(data), len(data), par.file_id)) cr.commit() except Exception: - logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) + _logger.exception('Cannot update db file #%d for close.', par.file_id) raise finally: cr.close() @@ -333,13 +330,12 @@ class document_storage(osv.osv): media. The referring document.directory-ies will control the placement of data into the storage. - + It is a bad idea to have multiple document.storage objects pointing to the same tree of filesystem storage. """ _name = 'document.storage' _description = 'Storage Media' - _doclog = logging.getLogger('document') _columns = { 'name': fields.char('Name', size=64, required=True, select=1), @@ -388,12 +384,12 @@ class document_storage(osv.osv): def __prepare_realpath(self, cr, file_node, ira, store_path, do_create=True): """ Cleanup path for realstore, create dirs if needed - + @param file_node the node @param ira ir.attachment browse of the file_node @param store_path the path of the parent storage object, list @param do_create create the directories, if needed - + @return tuple(path "/var/filestore/real/dir/", npath ['dir','fname.ext'] ) """ file_node.fix_ppath(cr, ira) @@ -402,18 +398,18 @@ class document_storage(osv.osv): npath = filter(lambda x: x is not None, npath) # if self._debug: - # self._doclog.debug('Npath: %s', npath) + # self._logger.debug('Npath: %s', npath) for n in npath: if n == '..': - raise ValueError("Invalid '..' element in path") + raise ValueError("Invalid '..' element in path.") for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?',): if ch in n: - raise ValueError("Invalid char %s in path %s" %(ch, n)) + raise ValueError("Invalid char %s in path %s." %(ch, n)) dpath = [store_path,] dpath += npath[:-1] path = os.path.join(*dpath) if not os.path.isdir(path): - self._doclog.debug("Create dirs: %s", path) + _logger.debug("Create dirs: %s", path) os.makedirs(path) return path, npath @@ -424,8 +420,8 @@ class document_storage(osv.osv): """ boo = self.browse(cr, uid, id, context=context) if not boo.online: - raise IOError(errno.EREMOTE, 'medium offline') - + raise IOError(errno.EREMOTE, 'Medium offline.') + if fil_obj: ira = fil_obj else: @@ -439,11 +435,11 @@ class document_storage(osv.osv): context = {} boo = self.browse(cr, uid, id, context=context) if not boo.online: - raise IOError(errno.EREMOTE, 'medium offline') - + raise IOError(errno.EREMOTE, 'Medium offline.') + if boo.readonly and mode not in ('r', 'rb'): - raise IOError(errno.EPERM, "Readonly medium") - + raise IOError(errno.EPERM, "Readonly medium.") + ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) if boo.type == 'filestore': if not ira.store_fname: @@ -451,8 +447,8 @@ class document_storage(osv.osv): # try to fix their directory. if mode in ('r','r+'): if ira.file_size: - self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id) - raise IOError(errno.ENOENT, 'No file can be located') + _logger.warning( "ir.attachment #%d does not have a filename, but is at filestore. This should get fixed." % ira.id) + raise IOError(errno.ENOENT, 'No file can be located.') else: store_fname = self.__get_random_fname(boo.path) cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s', @@ -474,7 +470,7 @@ class document_storage(osv.osv): do_create = (mode[0] in ('w','a')) ) fpath = os.path.join(path, npath[-1]) if (not os.path.exists(fpath)) and mode[0] == 'r': - raise IOError("File not found: %s" % fpath) + raise IOError("File not found: %s." % fpath) elif mode[0] in ('w', 'a') and not ira.store_fname: store_fname = os.path.join(*npath) cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s', @@ -482,10 +478,10 @@ class document_storage(osv.osv): return nodefd_file(file_node, path=fpath, mode=mode) elif boo.type == 'virtual': - raise ValueError('Virtual storage does not support static files') - + raise ValueError('Virtual storage does not support static file(s).') + else: - raise TypeError("No %s storage" % boo.type) + raise TypeError("No %s storage." % boo.type) def __get_data_3(self, cr, uid, boo, ira, context): if boo.type == 'filestore': @@ -493,7 +489,7 @@ class document_storage(osv.osv): # On a migrated db, some files may have the wrong storage type # try to fix their directory. if ira.file_size: - self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id) + _logger.warning( "ir.attachment #%d does not have a filename, but is at filestore. This should get fixed." % ira.id) return None fpath = os.path.join(boo.path, ira.store_fname) return file(fpath, 'rb').read() @@ -517,7 +513,7 @@ class document_storage(osv.osv): # On a migrated db, some files may have the wrong storage type # try to fix their directory. if ira.file_size: - self._doclog.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id) + _logger.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id) # sfname = ira.name fpath = os.path.join(boo.path,ira.store_fname or ira.name) if os.path.exists(fpath): @@ -525,13 +521,13 @@ class document_storage(osv.osv): elif not ira.store_fname: return None else: - raise IOError(errno.ENOENT, "File not found: %s" % fpath) + raise IOError(errno.ENOENT, "File not found: %s." % fpath) elif boo.type == 'virtual': - raise ValueError('Virtual storage does not support static files') + raise ValueError('Virtual storage does not support static file(s).') else: - raise TypeError("No %s storage" % boo.type) + raise TypeError("No %s storage!" % boo.type) def set_data(self, cr, uid, id, file_node, data, context=None, fil_obj=None): """ store the data. @@ -545,12 +541,12 @@ class document_storage(osv.osv): ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) if not boo.online: - raise IOError(errno.EREMOTE, 'medium offline') - + raise IOError(errno.EREMOTE, 'Medium offline.') + if boo.readonly: - raise IOError(errno.EPERM, "Readonly medium") + raise IOError(errno.EPERM, "Readonly medium.") - self._doclog.debug( "Store data for ir.attachment #%d" % ira.id) + _logger.debug( "Store data for ir.attachment #%d." % ira.id) store_fname = None fname = None if boo.type == 'filestore': @@ -561,15 +557,15 @@ class document_storage(osv.osv): fp = open(fname, 'wb') try: fp.write(data) - finally: + finally: fp.close() - self._doclog.debug( "Saved data to %s" % fname) + _logger.debug( "Saved data to %s." % fname) filesize = len(data) # os.stat(fname).st_size - + # TODO Here, an old file would be left hanging. except Exception, e: - self._doclog.warning( "Couldn't save data to %s", path, exc_info=True) + _logger.warning( "Cannot save data to %s.", path, exc_info=True) raise except_orm(_('Error!'), str(e)) elif boo.type == 'db': filesize = len(data) @@ -590,21 +586,21 @@ class document_storage(osv.osv): fp = open(fname,'wb') try: fp.write(data) - finally: + finally: fp.close() - self._doclog.debug("Saved data to %s", fname) + _logger.debug("Saved data to %s.", fname) filesize = len(data) # os.stat(fname).st_size store_fname = os.path.join(*npath) # TODO Here, an old file would be left hanging. except Exception,e : - self._doclog.warning("Couldn't save data:", exc_info=True) + _logger.warning("Cannot save data.", exc_info=True) raise except_orm(_('Error!'), str(e)) elif boo.type == 'virtual': - raise ValueError('Virtual storage does not support static files') + raise ValueError('Virtual storage does not support static file(s).') else: - raise TypeError("No %s storage" % boo.type) + raise TypeError("No %s storage!" % boo.type) # 2nd phase: store the metadata try: @@ -616,7 +612,7 @@ class document_storage(osv.osv): mime, icont = cntIndex.doIndex(data, ira.datas_fname, ira.file_type or None, fname) except Exception: - self._doclog.debug('Cannot index file:', exc_info=True) + _logger.debug('Cannot index file.', exc_info=True) pass try: @@ -633,7 +629,7 @@ class document_storage(osv.osv): file_node.content_type = mime return True except Exception, e : - self._doclog.warning("Couldn't save data:", exc_info=True) + self._logger.warning("Cannot save data.", exc_info=True) # should we really rollback once we have written the actual data? # at the db case (only), that rollback would be safe raise except_orm(_('Error at doc write!'), str(e)) @@ -643,10 +639,10 @@ class document_storage(osv.osv): files that have to be removed, too. """ if not storage_bo.online: - raise IOError(errno.EREMOTE, 'medium offline') - + raise IOError(errno.EREMOTE, 'Medium offline.') + if storage_bo.readonly: - raise IOError(errno.EPERM, "Readonly medium") + raise IOError(errno.EPERM, "Readonly medium.") if storage_bo.type == 'filestore': fname = fil_bo.store_fname @@ -663,7 +659,7 @@ class document_storage(osv.osv): path = storage_bo.path return ( storage_bo.id, 'file', os.path.join(path, fname)) else: - raise TypeError("No %s storage" % storage_bo.type) + raise TypeError("No %s storage!" % storage_bo.type) def do_unlink(self, cr, uid, unres): for id, ktype, fname in unres: @@ -671,9 +667,9 @@ class document_storage(osv.osv): try: os.unlink(fname) except Exception: - self._doclog.warning("Could not remove file %s, please remove manually.", fname, exc_info=True) + _logger.warning("Cannot remove file %s, please remove it manually.", fname, exc_info=True) else: - self._doclog.warning("Unknown unlink key %s" % ktype) + _logger.warning("Unlink unknown key %s." % ktype) return True @@ -681,17 +677,17 @@ class document_storage(osv.osv): """ A preparation for a file rename. It will not affect the database, but merely check and perhaps rename the realstore file. - + @return the dict of values that can safely be be stored in the db. """ sbro = self.browse(cr, uid, file_node.storage_id, context=context) assert sbro, "The file #%d didn't provide storage" % file_node.file_id if not sbro.online: - raise IOError(errno.EREMOTE, 'medium offline') - + raise IOError(errno.EREMOTE, 'Medium offline.') + if sbro.readonly: - raise IOError(errno.EPERM, "Readonly medium") + raise IOError(errno.EPERM, "Readonly medium.") if sbro.type in ('filestore', 'db', 'db64'): # nothing to do for a rename, allow to change the db field @@ -703,9 +699,9 @@ class document_storage(osv.osv): fname = ira.store_fname if not fname: - self._doclog.warning("Trying to rename a non-stored file") + _logger.warning("Trying to rename a non-stored file.") if fname != os.path.join(*npath): - self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(npath)) + _logger.warning("Inconsistency to realstore: %s != %s." , fname, repr(npath)) oldpath = os.path.join(path, npath[-1]) newpath = os.path.join(path, new_name) @@ -715,13 +711,13 @@ class document_storage(osv.osv): store_fname = os.path.join(*store_path) return { 'name': new_name, 'datas_fname': new_name, 'store_fname': store_fname } else: - raise TypeError("No %s storage" % sbro.type) + raise TypeError("No %s storage!" % sbro.type) def simple_move(self, cr, uid, file_node, ndir_bro, context=None): """ A preparation for a file move. It will not affect the database, but merely check and perhaps move the realstore file. - + @param ndir_bro a browse object of document.directory, where this file should move to. @return the dict of values that can safely be be stored in the db. @@ -730,10 +726,10 @@ class document_storage(osv.osv): assert sbro, "The file #%d didn't provide storage" % file_node.file_id if not sbro.online: - raise IOError(errno.EREMOTE, 'medium offline') - + raise IOError(errno.EREMOTE, 'Medium offline.') + if sbro.readonly: - raise IOError(errno.EPERM, "Readonly medium") + raise IOError(errno.EPERM, "Readonly medium.") par = ndir_bro psto = None @@ -743,8 +739,8 @@ class document_storage(osv.osv): break par = par.parent_id if file_node.storage_id != psto: - self._doclog.debug('Cannot move file %r from %r to %r', file_node, file_node.parent, ndir_bro.name) - raise NotImplementedError('Cannot move files between storage media') + _logger.debug('Cannot move file %r from %r to %r.', file_node, file_node.parent, ndir_bro.name) + raise NotImplementedError('Cannot move file(s) between storage media.') if sbro.type in ('filestore', 'db', 'db64'): # nothing to do for a rename, allow to change the db field @@ -756,30 +752,30 @@ class document_storage(osv.osv): fname = ira.store_fname if not fname: - self._doclog.warning("Trying to rename a non-stored file") + _logger.warning("Trying to rename a non-stored file.") if fname != os.path.join(*opath): - self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(opath)) + _logger.warning("Inconsistency to realstore: %s != %s." , fname, repr(opath)) oldpath = os.path.join(path, opath[-1]) - + npath = [sbro.path,] + (ndir_bro.get_full_path() or []) npath = filter(lambda x: x is not None, npath) newdir = os.path.join(*npath) if not os.path.isdir(newdir): - self._doclog.debug("Must create dir %s", newdir) + _logger.debug("Must create dir %s.", newdir) os.makedirs(newdir) npath.append(opath[-1]) newpath = os.path.join(*npath) - - self._doclog.debug("Going to move %s from %s to %s", opath[-1], oldpath, newpath) + + _logger.debug("Going to move %s from %s to %s.", opath[-1], oldpath, newpath) shutil.move(oldpath, newpath) - + store_path = npath[1:] + [opath[-1],] store_fname = os.path.join(*store_path) - + return { 'store_fname': store_fname } else: - raise TypeError("No %s storage" % sbro.type) + raise TypeError("No %s storage." % sbro.type) document_storage() |