diff options
Diffstat (limited to 'addons/import_base/import_framework.py')
| -rw-r--r-- | addons/import_base/import_framework.py | 501 |
1 files changed, 0 insertions, 501 deletions
diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py deleted file mode 100644 index 559f42c8974..00000000000 --- a/addons/import_base/import_framework.py +++ /dev/null @@ -1,501 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -############################################################################## -import pprint -import mapper -import pooler -import tools -from tools.translate import _ - -from threading import Thread -import datetime -import logging -import StringIO -import traceback -pp = pprint.PrettyPrinter(indent=4) -_logger = logging.getLogger(__name__) - - - -class import_framework(Thread): - """ - This class should be extends, - get_data and get_mapping have to extends - get_state_map and initialize can be extended - for advanced purpose get_default_hook can also be extended - @see dummy import for a minimal exemple - """ - - """ - for import_object, this domain will avoid to find an already existing object - """ - DO_NOT_FIND_DOMAIN = [('id', '=', 0)] - - #TODO don't use context to pass credential parameters - def __init__(self, obj, cr, uid, instance_name, module_name, email_to_notify=False, context=None): - Thread.__init__(self) - self.external_id_field = 'id' - self.obj = obj - self.cr = cr - self.uid = uid - self.instance_name = instance_name - self.module_name = module_name - self.context = context or {} - self.email = email_to_notify - self.table_list = [] - self.initialize() - - """ - Abstract Method to be implemented in - the real instance - """ - def initialize(self): - """ - init before import - usually for the login - """ - pass - - def init_run(self): - """ - call after intialize run in the thread, not in the main process - TO use for long initialization operation - """ - pass - - def get_data(self, table): - """ - @return: a list of dictionaries - each dictionnaries contains the list of pair external_field_name : value - """ - return [{}] - - def get_link(self, from_table, ids, to_table): - """ - @return: a dictionaries that contains the association between the id (from_table) - and the list (to table) of id linked - """ - return {} - - def get_external_id(self, data): - """ - @return the external id - the default implementation return self.external_id_field (that has 'id') by default - if the name of id field is different, you can overwrite this method or change the value - of self.external_id_field - """ - return data[self.external_id_field] - - def get_mapping(self): - """ - @return: { TABLE_NAME : { - 'model' : 'openerp.model.name', - #if true import the table if not just resolve dependencies, use for meta package, by default => True - #Not required - 'import' : True or False, - #Not required - 'dependencies' : [TABLE_1, TABLE_2], - #Not required - 'hook' : self.function_name, #get the val dict of the object, return the same val dict or False - 'map' : { @see mapper - 'openerp_field_name' : 'external_field_name', or val('external_field_name') - 'openerp_field_id/id' : ref(TABLE_1, 'external_id_field'), #make the mapping between the external id and the xml on the right - 'openerp_field2_id/id_parent' : ref(TABLE_1,'external_id_field') #indicate a self dependencies on openerp_field2_id - 'state' : map_val('state_equivalent_field', mapping), # use get_state_map to make the mapping between the value of the field and the value of the state - 'text_field' : concat('field_1', 'field_2', .., delimiter=':'), #concat the value of the list of field in one - 'description_field' : ppconcat('field_1', 'field_2', .., delimiter='\n\t'), #same as above but with a prettier formatting - 'field' : call(callable, arg1, arg2, ..), #call the function with all the value, the function should send the value : self.callable - 'field' : callable - 'field' : call(method, val('external_field') interface of method is self, val where val is the value of the field - 'field' : const(value) #always set this field to value - + any custom mapper that you will define - } - }, - - } - """ - return {} - - def default_hook(self, val): - """ - this hook will be apply on each table that don't have hook - here we define the identity hook - """ - return val - - def _import_table(self, table): - data = self.get_data(table) - map = self.get_mapping()[table]['map'] - hook = self.get_mapping()[table].get('hook', self.default_hook) - model = self.get_mapping()[table]['model'] - - final_data = [] - for val in data: - res = hook(val) - if res: - final_data.append(res) - return self._save_data(model, dict(map), final_data, table) - - def _save_data(self, model, mapping, datas, table): - """ - @param model: the model of the object to import - @param table : the external table where the data come from - @param mapping : definition of the mapping - @see: get_mapping - @param datas : list of dictionnaries - datas = [data_1, data_2, ..] - data_i is a map external field_name => value - and each data_i have a external id => in data_id['id'] - """ - _logger.info(' Importing %s into %s' % (table, model)) - if not datas: - return (0, 'No data found') - mapping['id'] = 'id_new' - res = [] - - - self_dependencies = [] - for k in mapping.keys(): - if '_parent' in k: - self_dependencies.append((k[:-7], mapping.pop(k))) - - for data in datas: - for k, field_name in self_dependencies: - data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table) - - data['id_new'] = self._generate_xml_id(self.get_external_id(data), table) - fields, values = self._fields_mapp(data, mapping, table) - res.append(values) - - model_obj = self.obj.pool.get(model) - if not model_obj: - raise ValueError(_("%s is not a valid model name.") % model) - _logger.debug(_(" fields imported : ") + str(fields)) - (p, r, warning, s) = model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context) - for (field, field_name) in self_dependencies: - self._import_self_dependencies(model_obj, field, datas) - return (len(res), warning) - - def _import_self_dependencies(self, obj, parent_field, datas): - """ - @param parent_field: the name of the field that generate a self_dependencies, we call the object referenced in this - field the parent of the object - @param datas: a list of dictionnaries - Dictionnaries need to contains - id_new : the xml_id of the object - field_new : the xml_id of the parent - """ - fields = ['id', parent_field] - for data in datas: - if data.get(parent_field): - values = [data['id_new'], data[parent_field]] - obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=True, context=self.context) - - def _preprocess_mapping(self, mapping): - """ - Preprocess the mapping : - after the preprocces, everything is - callable in the val of the dictionary - - use to allow syntaxical sugar like 'field': 'external_field' - instead of 'field' : value('external_field') - """ - map = dict(mapping) - for key, value in map.items(): - if isinstance(value, basestring): - map[key] = mapper.value(value) - #set parent for instance of dbmapper - elif isinstance(value, mapper.dbmapper): - value.set_parent(self) - return map - - - def _fields_mapp(self,dict_sugar, openerp_dict, table): - """ - call all the mapper and transform data - to be compatible with import_data - """ - fields=[] - data_lst = [] - mapping = self._preprocess_mapping(openerp_dict) - for key,val in mapping.items(): - if key not in fields and dict_sugar: - fields.append(key) - value = val(dict(dict_sugar)) - data_lst.append(value) - return fields, data_lst - - def _generate_xml_id(self, name, table): - """ - @param name: name of the object, has to be unique in for a given table - @param table : table where the record we want generate come from - @return: a unique xml id for record, the xml_id will be the same given the same table and same name - To be used to avoid duplication of data that don't have ids - """ - sugar_instance = self.instance_name - name = name.replace('.', '_').replace(',', '_') - return sugar_instance + "_" + table + "_" + name - - - """ - Public interface of the framework - those function can be use in the callable function defined in the mapping - """ - def xml_id_exist(self, table, external_id): - """ - Check if the external id exist in the openerp database - in order to check if the id exist the table where it come from - should be provide - @return the xml_id generated if the external_id exist in the database or false - """ - if not external_id: - return False - - xml_id = self._generate_xml_id(external_id, table) - id = self.obj.pool.get('ir.model.data').search(self.cr, self.uid, [('name', '=', xml_id), ('module', '=', self.module_name)]) - return id and xml_id or False - - def name_exist(self, table, name, model): - """ - Check if the object with the name exist in the openerp database - in order to check if the id exist the table where it come from - should be provide and the model of the object - """ - fields = ['name'] - data = [name] - return self.import_object(fields, data, model, table, name, [('name', '=', name)]) - - def get_mapped_id(self, table, external_id, context=None): - """ - @return return the databse id linked with the external_id - """ - if not external_id: - return False - - xml_id = self._generate_xml_id(external_id, table) - return self.obj.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, self.module_name, xml_id)[1] - - def import_object_mapping(self, mapping, data, model, table, name, domain_search=False): - """ - same as import_objects but instead of two list fields and data, - this method take a dictionnaries : external_field : value - and the mapping similar to the one define in 'map' key - @see import_object, get_mapping - """ - fields, datas = self._fields_mapp(data, mapping, table) - return self.import_object(fields, datas, model, table, name, domain_search) - - def import_object(self, fields, data, model, table, name, domain_search=False): - """ - This method will import an object in the openerp, usefull for field that is only a char in sugar and is an object in openerp - use import_data that will take care to create/update or do nothing with the data - this method return the xml_id - - To be use, when you want to create an object or link if already exist - use DO_NOT_LINK_DOMAIN to create always a new object - @param fields: list of fields needed to create the object without id - @param data: the list of the data, in the same order as the field - ex : fields = ['firstname', 'lastname'] ; data = ['John', 'Mc donalds'] - @param model: the openerp's model of the create/update object - @param table: the table where data come from in sugarcrm, no need to fit the real name of openerp name, just need to be unique - @param unique_name: the name of the object that we want to create/update get the id - @param domain_search : the domain that should find the unique existing record - - @return: the xml_id of the ressources - """ - domain_search = not domain_search and [('name', 'ilike', name)] or domain_search - obj = self.obj.pool.get(model) - if not obj: #if the model doesn't exist - return False - - xml_id = self._generate_xml_id(name, table) - xml_ref = self.mapped_id_if_exist(model, domain_search, table, name) - fields.append('id') - data.append(xml_id) - obj.import_data(self.cr, self.uid, fields, [data], mode='update', current_module=self.module_name, noupdate=True, context=self.context) - return xml_ref or xml_id - - - def mapped_id_if_exist(self, model, domain, table, name): - """ - To be use when we want link with and existing object, if the object don't exist - just ignore. - @param domain : search domain to find existing record, should return a unique record - @param xml_id: xml_id give to the mapping - @param name: external_id or name of the object to create if there is no id - @param table: the name of the table of the object to map - @return : the xml_id if the record exist in the db, False otherwise - """ - obj = self.obj.pool.get(model) - ids = obj.search(self.cr, self.uid, domain, context=self.context) - if ids: - xml_id = self._generate_xml_id(name, table) - ir_model_data_obj = obj.pool.get('ir.model.data') - id = ir_model_data_obj._update(self.cr, self.uid, model, - self.module_name, {}, mode='update', xml_id=xml_id, - noupdate=True, res_id=ids[0], context=self.context) - return xml_id - return False - - - def set_table_list(self, table_list): - """ - Set the list of table to import, this method should be call before run - @param table_list: the list of external table to import - ['Leads', 'Opportunity'] - """ - self.table_list = table_list - - def run(self): - """ - Import all data into openerp, - this is the Entry point to launch the process of import - - - """ - self.data_started = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - self.cr = pooler.get_db(self.cr.dbname).cursor() - error = False - result = [] - try: - self.init_run() - imported = set() #to invoid importing 2 times the sames modules - for table in self.table_list: - to_import = self.get_mapping()[table].get('import', True) - if not table in imported: - res = self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported) - result.extend(res) - if to_import: - (position, warning) = self._import_table(table) - result.append((table, position, warning)) - imported.add(table) - self.cr.commit() - - except Exception, err: - sh = StringIO.StringIO() - traceback.print_exc(file=sh) - error = sh.getvalue() - print error - - - self.date_ended = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - self._send_notification_email(result, error) - - self.cr.close() - - def _resolve_dependencies(self, dep, imported): - """ - import dependencies recursively - and avoid to import twice the same table - """ - result = [] - for dependency in dep: - if not dependency in imported: - to_import = self.get_mapping()[dependency].get('import', True) - res = self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported) - result.extend(res) - if to_import: - r = self._import_table(dependency) - (position, warning) = r - result.append((dependency, position, warning)) - imported.add(dependency) - return result - - def _send_notification_email(self, result, error): - if not self.email: - return False - email_obj = self.obj.pool.get('mail.mail') - email_id = email_obj.create(self.cr, self.uid, { - 'email_from' : 'import@module.openerp', - 'email_to' : self.email, - 'body_html' : self.get_email_body(result, error), - 'subject' : self.get_email_subject(result, error), - 'auto_delete' : True}) - email_obj.send(self.cr, self.uid, [email_id]) - if error: - _logger.error(_("Import failed due to an unexpected error")) - else: - _logger.info(_("Import finished, notification email sended")) - - def get_email_subject(self, result, error=False): - """ - This method define the subject of the email send by openerp at the end of import - @param result: a list of tuple - (table_name, number_of_record_created/updated, warning) for each table - @return the subject of the mail - - """ - if error: - return _("Data Import failed at %s due to an unexpected error") % self.date_ended - return _("Import of your data finished at %s") % self.date_ended - - def get_email_body(self, result, error=False): - """ - This method define the body of the email send by openerp at the end of import. The body is separated in two part - the header (@see get_body_header), and the generate body with the list of table and number of record imported. - If you want to keep this it's better to overwrite get_body_header - @param result: a list of tuple - (table_name, number_of_record_created/updated, warning) for each table - @return the subject of the mail - - """ - - body = _("started at %s and finished at %s \n") % (self.data_started, self.date_ended) - if error: - body += _("but failed, in consequence no data were imported to keep database consistency \n error : \n") + error - - for (table, nb, warning) in result: - if not warning: - warning = _("with no warning") - else: - warning = _("with warning : %s") % warning - body += _("%s has been successfully imported from %s %s, %s \n") % (nb, self.instance_name, table, warning) - return self.get_body_header(result) + "\n\n" + body - - def get_body_header(self, result): - """ - @return the first sentences written in the mail's body - """ - return _("The import of data \n instance name : %s \n") % self.instance_name - - - #TODO documentation test - def run_test(self): - back_get_data = self.get_data - back_get_link = self.get_link - back_init = self.initialize - self.get_data = self.get_data_test - self.get_link = self.get_link_test - self.initialize = self.intialize_test - self.run() - self.get_data = back_get_data - self.get_link = back_get_link - self.initialize = back_init - - def get_data_test(self, table): - return [{}] - - def get_link_test(self, from_table, ids, to_table): - return {} - - def intialize_test(self): - pass - -# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: |