diff options
Diffstat (limited to 'addons/import_base')
| -rw-r--r-- | addons/import_base/__init__.py | 23 | ||||
| -rw-r--r-- | addons/import_base/__openerp__.py | 39 | ||||
| -rw-r--r-- | addons/import_base/import_framework.py | 432 | ||||
| -rw-r--r-- | addons/import_base/mapper.py | 173 |
4 files changed, 667 insertions, 0 deletions
diff --git a/addons/import_base/__init__.py b/addons/import_base/__init__.py new file mode 100644 index 00000000000..0a5d7fb20b7 --- /dev/null +++ b/addons/import_base/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Openerp sa (<http://openerp.com>). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +import import_framework +import mapper
\ No newline at end of file diff --git a/addons/import_base/__openerp__.py b/addons/import_base/__openerp__.py new file mode 100644 index 00000000000..962541f427a --- /dev/null +++ b/addons/import_base/__openerp__.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +{ + 'name': 'Base framework for module that need to import complex data', + 'version': '0.9', + 'category': 'Generic Modules', + 'description': """ + This module provide a class import_framework to help importing + complex data from other software + """, + 'author': 'OpenERP SA', + 'website': 'http://www.openerp.com', + 'depends': ['base'], + 'init_xml': [], + 'update_xml': [], + 'demo_xml': [], + 'test': [], #TODO provide test + 'installable': True, + 'active': False, +} diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py new file mode 100644 index 00000000000..9b25ecb1ecf --- /dev/null +++ b/addons/import_base/import_framework.py @@ -0,0 +1,432 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## +import pprint +import mapper +import pooler +import tools + + +from threading import Thread +import datetime +import logging +pp = pprint.PrettyPrinter(indent=4) + + + + +class import_framework(Thread): + """ + This class should be extends, + get_data and get_mapping have to extends + get_state_map and initialize can be extended + for advanced purpose get_default_hook can also be extended + @see dummy import for a minimal exemple + """ + + """ + for import_object, this domain will avoid to find an already existing object + """ + DO_NOT_FIND_DOMAIN = [('id', '=', 0)] + + def __init__(self, obj, cr, uid, instance_name, module_name, email_to_notify=False, context=None): + Thread.__init__(self) + self.obj = obj + self.cr = cr + self.uid = uid + self.instance_name = instance_name + self.module_name = module_name + self.context = context or {} + self.email = email_to_notify + self.table_list = [] + self.initialize() + + + + """ + Abstract Method to be implemented in + the real instance + """ + def initialize(self): + """ + init before import + usually for the login + """ + pass + + def get_data(self, table): + """ + @return: a list of dictionaries + each dictionnaries contains the list of pair external_field_name : value + """ + return [{}] + + def get_mapping(self): + """ + @return: { TABLE_NAME : { + 'model' : 'openerp.model.name', + #if true import the table if not just resolve dependencies, use for meta package, by default => True + #Not required + 'import' : True or False, + #Not required + 'dependencies' : [TABLE_1, TABLE_2], + #Not required + 'hook' : self.function_name, #get the val dict of the object, return the same val dict or False + 'map' : { @see mapper + 'openerp_field_name' : 'external_field_name', or val('external_field_name') + 'openerp_field_id/id' : ref(TABLE_1, 'external_id_field'), #make the mapping between the external id and the xml on the right + 'openerp_field2_id/id_parent' : ref(TABLE_1,'external_id_field') #indicate a self dependencies on openerp_field2_id + 'state' : map_val('state_equivalent_field', mapping), # use get_state_map to make the mapping between the value of the field and the value of the state + 'text_field' : concat('field_1', 'field_2', .., delimiter=':'), #concat the value of the list of field in one + 'description_field' : ppconcat('field_1', 'field_2', .., delimiter='\n\t'), #same as above but with a prettier formatting + 'field' : call(callable, arg1, arg2, ..), #call the function with all the value, the function should send the value : self.callable + 'field' : callable + 'field' : call(method, val('external_field') interface of method is self, val where val is the value of the field + 'field' : const(value) #always set this field to value + + any custom mapper that you will define + } + }, + + } + """ + return {} + + def default_hook(self, val): + """ + this hook will be apply on each table that don't have hook + here we define the identity hook + """ + return val + + def _import_table(self, table): + data = self.get_data(table) + map = self.get_mapping()[table]['map'] + hook = self.get_mapping()[table].get('hook', self.default_hook) + model = self.get_mapping()[table]['model'] + + final_data = [] + for val in data: + res = hook(val) + if res: + final_data.append(res) + return self._save_data(model, dict(map), final_data, table) + + def _save_data(self, model, mapping, datas, table): + """ + @param model: the model of the object to import + @param table : the external table where the data come from + @param mapping : definition of the mapping + @see: get_mapping + @param datas : list of dictionnaries + datas = [data_1, data_2, ..] + data_i is a map external field_name => value + and each data_i have a external id => in data_id['id'] + """ + if not datas: + return (0, 'No data in this table') + mapping['id'] = 'id_new' + res = [] + + + self_dependencies = [] + for k in mapping.keys(): + if '_parent' in k: + self_dependencies.append((k[:-7], mapping.pop(k))) + + for data in datas: + for k, field_name in self_dependencies: + data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table) + + data['id_new'] = self._generate_xml_id(data['id'], table) + fields, values = self._fields_mapp(data, mapping, table) + res.append(values) + + model_obj = self.obj.pool.get(model) + if not model_obj: + raise ValueError("%s is not a valid model name" % model) + + (p, r, warning, s) = model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context) + for (field, field_name) in self_dependencies: + self._import_self_dependencies(model_obj, field, datas) + return (len(res), warning) + + def _import_self_dependencies(self, obj, parent_field, datas): + """ + @param parent_field: the name of the field that generate a self_dependencies, we call the object referenced in this + field the parent of the object + @param datas: a list of dictionnaries + Dictionnaries need to contains + id_new : the xml_id of the object + field_new : the xml_id of the parent + """ + fields = ['id', parent_field] + for data in datas: + if data.get(parent_field): + values = [data['id_new'], data[parent_field]] + obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=True, context=self.context) + + def _preprocess_mapping(self, mapping): + """ + Preprocess the mapping : + after the preprocces, everything is + callable in the val of the dictionary + + use to allow syntaxical sugar like 'field': 'external_field' + instead of 'field' : value('external_field') + """ + map = dict(mapping) + for key, value in map.items(): + if isinstance(value, basestring): + map[key] = mapper.value(value) + #set parent for instance of dbmapper + elif isinstance(value, mapper.dbmapper): + value.set_parent(self) + return map + + + def _fields_mapp(self,dict_sugar, openerp_dict, table): + """ + call all the mapper and transform data + to be compatible with import_data + """ + fields=[] + data_lst = [] + mapping = self._preprocess_mapping(openerp_dict) + for key,val in mapping.items(): + if key not in fields and dict_sugar: + fields.append(key) + value = val(dict(dict_sugar)) + data_lst.append(value) + return fields, data_lst + + def _generate_xml_id(self, name, table): + """ + @param name: name of the object, has to be unique in for a given table + @param table : table where the record we want generate come from + @return: a unique xml id for record, the xml_id will be the same given the same table and same name + To be used to avoid duplication of data that don't have ids + """ + sugar_instance = self.instance_name + name = name.replace('.', '_').replace(',', '_') + return sugar_instance + "_" + table + "_" + name + + + """ + Public interface of the framework + those function can be use in the callable function defined in the mapping + """ + def xml_id_exist(self, table, external_id): + """ + Check if the external id exist in the openerp database + in order to check if the id exist the table where it come from + should be provide + @return the xml_id generated if the external_id exist in the database or false + """ + if not external_id: + return False + + xml_id = self._generate_xml_id(external_id, table) + id = self.obj.pool.get('ir.model.data').search(self.cr, self.uid, [('name', '=', xml_id), ('module', '=', self.module_name)]) + return id and xml_id or False + + def name_exist(self, table, name, model): + """ + Check if the object with the name exist in the openerp database + in order to check if the id exist the table where it come from + should be provide and the model of the object + """ + fields = ['name'] + data = [name] + return self.import_object(fields, data, model, table, name, [('name', '=', name)]) + + def get_mapped_id(self, table, external_id, context=None): + """ + @return return the databse id linked with the external_id + """ + if not external_id: + return False + + xml_id = self._generate_xml_id(external_id, table) + return self.obj.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, self.module_name, xml_id)[1] + + def import_object_mapping(self, mapping, data, model, table, name, domain_search=False): + """ + same as import_objects but instead of two list fields and data, + this method take a dictionnaries : external_field : value + and the mapping similar to the one define in 'map' key + @see import_object, get_mapping + """ + fields, datas = self._fields_mapp(data, mapping, table) + return self.import_object(fields, datas, model, table, name, domain_search) + + def import_object(self, fields, data, model, table, name, domain_search=False): + """ + This method will import an object in the openerp, usefull for field that is only a char in sugar and is an object in openerp + use import_data that will take care to create/update or do nothing with the data + this method return the xml_id + + To be use, when you want to create an object or link if already exist + use DO_NOT_LINK_DOMAIN to create always a new object + @param fields: list of fields needed to create the object without id + @param data: the list of the data, in the same order as the field + ex : fields = ['firstname', 'lastname'] ; data = ['John', 'Mc donalds'] + @param model: the openerp's model of the create/update object + @param table: the table where data come from in sugarcrm, no need to fit the real name of openerp name, just need to be unique + @param unique_name: the name of the object that we want to create/update get the id + @param domain_search : the domain that should find the unique existing record + + @return: the xml_id of the ressources + """ + domain_search = not domain_search and [('name', 'ilike', name)] or domain_search + obj = self.obj.pool.get(model) + xml_id = self._generate_xml_id(name, table) + xml_ref = self.mapped_id_if_exist(model, domain_search, table, name) + fields.append('id') + data.append(xml_id) + obj.import_data(self.cr, self.uid, fields, [data], mode='update', current_module=self.module_name, noupdate=True, context=self.context) + return xml_ref or xml_id + + + def mapped_id_if_exist(self, model, domain, table, name): + """ + To be use when we want link with and existing object, if the object don't exist + just ignore. + @param domain : search domain to find existing record, should return a unique record + @param xml_id: xml_id give to the mapping + @param name: external_id or name of the object to create if there is no id + @param table: the name of the table of the object to map + @return : the xml_id if the record exist in the db, False otherwise + """ + obj = self.obj.pool.get(model) + ids = obj.search(self.cr, self.uid, domain, context=self.context) + if ids: + xml_id = self._generate_xml_id(name, table) + ir_model_data_obj = obj.pool.get('ir.model.data') + id = ir_model_data_obj._update(self.cr, self.uid, model, + self.module_name, {}, mode='update', xml_id=xml_id, + noupdate=True, res_id=ids[0], context=self.context) + return xml_id + return False + + + def set_table_list(self, table_list): + """ + Set the list of table to import, this method should be call before run + @param table_list: the list of external table to import + ['Leads', 'Opportunity'] + """ + self.table_list = table_list + + def run(self): + """ + Import all data into openerp, + this is the Entry point to launch the process of import + + + """ + self.data_started = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self.cr = pooler.get_db(self.cr.dbname).cursor() + try: + result = [] + imported = set() #to invoid importing 2 times the sames modules + for table in self.table_list: + to_import = self.get_mapping()[table].get('import', True) + if not table in imported: + res = self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported) + result.extend(res) + if to_import: + (position, warning) = self._import_table(table) + result.append((table, position, warning)) + imported.add(table) + self.cr.commit() + + finally: + self.cr.close() + self.date_ended = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self._send_notification_email(result) + + + def _resolve_dependencies(self, dep, imported): + """ + import dependencies recursively + and avoid to import twice the same table + """ + result = [] + for dependency in dep: + if not dependency in imported: + to_import = self.get_mapping()[dependency].get('import', True) + res = self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported) + result.extend(res) + if to_import: + r = self._import_table(dependency) + (position, warning) = r + result.append((dependency, position, warning)) + imported.add(dependency) + return result + + def _send_notification_email(self, result): + if not self.email: + return + tools.email_send( + 'import_sugarcrm@module.openerp', + self.email, + self.get_email_subject(result), + self.get_email_body(result), + ) + logger = logging.getLogger('import_sugarcam') + logger.info("Import finished, notification email sended") + + def get_email_subject(self, result): + """ + This method define the subject of the email send by openerp at the end of import + @param result: a list of tuple + (table_name, number_of_record_created/updated, warning) for each table + @return the subject of the mail + + """ + return "Import of your data finished at %s" % self.date_ended + + def get_email_body(self, result): + """ + This method define the body of the email send by openerp at the end of import. The body is separated in two part + the header (@see get_body_header), and the generate body with the list of table and number of record imported. + If you want to keep this it's better to overwrite get_body_header + @param result: a list of tuple + (table_name, number_of_record_created/updated, warning) for each table + @return the subject of the mail + + """ + + body = "started at %s and finished at %s \n" % (self.data_started, self.date_ended) + for (table, nb, warning) in result: + if not warning: + warning = "with no warning" + else: + warning = "with warning : %s" % warning + body += "%s records were imported from table %s, %s \n" % (nb, table, warning) + return self.get_body_header(result) + "\n\n" + body + + def get_body_header(self, result): + """ + @return the first sentences written in the mail's body + """ + return "The import of data \n instance name : %s \n" % self.instance_name + +#For example of use see import_sugarcrm diff --git a/addons/import_base/mapper.py b/addons/import_base/mapper.py new file mode 100644 index 00000000000..7f97ed0b615 --- /dev/null +++ b/addons/import_base/mapper.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + + +class mapper(object): + """ + super class for all mapper class + They are call before import data + to transform the mapping into real value that we + will import + + the call function receive a dictionary with external data + 'external_field' : value + """ + def __call__(self, external_values): + raise NotImplementedError() + +class dbmapper(mapper): + """ + Super class for mapper that need to access to + data base or any function of the import_framework + + self.parent contains a reference to the instance of + the import framework + """ + def set_parent(self, parent): + self.parent = parent + + +class concat(mapper): + """ + Use : contact('field_name1', 'field_name2', delimiter='_') + concat value of fields using the delimiter, delimiter is optional + and by default is a space + + """ + def __init__(self, *arg, **delimiter): + self.arg = arg + self.delimiter = delimiter and delimiter.get('delimiter', ' ') or ' ' + + def __call__(self, external_values): + return self.delimiter.join(map(lambda x : external_values.get(x,''), self.arg)) + +class ppconcat(mapper): + """ + Use : contact('field_name1', 'field_name2', delimiter='_') + concat external field name and value of fields using the delimiter, + delimiter is optional and by default is a two line feeds + + """ + def __init__(self, *arg, **delimiter): + self.arg = arg + self.delimiter = delimiter and delimiter.get('delimiter', ' ') or '\n\n' + + def __call__(self, external_values): + return self.delimiter.join(map(lambda x : x + ": " + external_values.get(x,''), self.arg)) + +class const(mapper): + """ + Use : const(arg) + return always arg + """ + def __init__(self, val): + self.val = val + + def __call__(self, external_values): + return self.val + +class value(mapper): + """ + Use : value(external_field_name) + Return the value of the external field name + this is equivalent to the a single string + + usefull for call if you want your call get the value + and don't care about the name of the field + call(self.method, value('field1')) + """ + def __init__(self, val, default='', fallback=False): + self.val = val + self.default = default + self.fallback = fallback + + def __call__(self, external_values): + val = external_values.get(self.val, self.default) + if self.fallback and (not val or val == self.default): + val = external_values.get(self.fallback, self.default) + return val + + + +class map_val(mapper): + """ + Use : map_val(external_field, val_mapping) + where val_mapping is a dictionary + with external_val : openerp_val + + usefull for selection field like state + to map value + """ + def __init__(self, val, map, default='draft'): + self.val = value(val) + self.map = map + self.default = default + + def __call__(self, external_values): + return self.map.get(self.val(external_values), self.default) + +class ref(dbmapper): + """ + Use : ref(table_name, external_id) + return the xml_id of the ressource + + to associate an already imported object with the current object + """ + def __init__(self, table, field_name): + self.table = table + self.field_name = field_name + + def __call__(self, external_values): + return self.parent.xml_id_exist(self.table, external_values.get(self.field_name)) + +class refbyname(dbmapper): + """ + Use : refbyname(table_name, external_name, res.model) + same as ref but use the name of the ressource to find it + """ + def __init__(self, table, field_name, model): + self.table = table + self.field_name = field_name + self.model = model + + def __call__(self, external_values): + v = external_values.get(self.field_name, '') + return self.parent.name_exist(self.table, v , self.model) + +class call(mapper): + """ + Use : call(function, arg1, arg2) + to call the function with external val follow by the arg specified + """ + def __init__(self, fun, *arg): + self.fun = fun + self.arg = arg + + def __call__(self, external_values): + args = [] + for arg in self.arg: + if isinstance(arg, mapper): + args.append(arg(external_values)) + else: + args.append(arg) + return self.fun(external_values, *args) + +
\ No newline at end of file |