diff options
| author | Benjamin Frantzen (bfr) <bfr@odoo.com> | 2020-11-24 09:41:01 +0000 |
|---|---|---|
| committer | Benjamin Frantzen (bfr) <bfr@odoo.com> | 2021-01-28 08:57:25 +0000 |
| commit | a9a46cf09b0c841c7fda95b5ed03033fc6936ca2 (patch) | |
| tree | b3d8715d149568fe50e4557877edf06c34ef2c27 /addons/account_edi/models | |
| parent | 06a66497b658d22d43187f9bd430cb747ee6c4a3 (diff) | |
[IMP] account_edi: added error_level + better management of Lock on account.move
- Some errors will never get fixed until user intervention, it doesn't make sense to run the CRON when there is such error.
+ some small improvements and esthetic changes
- When we are in a CRON, we need to commit the changes between each call to web-service to avoid loss of data.
- Small refactor of edi.document prepare_jobs and process_jobs
- Added an arbitrary key to create the batches.
Diffstat (limited to 'addons/account_edi/models')
| -rw-r--r-- | addons/account_edi/models/account_edi_document.py | 230 | ||||
| -rw-r--r-- | addons/account_edi/models/account_edi_format.py | 110 | ||||
| -rw-r--r-- | addons/account_edi/models/account_move.py | 28 |
3 files changed, 268 insertions, 100 deletions
diff --git a/addons/account_edi/models/account_edi_document.py b/addons/account_edi/models/account_edi_document.py index c5dc475e238..fd5396e955a 100644 --- a/addons/account_edi/models/account_edi_document.py +++ b/addons/account_edi/models/account_edi_document.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from odoo import models, fields, api, _ +from odoo import models, fields, api +from odoo.addons.account_edi_extended.models.account_edi_document import DEFAULT_BLOCKING_LEVEL from psycopg2 import OperationalError import logging @@ -17,7 +18,7 @@ class AccountEdiDocument(models.Model): edi_format_id = fields.Many2one('account.edi.format', required=True) attachment_id = fields.Many2one('ir.attachment', help='The file generated by edi_format_id when the invoice is posted (and this document is processed).') state = fields.Selection([('to_send', 'To Send'), ('sent', 'Sent'), ('to_cancel', 'To Cancel'), ('cancelled', 'Cancelled')]) - error = fields.Html() + error = fields.Html(help='The text of the last error that happened during Electronic Invoice operation.') # == Not stored fields == name = fields.Char(related='attachment_id.name') @@ -31,24 +32,32 @@ class AccountEdiDocument(models.Model): ), ] + def write(self, vals): + ''' If account_edi_extended is not installed, a default behaviour is used instead. + ''' + if 'blocking_level' in vals and 'blocking_level' not in self.env['account.edi.document']._fields: + vals.pop('blocking_level') + + return super().write(vals) + def _prepare_jobs(self): - """Creates a list of jobs to be performed by '_process_jobs' for the documents in self. + """Creates a list of jobs to be performed by '_process_job' for the documents in self. Each document represent a job, BUT if multiple documents have the same state, edi_format_id, doc_type (invoice or payment) and company_id AND the edi_format_id supports batching, they are grouped into a single job. - :returns: A list of tuples (key, documents) - * key: A tuple (edi_format_id, state, doc_type, company_id) - ** edi_format_id: The format to perform the operation with - ** state: The state of the documents of this job - ** doc_type: Are the moves of this job invoice or payments ? - ** company_id: The company the moves belong to - * documents: The documents related to this job. If edi_format_id does not support batch, length must be one + :returns: A list of tuples (documents, doc_type) + * documents: The documents related to this job. If edi_format_id does not support batch, length is one + * doc_type: Are the moves of this job invoice or payments ? """ - to_process = [] - batches = {} - for edi_doc in self.filtered(lambda d: d.state in ('to_send', 'to_cancel')): + # Classify jobs by (edi_format, edi_doc.state, doc_type, move.company_id, custom_key) + to_process = {} + if 'blocking_level' in self.env['account.edi.document']._fields: + documents = self.filtered(lambda d: d.state in ('to_send', 'to_cancel') and d.blocking_level != 'error') + else: + documents = self.filtered(lambda d: d.state in ('to_send', 'to_cancel')) + for edi_doc in documents: move = edi_doc.move_id edi_format = edi_doc.edi_format_id if move.is_invoice(include_receipts=True): @@ -58,20 +67,64 @@ class AccountEdiDocument(models.Model): else: continue - key = (edi_format, edi_doc.state, doc_type, move.company_id) - if edi_format._support_batching(): - if not batches.get(key, None): - batches[key] = self.env['account.edi.document'] - batches[key] |= edi_doc - else: - to_process.append((key, edi_doc)) - to_process.extend(batches.items()) - return to_process + custom_key = edi_format._get_batch_key(edi_doc.move_id, edi_doc.state) + key = (edi_format, edi_doc.state, doc_type, move.company_id, custom_key) + to_process.setdefault(key, self.env['account.edi.document']) + to_process[key] |= edi_doc + + # Order payments/invoice and create batches. + invoices = [] + payments = [] + for key, documents in to_process.items(): + edi_format, state, doc_type, company_id, custom_key = key + target = invoices if doc_type == 'invoice' else payments + batch = self.env['account.edi.document'] + for doc in documents: + if edi_format._support_batching(move=doc.move_id, state=state, company=company_id): + batch |= doc + else: + target.append((doc, doc_type)) + if batch: + target.append((batch, doc_type)) + return invoices + payments + + @api.model + def _convert_to_old_jobs_format(self, jobs): + """ See '_prepare_jobs' : + Old format : ((edi_format, state, doc_type, company_id), documents) + Since edi_format, state and company_id can be deduced from documents, this is redundant and more prone to unexpected behaviours. + New format : (doc_type, documents). + + However, for backward compatibility of 'process_jobs', we need a way to convert back to the old format. + """ + return [( + (documents.edi_format_id, documents[0].state, doc_type, documents.move_id.company_id), + documents + ) for documents, doc_type in jobs] @api.model def _process_jobs(self, to_process): + """ Deprecated, use _process_job instead. + + :param to_process: A list of tuples (key, documents) + * key: A tuple (edi_format_id, state, doc_type, company_id) + ** edi_format_id: The format to perform the operation with + ** state: The state of the documents of this job + ** doc_type: Are the moves of this job invoice or payments ? + ** company_id: The company the moves belong to + * documents: The documents related to this job. If edi_format_id does not support batch, length is one + """ + for key, documents in to_process: + edi_format, state, doc_type, company_id = key + self._process_job(documents, doc_type) + + @api.model + def _process_job(self, documents, doc_type): """Post or cancel move_id (invoice or payment) by calling the related methods on edi_format_id. Invoices are processed before payments. + + :param documents: The documents related to this job. If edi_format_id does not support batch, length is one + :param doc_type: Are the moves of this job invoice or payments ? """ def _postprocess_post_edi_results(documents, edi_result): attachments_to_unlink = self.env['ir.attachment'] @@ -83,6 +136,7 @@ class AccountEdiDocument(models.Model): values = { 'attachment_id': move_result['attachment'].id, 'error': move_result.get('error', False), + 'blocking_level': move_result.get('blocking_level', DEFAULT_BLOCKING_LEVEL) if 'error' in move_result else False, } if not values.get('error'): values.update({'state': 'sent'}) @@ -90,7 +144,10 @@ class AccountEdiDocument(models.Model): if not old_attachment.res_model or not old_attachment.res_id: attachments_to_unlink |= old_attachment else: - document.error = move_result.get('error', _("Error when processing the journal entry.")) + document.write({ + 'error': move_result.get('error', False), + 'blocking_level': move_result.get('blocking_level', DEFAULT_BLOCKING_LEVEL) if 'error' in move_result else False, + }) # Attachments that are not explicitly linked to a business model could be removed because they are not # supposed to have any traceability from the user. @@ -102,12 +159,13 @@ class AccountEdiDocument(models.Model): for document in documents: move = document.move_id move_result = edi_result.get(move, {}) - if move_result.get('success'): + if move_result.get('success') is True: old_attachment = document.attachment_id document.write({ 'state': 'cancelled', 'error': False, 'attachment_id': False, + 'blocking_level': False, }) if move.is_invoice(include_receipts=True) and move.state == 'posted': @@ -118,8 +176,11 @@ class AccountEdiDocument(models.Model): if not old_attachment.res_model or not old_attachment.res_id: attachments_to_unlink |= old_attachment - else: - document.error = move_result.get('error') or _("Error when cancelling the journal entry.") + elif not move_result.get('success'): + document.write({ + 'error': move_result.get('error', False), + 'blocking_level': move_result.get('blocking_level', DEFAULT_BLOCKING_LEVEL) if move_result.get('error') else False, + }) if invoice_ids_to_cancel: invoices = self.env['account.move'].browse(list(invoice_ids_to_cancel)) @@ -132,80 +193,65 @@ class AccountEdiDocument(models.Model): test_mode = self._context.get('edi_test_mode', False) - # ==== Process invoices ==== - payments = [] - for key, batches in to_process: - edi_format, state, doc_type, company_id = key - if doc_type == 'payment': - payments.append((key, batches)) - continue # payments are processed after invoices - - for documents in batches: - move_to_cancel = documents.filtered(lambda doc: doc.edi_format_id._needs_web_services() \ - and doc.attachment_id \ - and doc.state == 'to_cancel' \ - and doc.move_id.is_invoice(include_receipts=True) \ - and doc.edi_format_id._is_required_for_invoice(doc.move_id)).move_id - attachments_potential_unlink = documents.attachment_id.filtered(lambda a: not a.res_model and not a.res_id) - - try: - with self.env.cr.savepoint(): - # Locks the documents in DB. Avoid sending an invoice twice (the documents can be processed by the CRON but also manually). - self._cr.execute('SELECT * FROM account_edi_document WHERE id IN %s FOR UPDATE NOWAIT', [tuple(documents.ids)]) - - # Locks the move that will be cancelled. - if move_to_cancel: - self._cr.execute('SELECT * FROM account_move WHERE id IN %s FOR UPDATE NOWAIT', [tuple(move_to_cancel.ids)]) - - # Locks the attachments that might be unlinked - if attachments_potential_unlink: - self._cr.execute('SELECT * FROM ir_attachment WHERE id IN %s FOR UPDATE NOWAIT', [tuple(attachments_potential_unlink.ids)]) - - if state == 'to_send': - edi_result = edi_format._post_invoice_edi(documents.move_id, test_mode=test_mode) - _postprocess_post_edi_results(documents, edi_result) - elif state == 'to_cancel': - edi_result = edi_format._cancel_invoice_edi(documents.move_id, test_mode=test_mode) - _postprocess_cancel_edi_results(documents, edi_result) - - except OperationalError as e: - if e.pgcode == '55P03': - _logger.debug('Another transaction already locked documents rows. Cannot process documents.') - else: - raise e - - # ==== Process payments ==== - for key, batches in payments: - edi_format, state, doc_type, company_id = key - - for documents in batches: - try: - with self.env.cr.savepoint(): - self._cr.execute('SELECT * FROM account_edi_document WHERE id IN %s FOR UPDATE NOWAIT', [tuple(self.ids)]) - - if state == 'to_send': - edi_result = edi_format._post_payment_edi(documents.move_id, test_mode=test_mode) - _postprocess_post_edi_results(documents, edi_result) - elif state == 'to_cancel': - edi_result = edi_format._cancel_payment_edi(documents.move_id, test_mode=test_mode) - _postprocess_cancel_edi_results(documents, edi_result) - - except OperationalError as e: - if e.pgcode == '55P03': - _logger.debug('Another transaction already locked documents rows. Cannot process documents.') - else: - raise e + documents.edi_format_id.ensure_one() # All account.edi.document of a job should have the same edi_format_id + documents.move_id.company_id.ensure_one() # All account.edi.document of a job should be from the same company + if len(set(doc.state for doc in documents)) != 1: + raise ValueError('All account.edi.document of a job should have the same state') + + edi_format = documents.edi_format_id + state = documents[0].state + if doc_type == 'invoice': + if state == 'to_send': + edi_result = edi_format._post_invoice_edi(documents.move_id, test_mode=test_mode) + _postprocess_post_edi_results(documents, edi_result) + elif state == 'to_cancel': + edi_result = edi_format._cancel_invoice_edi(documents.move_id, test_mode=test_mode) + _postprocess_cancel_edi_results(documents, edi_result) + + elif doc_type == 'payment': + if state == 'to_send': + edi_result = edi_format._post_payment_edi(documents.move_id, test_mode=test_mode) + _postprocess_post_edi_results(documents, edi_result) + elif state == 'to_cancel': + edi_result = edi_format._cancel_payment_edi(documents.move_id, test_mode=test_mode) + _postprocess_cancel_edi_results(documents, edi_result) def _process_documents_no_web_services(self): """ Post and cancel all the documents that don't need a web service. """ jobs = self.filtered(lambda d: not d.edi_format_id._needs_web_services())._prepare_jobs() - self._process_jobs(jobs) + self._process_jobs(self._convert_to_old_jobs_format(jobs)) - def _process_documents_web_services(self, job_count=None): + def _process_documents_web_services(self, job_count=None, with_commit=True): """ Post and cancel all the documents that need a web service. This is called by CRON. :param job_count: Limit to the number of jobs to process among the ones that are available for treatment. """ jobs = self.filtered(lambda d: d.edi_format_id._needs_web_services())._prepare_jobs() - self._process_jobs(jobs[0:job_count or len(jobs)]) + jobs = jobs[0:job_count or len(jobs)] + for documents, doc_type in jobs: + move_to_cancel = documents.filtered(lambda doc: doc.attachment_id \ + and doc.state == 'to_cancel' \ + and doc.move_id.is_invoice(include_receipts=True) \ + and doc.edi_format_id._is_required_for_invoice(doc.move_id)).move_id + attachments_potential_unlink = documents.attachment_id.filtered(lambda a: not a.res_model and not a.res_id) + try: + with self.env.cr.savepoint(): + self._cr.execute('SELECT * FROM account_edi_document WHERE id IN %s FOR UPDATE NOWAIT', [tuple(documents.ids)]) + # Locks the move that will be cancelled. + if move_to_cancel: + self._cr.execute('SELECT * FROM account_move WHERE id IN %s FOR UPDATE NOWAIT', [tuple(move_to_cancel.ids)]) + + # Locks the attachments that might be unlinked + if attachments_potential_unlink: + self._cr.execute('SELECT * FROM ir_attachment WHERE id IN %s FOR UPDATE NOWAIT', [tuple(attachments_potential_unlink.ids)]) + + self._process_job(documents, doc_type) + except OperationalError as e: + if e.pgcode == '55P03': + _logger.debug('Another transaction already locked documents rows. Cannot process documents.') + else: + raise e + else: + if with_commit and len(jobs) > 1: + self.env.cr.commit() diff --git a/addons/account_edi/models/account_edi_format.py b/addons/account_edi/models/account_edi_format.py index 4313e78ddfc..36c36941625 100644 --- a/addons/account_edi/models/account_edi_format.py +++ b/addons/account_edi/models/account_edi_format.py @@ -4,6 +4,7 @@ from odoo import models, fields, api from odoo.exceptions import UserError from odoo.tools.pdf import OdooPdfFileReader, OdooPdfFileWriter +from odoo.osv import expression from lxml import etree import base64 @@ -97,7 +98,7 @@ class AccountEdiFormat(models.Model): # TO OVERRIDE return False - def _support_batching(self): + def _support_batching(self, move=None, state=None, company=None): """ Indicate if we can send multiple documents in the same time to the web services. If True, the _post_%s_edi methods will get multiple documents in the same time. Otherwise, these methods will be called with only one record at a time. @@ -107,6 +108,26 @@ class AccountEdiFormat(models.Model): # TO OVERRIDE return False + def _get_batch_key(self, move, state): + """ Returns a tuple that will be used as key to partitionnate the invoices/payments when creating batches + with multiple invoices/payments. + The type of move (invoice or payment), its company_id, its edi state and the edi_format are used by default, if + no further partition is needed for this format, this method should return (). + + :returns: The key to be used when partitionning the batches. + """ + move.ensure_one() + return () + + def _check_move_configuration(self, move): + """ Checks the move and relevant records for potential error (missing data, etc). + + :param invoice: The move to check. + :returns: A list of error messages. + """ + # TO OVERRIDE + return [] + def _post_invoice_edi(self, invoices, test_mode=False): """ Create the file content representing the invoice (and calls web services if necessary). @@ -115,6 +136,7 @@ class AccountEdiFormat(models.Model): :returns: A dictionary with the invoice as key and as value, another dictionary: * attachment: The attachment representing the invoice in this edi_format if the edi was successfully posted. * error: An error if the edi was not successfully posted. + * blocking_level: (optional, requires account_edi_extended) How bad is the error (how should the edi flow be blocked ?) """ # TO OVERRIDE self.ensure_one() @@ -128,6 +150,7 @@ class AccountEdiFormat(models.Model): :returns: A dictionary with the invoice as key and as value, another dictionary: * success: True if the invoice was successfully cancelled. * error: An error if the edi was not successfully cancelled. + * blocking_level: (optional, requires account_edi_extended) How bad is the error (how should the edi flow be blocked ?) """ # TO OVERRIDE self.ensure_one() @@ -141,6 +164,7 @@ class AccountEdiFormat(models.Model): :returns: A dictionary with the payment as key and as value, another dictionary: * attachment: The attachment representing the payment in this edi_format if the edi was successfully posted. * error: An error if the edi was not successfully posted. + * blocking_level: (optional, requires account_edi_extended) How bad is the error (how should the edi flow be blocked ?) """ # TO OVERRIDE self.ensure_one() @@ -154,6 +178,7 @@ class AccountEdiFormat(models.Model): :returns: A dictionary with the payment as key and as value, another dictionary: * success: True if the payment was successfully cancelled. * error: An error if the edi was not successfully cancelled. + * blocking_level: (optional, requires account_edi_extended) How bad is the error (how should the edi flow be blocked ?) """ # TO OVERRIDE self.ensure_one() @@ -223,7 +248,7 @@ class AccountEdiFormat(models.Model): """ attachments = [] for edi_format in self: - attachment = invoice.edi_document_ids.filtered(lambda d: d.edi_format_id == edi_format).attachment_id + attachment = invoice._get_edi_attachment(edi_format) if attachment and edi_format._is_embedding_to_invoice_pdf_needed(): datas = base64.b64decode(attachment.with_context(bin_size=False).datas) attachments.append({'name': attachment.name, 'datas': datas}) @@ -382,3 +407,84 @@ class AccountEdiFormat(models.Model): res.write({'extract_state': 'done'}) return res return self.env['account.move'] + + #################################################### + # Import helpers + #################################################### + + def _find_value(self, xpath, xml_element, namespaces=None): + element = xml_element.xpath(xpath, namespaces=namespaces) + return element[0].text if element else None + + def _retrieve_partner(self, name=None, phone=None, mail=None, vat=None): + '''Search all partners and find one that matches one of the parameters. + + :param name: The name of the partner. + :param phone: The phone or mobile of the partner. + :param mail: The mail of the partner. + :param vat: The vat number of the partner. + :returns: A partner or an empty recordset if not found. + ''' + domains = [] + for value, domain in ( + (name, [('name', 'ilike', name)]), + (phone, expression.OR([[('phone', '=', phone)], [('mobile', '=', phone)]])), + (mail, [('email', '=', mail)]), + (vat, [('vat', 'like', vat)]), + ): + if value is not None: + domains.append(domain) + + domain = expression.OR(domains) + return self.env['res.partner'].search(domain, limit=1) + + def _retrieve_product(self, name=None, default_code=None, barcode=None): + '''Search all products and find one that matches one of the parameters. + + :param name: The name of the product. + :param default_code: The default_code of the product. + :param barcode: The barcode of the product. + :returns: A product or an empty recordset if not found. + ''' + domains = [] + for value, domain in ( + (name, ('name', 'ilike', name)), + (default_code, ('default_code', '=', default_code)), + (barcode, ('barcode', '=', barcode)), + ): + if value is not None: + domains.append([domain]) + + domain = expression.OR(domains) + return self.env['product.product'].search(domain, limit=1) + + def _retrieve_tax(self, amount, type_tax_use): + '''Search all taxes and find one that matches all of the parameters. + + :param amount: The amount of the tax. + :param type_tax_use: The type of the tax. + :returns: A tax or an empty recordset if not found. + ''' + domains = [ + [('amount', '=', float(amount))], + [('type_tax_use', '=', type_tax_use)] + ] + + return self.env['account.tax'].search(expression.AND(domains), order='sequence ASC', limit=1) + + def _retrieve_currency(self, code): + '''Search all currencies and find one that matches the code. + + :param code: The code of the currency. + :returns: A currency or an empty recordset if not found. + ''' + return self.env['res.currency'].search([('name', '=', code.upper())], limit=1) + + #################################################### + # Other helpers + #################################################### + + @api.model + def _format_error_message(self, error_title, errors): + bullet_list_msg = ''.join('<li>%s</li>' % msg for msg in errors) + return '%s<ul>%s</ul>' % (error_title, bullet_list_msg) diff --git a/addons/account_edi/models/account_move.py b/addons/account_edi/models/account_move.py index 46d8e1d67b7..62855e5e49f 100644 --- a/addons/account_edi/models/account_move.py +++ b/addons/account_edi/models/account_move.py @@ -111,6 +111,7 @@ class AccountMove(models.Model): existing_edi_document.write({ 'state': 'to_send', 'error': False, + 'blocking_level': False, }) else: edi_document_vals_list.append({ @@ -122,6 +123,7 @@ class AccountMove(models.Model): existing_edi_document.write({ 'state': False, 'error': False, + 'blocking_level': False, }) self.env['account.edi.document'].create(edi_document_vals_list) @@ -138,6 +140,10 @@ class AccountMove(models.Model): is_edi_needed = move.is_invoice(include_receipts=False) and edi_format._is_required_for_invoice(move) if is_edi_needed: + errors = edi_format._check_move_configuration(move) + if errors: + raise UserError(_("Invalid invoice configuration:\n\n%s") % '\n'.join(errors)) + existing_edi_document = move.edi_document_ids.filtered(lambda x: x.edi_format_id == edi_format) if existing_edi_document: existing_edi_document.write({ @@ -160,8 +166,8 @@ class AccountMove(models.Model): # Set the electronic document to be canceled and cancel immediately for synchronous formats. res = super().button_cancel() - self.edi_document_ids.filtered(lambda doc: doc.attachment_id).write({'state': 'to_cancel', 'error': False}) - self.edi_document_ids.filtered(lambda doc: not doc.attachment_id).write({'state': 'cancelled', 'error': False}) + self.edi_document_ids.filtered(lambda doc: doc.attachment_id).write({'state': 'to_cancel', 'error': False, 'blocking_level': False}) + self.edi_document_ids.filtered(lambda doc: not doc.attachment_id).write({'state': 'cancelled', 'error': False, 'blocking_level': False}) self.edi_document_ids._process_documents_no_web_services() return res @@ -177,7 +183,7 @@ class AccountMove(models.Model): res = super().button_draft() - self.edi_document_ids.write({'state': False, 'error': False}) + self.edi_document_ids.write({'state': False, 'error': False, 'blocking_level': False}) return res @@ -198,7 +204,13 @@ class AccountMove(models.Model): if is_move_marked: move.message_post(body=_("A cancellation of the EDI has been requested.")) - to_cancel_documents.write({'state': 'to_cancel', 'error': False}) + to_cancel_documents.write({'state': 'to_cancel', 'error': False, 'blocking_level': False}) + + def _get_edi_document(self, edi_format): + return self.edi_document_ids.filtered(lambda d: d.edi_format_id == edi_format) + + def _get_edi_attachment(self, edi_format): + return self._get_edi_document(edi_format).attachment_id #################################################### # Import Electronic Document @@ -242,8 +254,12 @@ class AccountMove(models.Model): #################################################### def action_process_edi_web_services(self): - self.edi_document_ids.filtered(lambda d: d.state in ('to_send', 'to_cancel'))._process_documents_web_services() - + docs = self.edi_document_ids.filtered(lambda d: d.state in ('to_send', 'to_cancel')) + if 'blocking_level' in self.env['account.edi.document']._fields: + docs = docs.filtered(lambda d: d.blocking_level != 'error') + else: + docs = docs.filtered(lambda d: not d.error) + docs._process_documents_web_services(with_commit=False) class AccountMoveLine(models.Model): _inherit = 'account.move.line' |
