summaryrefslogtreecommitdiff
path: root/addons/base_import/static/src
diff options
context:
space:
mode:
Diffstat (limited to 'addons/base_import/static/src')
-rw-r--r--addons/base_import/static/src/js/import_action.js196
-rw-r--r--addons/base_import/static/src/scss/base_import.scss65
-rw-r--r--addons/base_import/static/src/xml/base_import.xml72
3 files changed, 250 insertions, 83 deletions
diff --git a/addons/base_import/static/src/js/import_action.js b/addons/base_import/static/src/js/import_action.js
index c75d4dbda7f..57974f33a0a 100644
--- a/addons/base_import/static/src/js/import_action.js
+++ b/addons/base_import/static/src/js/import_action.js
@@ -6,6 +6,8 @@ var config = require('web.config');
var core = require('web.core');
var session = require('web.session');
var time = require('web.time');
+var AbstractWebClient = require('web.AbstractWebClient');
+var Loading = require('web.Loading');
var QWeb = core.qweb;
var _t = core._t;
@@ -194,7 +196,7 @@ var DataImport = AbstractAction.extend({
this.$buttons = $(QWeb.render("ImportView.buttons", this));
this.$buttons.filter('.o_import_validate').on('click', this.validate.bind(this));
this.$buttons.filter('.o_import_import').on('click', this.import.bind(this));
- this.$buttons.filter('.o_import_file_reload').on('click', this.loaded_file.bind(this));
+ this.$buttons.filter('.o_import_file_reload').on('click', this.loaded_file.bind(this, null));
this.$buttons.filter('.oe_import_file').on('click', function () {
self.$('.o_content .oe_import_file').click();
});
@@ -291,6 +293,9 @@ var DataImport = AbstractAction.extend({
advanced: this.$('input.oe_import_advanced_mode').prop('checked'),
keep_matches: this.do_not_change_match,
name_create_enabled_fields: {},
+ // start at row 1 = skip 0 lines
+ skip: Number(this.$('#oe_import_row_start').val()) - 1 || 0,
+ limit: Number(this.$('#oe_import_batch_limit').val()) || null,
};
_(this.opts).each(function (opt) {
options[opt.name] =
@@ -319,7 +324,12 @@ var DataImport = AbstractAction.extend({
},
//- File & settings change section
- onfile_loaded: function () {
+ onfile_loaded: function (event, from, to, arg) {
+ // arg is null if reload -> don't reset partial import
+ if (arg != null ) {
+ this.toggle_partial(null);
+ }
+
this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('d-none');
if (!this.$('input.oe_import_file').val()) { return this['settings_changed'](); }
this.$('.oe_import_date_format').select2('val', '');
@@ -378,12 +388,16 @@ var DataImport = AbstractAction.extend({
this.$('input.oe_import_advanced_mode').prop('checked', result.advanced_mode);
this.$('.oe_import_grid').html(QWeb.render('ImportView.preview', result));
+ this.$('.o_import_batch_alert').toggleClass('d-none', !result.batch);
+
+ var messages = [];
if (result.headers.length === 1) {
+ messages.push({type: 'warning', message: _t("A single column was found in the file, this often means the file separator is incorrect")});
+ }
+
+ if (!_.isEmpty(messages)) {
this.$('.oe_import_options').show();
- this.onresults(null, null, null, {'messages': [{
- type: 'warning',
- message: _t("A single column was found in the file, this often means the file separator is incorrect")
- }]});
+ this.onresults(null, null, null, {'messages': messages});
}
// merge option values back in case they were updated/guessed
@@ -392,10 +406,8 @@ var DataImport = AbstractAction.extend({
});
this.$('.oe_import_date_format').select2('val', time.strftime_to_moment_format(result.options.date_format));
this.$('.oe_import_datetime_format').val(time.strftime_to_moment_format(result.options.datetime_format));
- if (result.debug === false){
- this.$('.oe_import_tracking').hide();
- this.$('.oe_import_deferparentstore').hide();
- }
+ // hide all "true debug" options when not in debug mode
+ this.$('.oe_import_debug_option').toggleClass('d-none', !result.debug);
var $fields = this.$('.oe_import_fields input');
this.render_fields_matches(result, $fields);
@@ -561,19 +573,28 @@ var DataImport = AbstractAction.extend({
}).get();
var tracking_disable = 'tracking_disable' in kwargs ? kwargs.tracking_disable : !this.$('#oe_import_tracking').prop('checked')
- var defer_parent_store = 'defer_parent_store' in kwargs ? kwargs.defer_parent_store : !!this.$('#oe_import_deferparentstore').prop('checked')
delete kwargs.tracking_disable;
- delete kwargs.defer_parent_store;
kwargs.context = _.extend(
{}, this.parent_context,
- {tracking_disable: tracking_disable, defer_parent_store_computation: defer_parent_store}
+ {tracking_disable: tracking_disable}
);
- return this._rpc({
- model: 'base_import.import',
- method: 'do',
- args: [this.id, fields, columns, this.import_options()],
- kwargs : kwargs,
- }).then(null, function (reason) {
+ var self = this;
+ this.trigger_up('with_client', {callback: function () {
+ this.loading.ignore_events = true;
+ }});
+ $.blockUI({message: QWeb.render('Throbber')});
+ $(document.body).addClass('o_ui_blocked');
+ var opts = this.import_options();
+
+ var $el = $('.oe_throbber_message');
+ var msg = kwargs.dryrun ? _t("%d records tested...")
+ : _t("%d records successfully imported...");
+ opts.callback = function (count) {
+ $el.text(_.str.sprintf(msg, count));
+ };
+
+ return this._batchedImport(opts, [this.id, fields, columns], kwargs, {done: 0, prev: 0})
+ .then(null, function (reason) {
var error = reason.message;
var event = reason.event;
// In case of unexpected exception, convert
@@ -582,8 +603,9 @@ var DataImport = AbstractAction.extend({
if (event) { event.preventDefault(); }
var msg;
- if (error.data.type === 'xhrerror') {
- var xhr = error.data.objects[0];
+ var errordata = error.data || {};
+ if (errordata.type === 'xhrerror') {
+ var xhr = errordata.objects[0];
switch (xhr.status) {
case 504: // gateway timeout
msg = _t("Import timed out. Please retry. If you still encounter this issue, the file may be too big for the system's configuration, try to split it (import less records per file).");
@@ -592,7 +614,7 @@ var DataImport = AbstractAction.extend({
msg = _t("An unknown issue occurred during import (possibly lost connection, data limit exceeded or memory limits exceeded). Please retry in case the issue is transient. If the issue still occurs, try to split the file rather than import it at once.");
}
} else {
- msg = (error.data.arguments && error.data.arguments[1] || error.data.arguments[0])
+ msg = errordata.arguments && (errordata.arguments[1] || errordata.arguments[0])
|| error.message;
}
@@ -601,7 +623,76 @@ var DataImport = AbstractAction.extend({
record: false,
message: msg,
}]});
- }) ;
+ }).finally(function () {
+ $(document.body).removeClass('o_ui_blocked');
+ $.unblockUI();
+ self.trigger_up('with_client', {callback: function () {
+ delete this.loading.ignore_events;
+ }});
+ });
+ }, /**
+ *
+ * @param opts import options
+ * @param args positional arguments to pass along (augmented with the options)
+ * @param kwargs keyword arguments to pass along (directly)
+ * @param {Object} rec recursion information record
+ * @param {Number} rec.done how many records have been loaded so far
+ * @param {Number} rec.prev nextrow of the previous call so we can know
+ * how many rows the call we're here performing
+ * will have consumed, and thus by how much we
+ * need to offset the messages of the *next* call
+ * @returns {Promise<{name, ids, messages}>}
+ * @private
+ */
+ _batchedImport: function (opts, args, kwargs, rec) {
+ opts.callback && opts.callback(rec.done || 0);
+ var self = this;
+ return this._rpc({
+ model: 'base_import.import',
+ method: 'do',
+ args: args.concat([opts]),
+ kwargs: kwargs
+ }).then(function (results) {
+ _.each(results.messages, offset_by(opts.skip));
+ if (!kwargs.dryrun && !results.ids) {
+ // update skip to failed batch
+ self.$('#oe_import_row_start').val(opts.skip + 1);
+ if (opts.skip) {
+ // there's been an error during a "proper" import, stop & warn
+ // about partial import maybe
+ results.messages.push({
+ type: 'info',
+ priority: true,
+ message: _.str.sprintf(_t("This file has been successfully imported up to line %d."), opts.skip)
+ });
+ }
+ return results;
+ }
+ if (!results.nextrow) {
+ // we're done
+ return results;
+ }
+
+ // do the next batch
+ return self._batchedImport(
+ // avoid modifying opts in-place
+ _.defaults({skip: results.nextrow}, opts),
+ args, kwargs, {
+ done: rec.done + (results.ids || []).length,
+ prev: results.nextrow
+ }
+ ).then(function (r2) {
+ return {
+ name: _.zip(results.name, r2.name).map(function (names) {
+ return names[0] || names[1];
+ }),
+ ids: (results.ids || []).concat(r2.ids || []),
+ messages: results.messages.concat(r2.messages),
+ skip: r2.skip || results.nextrow,
+ nextrow: r2.nextrow
+ }
+ });
+ });
},
onvalidate: function () {
var prom = this.call_import({ dryrun: true, tracking_disable: true });
@@ -641,19 +732,31 @@ var DataImport = AbstractAction.extend({
type: 'info',
message: _t("Everything seems valid.")
});
+ } else if (event === 'import_failed' && results.ids) {
+ // both ids in a failed import -> partial import
+ this.toggle_partial(results);
}
+
// row indexes come back 0-indexed, spreadsheets
// display 1-indexed.
var offset = 1;
// offset more if header
if (this.import_options().headers) { offset += 1; }
- var messagesSorted = _.sortBy(_(message).groupBy('message'), function (messageGroupped) {
+ var messagesSorted = _.sortBy(_(message).groupBy('message'), function (group) {
+ if (group[0].priority){
+ return -2;
+ }
+
+ // sort by gravity, then, order of field in list
var order = 0;
- if (messageGroupped[0].type === 'warning') {
- order = fields.length + 1;
+ switch (group[0].type) {
+ case 'error': order = 0; break;
+ case 'warning': order = fields.length + 1; break;
+ case 'info': order = 2 * (fields.length + 1); break;
+ default: order = 3 * (fields.length + 1); break;
}
- return order + _.indexOf(fields, messageGroupped[0].field);
+ return order + _.indexOf(fields, group[0].field);
});
this.$form.addClass('oe_import_error');
@@ -722,6 +825,24 @@ var DataImport = AbstractAction.extend({
},
}));
},
+ toggle_partial: function (result) {
+ var $form = this.$('.oe_import');
+ var $partial_warning = this.$('.o_import_partial_alert');
+ var $partial_count = this.$('.o_import_partial_count');
+ if (result == null) {
+ $partial_warning.addClass('d-none');
+ $form.add(this.$buttons).removeClass('o_import_partial_mode');
+ var $skip = this.$('#oe_import_row_start');
+ $skip.val($skip.attr('value'));
+ $partial_count.text('');
+ return;
+ }
+
+ this.$('.o_import_batch_alert').addClass('d-none');
+ $partial_warning.removeClass('d-none');
+ $form.add(this.$buttons).addClass('o_import_partial_mode');
+ $partial_count.text((result.skip || 0) + 1);
+ }
});
core.action_registry.add('import', DataImport);
@@ -746,6 +867,27 @@ StateMachine.create({
],
});
+Loading.include({
+ on_rpc_event: function () {
+ if (this.ignore_events) {
+ return
+ }
+ this._super.apply(this, arguments);
+ }
+});
+AbstractWebClient.prototype.custom_events['with_client'] = function (ev) {
+ ev.data.callback.call(this);
+};
+
+function offset_by(by) {
+ return function offset_message(msg) {
+ if (msg.rows) {
+ msg.rows.from += by;
+ msg.rows.to += by;
+ }
+ }
+}
+
return {
DataImport: DataImport,
};
diff --git a/addons/base_import/static/src/scss/base_import.scss b/addons/base_import/static/src/scss/base_import.scss
index 5499c5c4553..bed08873898 100644
--- a/addons/base_import/static/src/scss/base_import.scss
+++ b/addons/base_import/static/src/scss/base_import.scss
@@ -14,17 +14,13 @@
text-align: justify
}
h2 {
- margin-top: 0;
+ margin-top: 0.5em;
font-size: large; // override h2 font-size which is too large
}
.oe_padding {
padding: 13px 0;
}
- .oe_import_advanced_mode {
- margin-left: 20px;
- }
-
.oe_import_box {
padding: 8px;
background: #F0EEEE;
@@ -68,6 +64,19 @@
.oe_import_with_file label {
font-weight: normal;
}
+ .oe_import_debug_options {
+ max-width: 800px;
+ columns: 1;
+ @include media-breakpoint-up(md) {
+ columns: 2;
+ }
+ // try to keep the batch fields together, doesn't work on firefox &
+ // not sure how to do that (except by adding intermediate dom
+ // elements)
+ .oe_import_batch_limit {
+ break-before: column;
+ }
+ }
&.oe_import_preview .oe_import_grid {
display: table;
@@ -81,27 +90,27 @@
}
/* ------------- ERRORS AND WARNINGS REPORT ------------ */
- .oe_import_error_report {
- > ul {
- padding: 0;
+ .oe_import_error_report > ul {
+ padding: 0;
+ }
+ .oe_import_report {
+ list-style: none;
+ }
+ .alert {
+ padding: 0.50rem 1.25rem;
+ margin: 0.25rem 0;
+
+ a {
+ @extend .alert-link;
+ &:hover {opacity: 0.8;}
}
- .oe_import_report {
- padding: 4px;
- margin: 2px 0;
- list-style: none;
- border-radius: $border-radius;
- &.bg-error {
- @extend .bg-danger;
- }
- &.text-error {
- @extend .text-danger;
- }
+
+ // alias -error to -danger
+ &.alert-error {
+ @extend .alert-danger;
}
- .oe_import_report_count, .oe_import_see_all {
- color: white;
- &:hover {
- color: lightgrey;
- }
+ &.text-error {
+ @extend .text-danger;
}
}
@@ -123,9 +132,13 @@
.select2-default{
color: #F00 !important;
}
-
}
-
+/* ------------- PARTIAL MODE buttons ------------ */
+// hide import in partial mode, resume otherwise
+.o_import_import_full.o_import_partial_mode,
+.o_import_import_partial:not(.o_import_partial_mode) {
+ display: none;
+}
/* Field dropdown */
.oe_import_selector {
diff --git a/addons/base_import/static/src/xml/base_import.xml b/addons/base_import/static/src/xml/base_import.xml
index 6d69d6995aa..aa0e956f9fd 100644
--- a/addons/base_import/static/src/xml/base_import.xml
+++ b/addons/base_import/static/src/xml/base_import.xml
@@ -4,7 +4,7 @@
<form action="" method="post" enctype="multipart/form-data" class="oe_import">
<input type="hidden" name="csrf_token" t-att-value="csrf_token"/>
<input type="hidden" name="import_id"/>
- <div class="oe_import_box col-lg-12 d-none">
+ <div class="oe_import_box d-none">
<input accept=".csv, .xls, .xlsx, .xlsm, .ods" t-attf-id="file_#{_id}"
name="file" class="oe_import_file" type="file" style="display:none;"/>
<div class="oe_import_with_file row">
@@ -36,34 +36,45 @@
</div>
</div>
</div>
+ <div class="o_import_batch_alert alert alert-warning d-none">
+ Due to its large size, the file will be imported by batches.
+ </div>
+ <div class="o_import_partial_alert alert alert-warning d-none">
+ Click 'Resume' to proceed with the import, resuming at line
+ <span class="o_import_partial_count">0</span>.<br/>
+ You can test or reload your file before resuming the import.
+ </div>
- <div class="oe_import_with_file oe_padding col-md-12 d-none">
+ <div class="oe_import_with_file d-none">
<h2>Map your columns to import</h2>
- <div title="If the model uses openchatter, history tracking
- will set up subscriptions and send notifications
- during the import, but lead to a slower import."
- class="oe_import_tracking">
- <input type="checkbox" id="oe_import_tracking"/>
- <label for="oe_import_tracking">
- Track history during import
- </label>
- </div>
- <div title="If the model uses parent/child relations, computing the
- parent / child relation occurs on every line, and lead to a slower import.
- Defering it can speed up import."
- class="oe_import_deferparentstore">
- <input type="checkbox" id="oe_import_deferparentstore" checked="checked"/>
- <label for="oe_import_deferparentstore">
- Defer parent/child computation
- </label>
+ <div class="oe_import_debug_options">
+ <div title="If the model uses openchatter, history tracking will set up subscriptions and send notifications during the import, but lead to a slower import." class="oe_import_debug_option">
+ <input type="checkbox" id="oe_import_tracking"/>
+ <label for="oe_import_tracking">
+ Track history during import
+ </label>
+ </div>
+ <div>
+ <input type="checkbox" class="oe_import_has_header"
+ id="oe_import_has_header" checked="checked"/>
+ <label for="oe_import_has_header">The first row
+ contains the label of the column</label>
+ </div>
+ <div class="js_import_options oe_import_debug_option oe_import_batch_limit">
+ <label for="oe_import_batch_limit">Batch limit</label>
+ <input id="oe_import_batch_limit" value="2000"/>
+ </div>
+ <div class="js_import_options oe_import_debug_option" title="Warning: ignores the labels line, empty lines and
+ lines composed only of empty cells">
+ <label for="oe_import_row_start">Start at line</label>
+ <input id="oe_import_row_start" value="1"/>
+ </div>
+ <div class="oe_import_debug_option">
+ <input type="checkbox" class="oe_import_advanced_mode" checked="checked"
+ id="oe_import_advanced_mode"/>
+ <label for="oe_import_advanced_mode">Show fields of relation fields (advanced)</label>
+ </div>
</div>
- <input type="checkbox" class="oe_import_has_header"
- id="oe_import_has_header" checked="checked"/>
- <label for="oe_import_has_header">The first row
- contains the label of the column</label>
- <input type="checkbox" class="oe_import_advanced_mode" checked="checked"
- id="oe_import_advanced_mode"/>
- <label for="oe_import_advanced_mode">Show fields of relation fields (advanced)</label>
<p class="oe_import_noheaders text-muted">If the file contains
the column names, Odoo can try auto-detecting the
field corresponding to the column. This makes imports
@@ -95,8 +106,9 @@
</t>
<t t-name="ImportView.buttons">
- <button type="button" class="btn btn-primary o_import_import d-none">Import</button>
- <button type="button" class="btn btn-secondary o_import_validate d-none">Test Import</button>
+ <button type="button" class="btn btn-primary o_import_import o_import_import_full d-none">Import</button>
+ <button type="button" class="btn btn-primary o_import_import o_import_import_partial d-none">Resume</button>
+ <button type="button" class="btn btn-secondary o_import_validate d-none">Test</button>
<button type="button" class="btn btn-secondary o_import_file_reload d-none">Reload File</button>
<button type="button" class="btn btn-primary oe_import_file">Load File</button>
<button type="button" class="btn btn-secondary o_import_cancel">Cancel</button>
@@ -132,7 +144,7 @@
</tbody>
</t>
<t t-name="ImportView.preview.error">
- <div class="oe_import_report bg-danger">
+ <div class="oe_import_report alert alert-danger">
<p>Import preview failed due to: <t t-esc="error"/>.</p>
<p>For CSV files, you may need to select the correct separator.</p>
<p t-if="preview">Here is the start of the file we could not import:</p>
@@ -141,7 +153,7 @@
</t>
<ul t-name="ImportView.error">
<li t-foreach="errors" t-as="error"
- t-attf-class="oe_import_report bg-#{error_value[0].type}">
+ t-attf-class="oe_import_report alert alert-#{error_value[0].type}">
<t t-if="error_value.length gt 1">
<t t-call="ImportView.error.multi.header">
<t t-set="error" t-value="error_value[0]"/>
generated by cgit v1.2.3 (git 2.25.1) at 2026-10-23 19:11:08 +0000