diff options
Diffstat (limited to 'setup/package.py')
| -rwxr-xr-x | setup/package.py | 108 |
1 files changed, 68 insertions, 40 deletions
diff --git a/setup/package.py b/setup/package.py index 03d7fe61027..d4b0a2c294b 100755 --- a/setup/package.py +++ b/setup/package.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. from __future__ import print_function +import logging import optparse import os import pexpect @@ -10,6 +11,7 @@ import signal import subprocess import tempfile import time +import traceback try: from xmlrpc import client as xmlrpclib except ImportError: @@ -26,7 +28,7 @@ from tempfile import NamedTemporaryFile # Utils #---------------------------------------------------------- exec(open(join(dirname(__file__), '..', 'odoo', 'release.py'), 'rb').read()) -version = version.split('-')[0] +version = version.split('-')[0].replace('saas~','') docker_version = version.replace('+', '') timestamp = time.strftime("%Y%m%d", time.gmtime()) GPGPASSPHRASE = os.getenv('GPGPASSPHRASE') @@ -58,7 +60,7 @@ def mkdir(d): os.makedirs(d) def system(l, chdir=None): - print (l) + logging.info("System call: {}".format(l)) if chdir: cwd = os.getcwd() os.chdir(chdir) @@ -82,12 +84,12 @@ def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'): dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')] ) if toinstallmodules: - print("Package test: FAILED. Not able to install dependencies of base.") + logging.error("Package test: FAILED. Not able to install dependencies of base.") raise Exception("Installation of package failed") else: - print("Package test: successfuly installed %s modules" % len(modules)) + logging.info("Package test: successfuly installed %s modules" % len(modules)) else: - print("Package test: FAILED. Not able to install base.") + logging.error("Package test: FAILED. Not able to install base.") raise Exception("Installation of package failed") def publish(o, type, extensions): @@ -114,8 +116,9 @@ def publish(o, type, extensions): published = [] for extension in extensions: - release = glob("%s/odoo_*.%s" % (o.build_dir, extension))[0] - published.append(_publish(o, release)) + release = glob("%s/odoo_*.%s" % (o.build_dir, extension)) + if release: + published.append(_publish(o, release[0])) return published class OdooDocker(object): @@ -147,10 +150,10 @@ class OdooDocker(object): try: _rpc_count_modules(port=str(self.port)) except Exception as e: - print('Exception during docker execution: %s:' % str(e)) - print('Error during docker execution: printing the bash output:') + logging.error('Exception during docker execution: %s:' % str(e)) + logging.error('Error during docker execution: printing the bash output:') with open(self.log_file.name) as f: - print('\n'.join(f.readlines())) + print('\n'.join(f.readlines()), file=stderr) raise finally: self.docker.close() @@ -178,16 +181,16 @@ class KVM(object): self.login = login def timeout(self,signum,frame): - print("vm timeout kill",self.pid) + logging.warning("vm timeout kill",self.pid) os.kill(self.pid,15) def start(self): - l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -m 1024 -drive".split(" ") + l="kvm -cpu core2duo -smp 2,sockets=2,cores=1,threads=1 -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -m 1024 -drive".split(" ") #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image) l.append('file=%s,snapshot=on'%self.image) #l.extend(['-vnc','127.0.0.1:1']) l.append('-nographic') - print( " ".join(l)) + logging.info("Starting kvm: {}".format( " ".join(l))) self.pid=os.spawnvp(os.P_NOWAIT, l[0], l) time.sleep(20) signal.alarm(2400) @@ -203,7 +206,7 @@ class KVM(object): l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd] system(l) - def rsync(self,args,options='--delete --exclude .bzrignore'): + def rsync(self,args,options='--delete --exclude .git --exclude .tx --exclude __pycache__'): cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args) system(cmd) @@ -213,17 +216,20 @@ class KVM(object): class KVMWinBuildExe(KVM): def run(self): with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f: - f.write("VERSION=%s\n" % version) + f.write("VERSION=%s\n" % version.replace('~', '_')) with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f: f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', '')) with open(join(self.o.build_dir, 'setup/win32/Makefile.servicename'), 'w') as f: f.write("SERVICENAME=%s\n" % nt_service_name) + remote_build_dir = '/cygdrive/c/odoobuild/server/' + self.ssh("mkdir -p build") - self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login)) - self.ssh("cd build/server/setup/win32;time make allinone;") - self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '') - print("KVMWinBuildExe.run(): done") + logging.info("Syncing Odoo files to virtual machine...") + self.rsync('%s/ %s@127.0.0.1:%s' % (self.o.build_dir, self.login, remote_build_dir)) + self.ssh("cd {}setup/win32;time make allinone;".format(remote_build_dir)) + self.rsync('%s@127.0.0.1:%ssetup/win32/release/ %s/' % (self.login, remote_build_dir, self.o.build_dir), '') + logging.info("KVMWinBuildExe.run(): done") class KVMWinTestExe(KVM): def run(self): @@ -242,18 +248,25 @@ class KVMWinTestExe(KVM): # Stage: building #---------------------------------------------------------- def _prepare_build_dir(o, win32=False): + cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo'] if not win32: cmd += ['--exclude', 'setup/win32'] system(cmd + ['%s/' % o.odoo_dir, o.build_dir]) - try: - for addon_path in glob(join(o.build_dir, 'addons/*')): - if addon_path.split(os.path.sep)[-1] not in ADDONS_NOT_TO_PUBLISH: + for addon_path in glob(join(o.build_dir, 'addons/*')): + if addon_path.split(os.path.sep)[-1] not in ADDONS_NOT_TO_PUBLISH: + try: shutil.move(addon_path, join(o.build_dir, 'odoo/addons')) - except shutil.Error: - # Thrown when the add-on is already in odoo/addons (if _prepare_build_dir - # has already been called once) - pass + except shutil.Error as e: + # Thrown when the add-on is already in odoo/addons (if _prepare_build_dir + # has already been called once) + logging.warning("Warning '{}' while moving addon '{}'".format(e,addon_path)) + if addon_path.startswith(o.build_dir) and os.path.isdir(addon_path): + logging.info("Removing '{}'".format(addon_path)) + try: + shutil.rmtree(addon_path) + except shutil.Error as rm_error: + logging.warning("Cannot remove '{}': {}".format(addon_path, rm_error)) def build_tgz(o): system(['python3', 'setup.py', 'sdist', '--quiet', '--formats=gztar,zip'], o.build_dir) @@ -266,7 +279,7 @@ def build_deb(o): subprocess.call(cmd, cwd=o.build_dir) if not o.no_debsign: deb = pexpect.spawn('dpkg-buildpackage -rfakeroot -k%s' % GPGID, cwd=o.build_dir) - deb.logfile = stdout + deb.logfile = stdout.buffer if GPGPASSPHRASE: deb.expect_exact('Enter passphrase: ', timeout=1200) deb.send(GPGPASSPHRASE + '\r\n') @@ -292,7 +305,9 @@ def build_exe(o): # Stage: testing #---------------------------------------------------------- def _prepare_testing(o): + logging.info('Preparing testing') if not o.no_tarball: + logging.info('Preparing docker container instance for tarball') subprocess.call(["mkdir", "docker_src"], cwd=o.build_dir) subprocess.call(["cp", "package.dfsrc", os.path.join(o.build_dir, "docker_src", "Dockerfile")], cwd=os.path.join(o.odoo_dir, "setup")) @@ -302,6 +317,7 @@ def _prepare_testing(o): subprocess.call(["docker", "build", "-t", "odoo-%s-src-nightly-tests" % docker_version, "."], cwd=os.path.join(o.build_dir, "docker_src")) if not o.no_debian: + logging.info('Preparing docker container instance for debian') subprocess.call(["mkdir", "docker_debian"], cwd=o.build_dir) subprocess.call(["cp", "package.dfdebian", os.path.join(o.build_dir, "docker_debian", "Dockerfile")], cwd=os.path.join(o.odoo_dir, "setup")) @@ -311,6 +327,7 @@ def _prepare_testing(o): subprocess.call(["docker", "build", "-t", "odoo-%s-debian-nightly-tests" % docker_version, "."], cwd=os.path.join(o.build_dir, "docker_debian")) if not o.no_rpm: + logging.info('Preparing docker container instance for RPM') subprocess.call(["mkdir", "docker_fedora"], cwd=o.build_dir) subprocess.call(["cp", "package.dffedora", os.path.join(o.build_dir, "docker_fedora", "Dockerfile")], cwd=os.path.join(o.odoo_dir, "setup")) @@ -318,6 +335,7 @@ def _prepare_testing(o): cwd=os.path.join(o.build_dir, "docker_fedora")) def test_tgz(o): + logging.info('Testing tarball in docker') with docker('odoo-%s-src-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy: wheezy.release = '*.tar.gz' wheezy.system("service postgresql start") @@ -331,6 +349,7 @@ def test_tgz(o): wheezy.system('su odoo -s /bin/bash -c "odoo --addons-path=/usr/local/lib/python2.7/dist-packages/odoo/addons -d mycompany &"') def test_deb(o): + logging.info('Testing deb package in docker') with docker('odoo-%s-debian-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy: wheezy.release = '*.deb' wheezy.system("service postgresql start") @@ -341,6 +360,7 @@ def test_deb(o): wheezy.system('su odoo -s /bin/bash -c "odoo -c /etc/odoo/odoo.conf -d mycompany &"') def test_rpm(o): + logging.info('Testing rpm in docker') with docker('odoo-%s-fedora-nightly-tests' % docker_version, o.build_dir, o.pub) as fedora24: fedora24.release = '*.noarch.rpm' # Start postgresql @@ -353,6 +373,7 @@ def test_rpm(o): fedora24.system('su odoo -s /bin/bash -c "odoo -c /etc/odoo/odoo.conf -d mycompany &"') def test_exe(o): + logging.info('Testng windows installer in KVM') KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start() #--------------------------------------------------------- @@ -379,7 +400,7 @@ def gen_deb_package(o, published_files): ] # Generate files for command in commands: - _gen_file(o, *command, temp_path) + _gen_file(o, command[0], command[-1], temp_path) # Remove temp directory shutil.rmtree(temp_path) @@ -420,6 +441,8 @@ def options(): root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) build_dir = "%s-%s" % (root, timestamp) + log_levels = { "debug" : logging.DEBUG, "info": logging.INFO, "warning": logging.WARN, "error": logging.ERROR, "critical": logging.CRITICAL } + op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR") op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR") op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages") @@ -431,13 +454,16 @@ def options(): op.add_option("", "--no-windows", action="store_true", help="don't build the windows package") # Windows VM - op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/win1035/win10_py35.qcow2', help="%default") - op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/win1035/id_rsa', help="%default") + op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/win1036/win10_winpy36.qcow2', help="%default") + op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/win1036/id_rsa', help="%default") op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)") - op.add_option("", "--vm-winxp-python-version", default='3.5', help="Windows Python version installed in the VM (default: %default)") + op.add_option("", "--vm-winxp-python-version", default='3.6', help="Windows Python version installed in the VM (default: %default)") + op.add_option("", "--no-remove", action="store_true", help="don't remove build dir") + op.add_option("", "--logging", action="store", type="choice", choices=list(log_levels.keys()), default="info", help="Logging level") (o, args) = op.parse_args() + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %I:%M:%S', level=log_levels[o.logging]) # derive other options o.odoo_dir = root o.pkg = join(o.build_dir, 'pkg') @@ -459,16 +485,17 @@ def main(): test_tgz(o) published_files = publish(o, 'tarball', ['tar.gz', 'zip']) except Exception as e: - print("Won't publish the tgz release.\n Exception: %s" % str(e)) + logging.error("Won't publish the tgz release.\n Exception: %s" % str(e)) if not o.no_debian: build_deb(o) try: if not o.no_testing: test_deb(o) - published_files = publish(o, 'debian', ['deb', 'dsc', 'changes', 'tar.gz']) + published_files = publish(o, 'debian', ['deb', 'dsc', 'changes', 'tar.xz']) gen_deb_package(o, published_files) except Exception as e: - print("Won't publish the deb release.\n Exception: %s" % str(e)) + logging.error("Won't publish the deb release.\n Exception: %s" % str(e)) + traceback.print_exc() if not o.no_rpm: build_rpm(o) try: @@ -477,7 +504,7 @@ def main(): published_files = publish(o, 'redhat', ['noarch.rpm']) gen_rpm_repo(o, published_files[0]) except Exception as e: - print("Won't publish the rpm release.\n Exception: %s" % str(e)) + logging.error("Won't publish the rpm release.\n Exception: %s" % str(e)) if not o.no_windows: _prepare_build_dir(o, win32=True) build_exe(o) @@ -486,19 +513,20 @@ def main(): test_exe(o) published_files = publish(o, 'windows', ['exe']) except Exception as e: - print("Won't publish the exe release.\n Exception: %s" % str(e)) + logging.error("Won't publish the exe release.\n Exception: %s" % str(e)) except Exception as e: - print('Something bad happened ! : {}'.format(e), file=stderr) + logging.error('Something bad happened ! : {}'.format(e)) + traceback.print_exc() finally: if o.no_remove: - print('Build dir "{}" not removed'.format(o.build_dir)) + logging.info('Build dir "{}" not removed'.format(o.build_dir)) else: shutil.rmtree(o.build_dir) - print('Build dir %s removed' % o.build_dir) + logging.info('Build dir %s removed' % o.build_dir) if not o.no_testing: system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null") - print('Remaining dockers removed') + logging.info('Remaining dockers removed') if __name__ == '__main__': |