From cd61ef825a093bedcfb6e058b5fc3e896131974d Mon Sep 17 00:00:00 2001 From: Yannick Vaucher Date: Wed, 25 Jul 2018 10:48:13 +0200 Subject: [PATCH] Add capability to use intermediate state DB dumps on unit test and migration. Modifiy `bin/runtests` to use database dumps on demand. Add a script `bin/runmigration` to wrap migration for test purpose of the migration steps. Also add a script `bin/list_dependencies.py` which returns a set of odoo addons dependencies read from the manifest files of local-src addons. The databases will be saved in .cachedb directory. You will want to have a volume on this to save them. This comes handy to speed up travis tests. Can be activated using env vars. --- Makefile | 24 ++++++++++- README.md | 87 +++++++++++++++++++++++++++++++++++++- bin/list_dependencies.py | 36 ++++++++++++++++ bin/runmigration | 82 +++++++++++++++++++++++++++++++++++ bin/runtests | 59 ++++++++++++++++++++++++-- example/docker-compose.yml | 12 ++++++ 6 files changed, 292 insertions(+), 8 deletions(-) create mode 100755 bin/list_dependencies.py create mode 100755 bin/runmigration diff --git a/Makefile b/Makefile index 08488e7b..83fc9fc2 100644 --- a/Makefile +++ b/Makefile @@ -68,8 +68,28 @@ test: echo '>>> Run test for base image' sed 's|FROM .*|FROM $(IMAGE_LATEST)|' -i $(TMP)/odoo/Dockerfile cat $(TMP)/odoo/Dockerfile - cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) odoo odoo --stop-after-init - cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) odoo runtests + mkdir $(TMP)/.cachedb + # migration: standard + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -e LOAD_DB_CACHE="false" odoo odoo --stop-after-init + # runmigration: create the dump + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -v $(TMP)/.cachedb:/opt/.cachedb -e CREATE_DB_CACHE="true" odoo runmigration + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) odoo dropdb odoodb + # runmigration: use dump + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -v $(TMP)/.cachedb:/opt/.cachedb -e LOAD_DB_CACHE="true" odoo runmigration + cd $(TMP) && docker-compose -f docker-compose.yml down + echo " - version: 9.0.1" >> $(TMP)/odoo/migration.yml + echo " operations:">> $(TMP)/odoo/migration.yml + echo " post:" >> $(TMP)/odoo/migration.yml + echo " - anthem songs.install.demo::create_partners" >> $(TMP)/odoo/migration.yml + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) odoo dropdb odoodb + # runmigration: ceil + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -v $(TMP)/.cachedb:/opt/.cachedb -e LOAD_DB_CACHE="true" -e MIG_LOAD_VERSION_CEIL="9.0.1" odoo runmigration + # runtests: standard + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -e LOAD_DB_CACHE="false" -e CREATE_DB_CACHE="false" odoo runtests + ## runtests: create the dump + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -v $(TMP)/.cachedb:/opt/.cachedb -e CREATE_DB_CACHE="true" -e SUBS_MD5=testcache odoo runtests + ## runtests: use dump + cd $(TMP) && docker-compose -f docker-compose.yml run --rm -e LOCAL_USER_ID=$(shell id -u) -v $(TMP)/.cachedb:/opt/.cachedb -e LOAD_DB_CACHE="true" -e SUBS_MD5=testcache odoo runtests cd $(TMP) && docker-compose -f docker-compose.yml down echo '>>> Run test for onbuild image' cp $(TMP)/odoo/Dockerfile-onbuild $(TMP)/odoo/Dockerfile diff --git a/README.md b/README.md index 796a9125..7469e55d 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,49 @@ Instead, you can set the ID of the host's system in `LOCAL_USER_ID`, which will then be shared by the container. All the files created in host volumes will then share the same user. +### CREATE_DB_CACHE + +Used in `bin/runtests` and `bin/runmigration`. + +If set to "true", will create a dump in `.cachedb` of an intermediate state of the tests or migration. +By default not set, thus unactivated. + +### LOAD_DB_CACHE + +Used in `bin/runtests` and `bin/runmigration`. + +If set to "false", will skip trying to reload a cached dump from `.cachedb`. + +### SUBS_MD5 + +This value is used in `bin/runtests` to determine the name of the intermediate state to +load or create. + +Value to tag a database dump of `bin/runtests`, for instance it can be based on +submodules in .travis.yml of your git repositories in odoo/src and in odoo/external-src: + +``` +export SUBS_MD5=$(git submodule status | md5sum | cut -d ' ' -f1) +``` + +You want this value to be unique and identify your dependencies, thus if a +dependency change you need to generate a new one. + +### MIG_LOAD_VERSION_CEIL + +Used in `bin/runmigration` to specify from which dump we want to play the migration. +In case you have a dump per version, you can play the migration against the version of your choice. +If the version specified does not exists, it will search for a lower occurence. + +It will load a dump lower than "odoo_sample_$MIG_LOAD_VERSION_CEIL.dmp" +This is useful if you bumped odoo/VERSION as it won't match existing +dumps. + +For instance you have made a dump 10.1.0, you are now on the version +10.2.0, if you pass your current version it will search for a dump +lower than 10.2.0 and restore the 10.1.0. Then play the remaining +steps on top of it. + ### Odoo Configuration Options The main configuration options of Odoo can be configured through environment variables. The name of the environment variables are the same of the options but uppercased (eg. `workers` becomes `WORKERS`). @@ -205,16 +248,25 @@ By the way, you can add other `ENV` variables in your project's `Dockerfile` if ## Running tests +### runtests + Inside the container, a script `runtests` is used for running the tests on Travis. -It will create a new database, find the local addons, install them and run their tests. + +Unless `LOAD_DB_CACHE is set to `false` it will search for a dump of dependencies and restore it. +Otherwise, will create a new database, find the `odoo/external-src` and `odoo/src` dependencies of the local addons and +if `CREATE_DB_CACHE` is activated creates a dump of that state. + +Then it will install local addons, run their tests and show the code coverage. ``` -docker-compose run --rm odoo runtests +docker-compose run --rm [-e CREATE_DB_CACHE=true] [-e LOAD_DB_CACHE=false] [-e SUBS_MD5=] odoo runtests ``` This is not the day-to-day tool for running the tests as a developer. +### pytests + pytest is included and can be invoked when starting a container. It needs an existing database to run the tests: ``` @@ -238,6 +290,37 @@ docker-compose run --rm odoo dropdb testdb Pytest uses a plugin (https://github.com/camptocamp/pytest-odoo) that corrects the Odoo namespaces (`openerp.addons`/`odoo.addons`) when running the tests. +### runmigration + +Inside the container, a script `runmigration` is used to run the migration steps on Travis. + +Then when launched, it will search for database dump of the content of `odoo/VERSION` file. +Or if you provided `MIG_LOAD_VERSION_CEIL` which will allow you to search for an other version. +If no dump is available (or `LOAD_DB_CACHE` is set to `false`), migration will start from scratch. + +The migration steps are then run. + +If migration succeed a dump is created if `CREATE_DB_CACHE` is set to `true`. + +``` +docker-compose run --rm [-e CREATE_DB_CACHE=true] [-e LOAD_DB_CACHE=false] [-e MIG_LOAD_VERSION_CEIL=x.y.z] odoo runmigration +``` + +This tools really speed up the process of testing migration steps as you can be executing only a single step instead of redoing all. + +### cached dumps (runtests / runmigration) + +To use database dumps you will need a volume on `/opt/.cachedb` to have persistant dumps. + +On travis you will also want to activate the cache, if your volume definition is `- "$HOME/.cachedb:/opt/.cachedb"` +add this in `.travis.yml`: + +``` +cache: + directories: + - $HOME/.cachedb +``` + ## Start entrypoint Any script in any language placed in `/opt/odoo/start-entrypoint.d` will be diff --git a/bin/list_dependencies.py b/bin/list_dependencies.py new file mode 100755 index 00000000..10ffcee2 --- /dev/null +++ b/bin/list_dependencies.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Provide a list of module which are dependencies +# of local-src modules excluding local-src modules +# +# Arguments: +# list of module (coma separated), restrict list of +# dependencies to the dependencies of this list. +# +# Usage: +# ./odoo/bin/list_dependencies.py local_module1,local_module2 +import sys +import os +import ast + +BASE_DIR = os.getcwd() +LOCAL_SRC_DIR = os.path.join(BASE_DIR, 'odoo', 'local-src') + +dependencies = set() +local_modules = os.listdir(LOCAL_SRC_DIR) +if len(sys.argv) > 1: + modules = sys.argv[1].split(',') +else: + modules = local_modules +for mod in modules: + # read __manifest__ + manifest_path = os.path.join(LOCAL_SRC_DIR, mod, '__manifest__.py') + if not os.path.isfile(manifest_path): + continue + with open(manifest_path) as manifest: + data = ast.literal_eval(manifest.read()) + dependencies.update(data['depends']) + +# remove local-src from list of dependencies +dependencies = dependencies.difference(local_modules) +print(','.join(dependencies) or 'base') diff --git a/bin/runmigration b/bin/runmigration new file mode 100755 index 00000000..8a503d0e --- /dev/null +++ b/bin/runmigration @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Run marabunta steps +# +# Mainly used from Travis, +# +# If a cached dump of previous version is found, restore it. +# Then play remaining marabunta steps from this state. +# +# Otherwise install from scratch. +# +# And finally make a database dump if none exists for current VERSION. +# +# TODO: store cache on S3 to store one DB per version +# +# Environment variables: +# +# CREATE_DB_CACHE: +# if set to "true", will create a dump named "odoo_sample_$VERSION.dmp" +# +# LOAD_DB_CACHE: +# if set to "false", will skip trying to reload cached dump named "odoo_sample_$VERSION.dmp" +# +# MIG_LOAD_VERSION_CEIL: +# Version number passed to search for an older dump. +# +# It will load a dump lower than "odoo_sample_$MIG_LOAD_VERSION_CEIL.dmp" +# This is useful if you bumped odoo/VERSION as it won't match existing +# dumps. +# +# For instance you have made a dump 10.1.0, you are now on the version +# 10.2.0, if you pass your current version it will search for a dump +# lower than 10.2.0 and restore the 10.1.0. Then play the remaining +# steps on top of it. +set -e + +wait_postgres.sh +CACHE_DIR=/opt/.cachedb + +echo $CACHE_DIR + +VERSION=$(cat /opt/odoo/VERSION) + +CACHED_DUMP="$CACHE_DIR/odoo_sample_$VERSION.dmp" + +if [ "$LOAD_DB_CACHE" != "false" ]; then + + # If we want to run the migration steps on top of a previous dump + # useful when odoo/VERSION was edited + if [ -n "$MIG_LOAD_VERSION_CEIL" ]; then + echo "New version - Searching for previous version dump 🔭" + if [ -d "$CACHE_DIR" ]; then + # Filter dumps of higher releases + export MAX_DUMP="$CACHE_DIR/odoo_sample_${MIG_LOAD_VERSION_CEIL}.dmp" + CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1) + else + echo "No cached migration sample dump found" + fi + fi +else + echo "Dump cache load disabled." +fi + +if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then + echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" + echo "Restore Database dump from cache 📦⮕ 🐘" + createdb -O $DB_USER $DB_NAME + psql -q -o /dev/null -f "$CACHED_DUMP" + echo "Do migration on top of restored dump" +else + echo "Do migration from scratch 🐢 🐢 🐢" +fi + +migrate + +# Create a dump if none exist for the current VERSION +if [ "$CREATE_DB_CACHE" == "true" -a ! -f "$CACHED_DUMP" ]; then + echo "Save DB to cache $CACHED_DUMP 🐘⮕ 📦" + mkdir -p "$CACHE_DIR" + pg_dump -Fp -O -f "$CACHED_DUMP" + ls -l $CACHED_DUMP +fi diff --git a/bin/runtests b/bin/runtests index 13368ebe..cb163fd1 100755 --- a/bin/runtests +++ b/bin/runtests @@ -1,10 +1,29 @@ #!/bin/bash # -# mainly used from Travis, it creates a database, runs tests on it and drops it. +# Run unit tests of local-src modules +# +# mainly used from Travis, +# it creates a database, +# if a cached dump of official and oca modules exists restore it +# otherwise install dependencies and create a dump to cache +# then install local addons +# runs tests on it and drops it. # # Arguments: # optional: name of the addons to test, separated by , # +# Environment variables: +# +# CREATE_DB_CACHE: +# if set to "true", will create a dump named "odoo_test_$VERSION.dmp" +# +# LOAD_DB_CACHE: +# if set to "false", will skip trying to reload cached dump named "odoo_test_$VERSION.dmp" +# +# SUBS_MD5: +# value to tag the database dump. Will search for a dump named +# "odoo_test_$SUBS_MD5.dmp" if not found it will create one. +# set -e # TODO: if we are not in TRAVIS, make a template then run tests on a copy @@ -16,7 +35,7 @@ if [ "$ODOO_VERSION" == "9.0" ] then ODOO_BIN_PATH=/opt/odoo/src/odoo.py fi - +CACHE_DIR=/opt/.cachedb if [ -z $1 ] then @@ -27,11 +46,43 @@ else LOCAL_ADDONS=$1 fi +DEPS_ADDONS=$(list_dependencies.py "$LOCAL_ADDONS") + DB_NAME_TEST=${DB_NAME}_test -PGPASSWORD=$DB_PASSWORD createdb -h $DB_HOST -U $DB_USER -O $DB_USER ${DB_NAME_TEST} + +echo "Create database" +createdb -O $DB_USER ${DB_NAME_TEST} + +if [[ ! -z "$SUBS_MD5" ]]; then + CACHED_DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp" +fi + +echo "Submodule addons MD5 is: $SUBS_MD5" + +if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then + echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘" + echo "Restore Database dump from cache matching MD5 📦⮕ 🐘" + psql -q -o /dev/null -d $DB_NAME_TEST -f "$CACHED_DUMP" + psql -d $DB_NAME_TEST -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name" +else + if [ "$LOAD_DB_CACHE" == "false" ]; then + echo "Dump cache load disabled." + else + echo "No cached dump found matching MD5 🐢 🐢 🐢" + fi + echo "🔨🔨 Install official/OCA modules 🔨🔨" + odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" -i ${DEPS_ADDONS} + if [ "$CREATE_DB_CACHE" == "true" -a ! -z "$CACHED_DUMP" ]; then + echo "Generate dump $CACHED_DUMP into cache 🐘⮕ 📦" + mkdir -p "$CACHE_DIR" + pg_dump -Fp -d $DB_NAME_TEST -O -f "$CACHED_DUMP" + fi +fi +echo "🔧🔧 Install local-src modules 🔧🔧" odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" -i ${LOCAL_ADDONS} +odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --test-enable --log-level=test --log-handler=":INFO" -u ${LOCAL_ADDONS} coverage run --source="${LOCAL_SRC_DIR}" "${ODOO_BIN_PATH}" --stop-after-init --workers=0 --database $DB_NAME_TEST --test-enable --log-level=test --log-handler=":INFO" -u ${LOCAL_ADDONS} -PGPASSWORD=$DB_PASSWORD dropdb -h $DB_HOST -U $DB_USER ${DB_NAME_TEST} +dropdb ${DB_NAME_TEST} coverage report -m diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 872e129a..90287590 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -17,6 +17,18 @@ services: - RUNNING_ENV=dev - MARABUNTA_MODE=demo - LOG_HANDLER=:WARN + # cached database dumps config for `runmigration` and `runtests` + - CREATE_DB_CACHE=false # set it to 'true' to create dumps + - LOAD_DB_CACHE=true # by default will always search for existing dumps + # SUBS_MD5 is `runtests` only, you need to define it to identify dumps, + # a good practice is to generate it based on the content you have in + # your dependencies (`odoo/src`, `odoo/external-src`) + # See README.md for more. + - SUBS_MD5= + # MIG_LOAD_VERSION is `runmigration` only, define it if you want to load + # a prior release dump that doesn't match `odoo/VERSION` number. + # See README.md for more. + - MIG_LOAD_VERSION_CEIL= db: image: postgres:9.6