From 85ad1392f818df26298200ae7a469b47b5400b82 Mon Sep 17 00:00:00 2001 From: saga92 Date: Mon, 9 May 2016 14:11:34 +0800 Subject: [PATCH] add data migration module --- .gitignore | 1 + Deploy/db/registry.sql | 6 + migration/Dockerfile | 23 ++++ migration/README.md | 51 ++++++++ migration/alembic.sql | 4 + migration/alembic.tpl | 68 ++++++++++ migration/install.sh | 3 + migration/migration.cfg | 4 + migration/migration_harbor/env.py | 85 ++++++++++++ migration/migration_harbor/script.py.mako | 24 ++++ migration/migration_harbor/versions/0_1_1.py | 128 +++++++++++++++++++ migration/run.sh | 94 ++++++++++++++ 12 files changed, 491 insertions(+) create mode 100644 migration/Dockerfile create mode 100644 migration/README.md create mode 100644 migration/alembic.sql create mode 100644 migration/alembic.tpl create mode 100755 migration/install.sh create mode 100644 migration/migration.cfg create mode 100644 migration/migration_harbor/env.py create mode 100644 migration/migration_harbor/script.py.mako create mode 100644 migration/migration_harbor/versions/0_1_1.py create mode 100755 migration/run.sh diff --git a/.gitignore b/.gitignore index 6b04c4ad3..b53274b48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Deploy/config/ui/app.conf Deploy/config/db/env Deploy/harbor.cfg ui/ui +*.pyc diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index bd0644b33..1fd0eafa3 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -111,3 +111,9 @@ create table properties ( insert into properties (k, v) values ('schema_version', '0.1.1'); + +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +insert into alembic_version values ('0.1.1'); diff --git a/migration/Dockerfile b/migration/Dockerfile new file mode 100644 index 000000000..eb961aac9 --- /dev/null +++ b/migration/Dockerfile @@ -0,0 +1,23 @@ +FROM mysql:5.6 + +MAINTAINER bhe@vmware.com + +RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list + +RUN apt-get update + +RUN apt-get install -y curl python python-pip git python-mysqldb + +RUN pip install alembic + +RUN mkdir -p /harbor-migration + +WORKDIR /harbor-migration + +COPY ./ ./ + +COPY ./migration.cfg ./ + +RUN ./install.sh + +ENTRYPOINT ["./run.sh"] diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 000000000..1d013e47f --- /dev/null +++ b/migration/README.md @@ -0,0 +1,51 @@ +# migration +Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor) + +**WARNING!!** You must backup your data before migrating + +###installation +- step 1: modify migration.cfg +- step 2: build image from dockerfile + ``` + cd harbor-migration + + docker build -t your-image-name . + ``` + +###migration operation +- show instruction of harbor-migration + + ```docker run your-image-name help``` + +- create backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup + ``` + +- restore from backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore + ``` + +- perform database schema upgrade + + ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` + +- perform database schema downgrade(downgrade has been disabled) + + ```docker run -v /data/database:/var/lib/mysql your-image-name down base``` + +###migration step +- step 1: stop and remove harbor service + + ``` + docker-compose stop && docker-compose rm -f + ``` +- step 2: perform migration operation +- step 3: rebuild newest harbor images and restart service + + ``` + docker-compose build && docker-compose up -d + ``` diff --git a/migration/alembic.sql b/migration/alembic.sql new file mode 100644 index 000000000..21dc7a1de --- /dev/null +++ b/migration/alembic.sql @@ -0,0 +1,4 @@ +use `registry`; +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/migration/alembic.tpl b/migration/alembic.tpl new file mode 100644 index 000000000..548c6028d --- /dev/null +++ b/migration/alembic.tpl @@ -0,0 +1,68 @@ +echo " +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migration_harbor + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migration_harbor/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S" diff --git a/migration/install.sh b/migration/install.sh new file mode 100755 index 000000000..4d707407c --- /dev/null +++ b/migration/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source ./migration.cfg +source ./alembic.tpl > ./alembic.ini diff --git a/migration/migration.cfg b/migration/migration.cfg new file mode 100644 index 000000000..a383853ac --- /dev/null +++ b/migration/migration.cfg @@ -0,0 +1,4 @@ +db_username="root" +db_password="root123" +db_port="3306" +db_name="registry" diff --git a/migration/migration_harbor/env.py b/migration/migration_harbor/env.py new file mode 100644 index 000000000..646f39862 --- /dev/null +++ b/migration/migration_harbor/env.py @@ -0,0 +1,85 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migration/migration_harbor/script.py.mako b/migration/migration_harbor/script.py.mako new file mode 100644 index 000000000..43c09401b --- /dev/null +++ b/migration/migration_harbor/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py new file mode 100644 index 000000000..0f21b5436 --- /dev/null +++ b/migration/migration_harbor/versions/0_1_1.py @@ -0,0 +1,128 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""0.1.0 to 0.1.1 + +Revision ID: 0.1.1 +Revises: +Create Date: 2016-04-18 18:32:14.101897 + +""" + +# revision identifiers, used by Alembic. +revision = '0.1.1' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from datetime import datetime + +Session = sessionmaker() + +Base = declarative_base() + +class Properties(Base): + __tablename__ = 'properties' + + k = sa.Column(sa.String(64), primary_key = True) + v = sa.Column(sa.String(128), nullable = False) + +class ProjectMember(Base): + __tablename__ = 'project_member' + + project_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), primary_key = True) + role = sa.Column(sa.Integer(), nullable = False) + creation_time = sa.Column(sa.DateTime(), nullable = True) + update_time = sa.Column(sa.DateTime(), nullable = True) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), + sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), + +class UserProjectRole(Base): + __tablename__ = 'user_project_role' + + upr_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) + pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) + project_role = relationship("ProjectRole") + +class ProjectRole(Base): + __tablename__ = 'project_role' + + pr_id = sa.Column(sa.Integer(), primary_key = True) + project_id = sa.Column(sa.Integer(), nullable = False) + role_id = sa.Column(sa.Integer(), nullable = False) + sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) + +class Access(Base): + __tablename__ = 'access' + + access_id = sa.Column(sa.Integer(), primary_key = True) + access_code = sa.Column(sa.String(1)) + comment = sa.Column(sa.String(30)) + +def upgrade(): + """ + update schema&data + """ + bind = op.get_bind() + session = Session(bind=bind) + + #delete M from table access + acc = session.query(Access).filter_by(access_id=1).first() + session.delete(acc) + + #create table property + Properties.__table__.create(bind) + session.add(Properties(k='schema_version', v='0.1.1')) + + #create table project_member + ProjectMember.__table__.create(bind) + + #fill data + join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all() + for result in join_result: + session.add(ProjectMember(project_id=result.project_role.project_id, \ + user_id=result.user_id, role=result.project_role.role_id, \ + creation_time=datetime.now(), update_time=datetime.now())) + + #drop user_project_role table before drop project_role + #because foreign key constraint + op.drop_table('user_project_role') + op.drop_table('project_role') + + #add column to table project + op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True)) + + #add column to table role + op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) + + #add column to table user + op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) + op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + session.commit() + +def downgrade(): + """ + Downgrade has been disabled. + """ + pass diff --git a/migration/run.sh b/migration/run.sh new file mode 100755 index 000000000..806378ff5 --- /dev/null +++ b/migration/run.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +source ./migration.cfg + +WAITTIME=60 + +DBCNF="-hlocalhost -u${db_username}" + +#prevent shell to print insecure message +export MYSQL_PWD="${db_password}" + +if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then + echo "Usage:" + echo "backup perform database backup" + echo "restore perform database restore" + echo "up, upgrade perform database schema upgrade" + echo "h, help usage help" + exit 0 +fi + +if [[ $1 = "up" || $1 = "upgrade" ]]; then + echo "Please backup before upgrade." + read -p "Enter y to continue updating or n to abort:" ans + case $ans in + [Yy]* ) + ;; + [Nn]* ) + exit 0 + ;; + * ) echo "illegal answer: $ans. Upgrade abort!!" + exit 1 + ;; + esac + +fi + +echo 'Trying to start mysql server...' +DBRUN=0 +nohup mysqld 2>&1 > ./nohup.log& +for i in $(seq 1 $WAITTIME); do + echo "$(/usr/sbin/service mysql status)" + if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then + sleep 1 + else + DBRUN=1 + break + fi +done + +if [[ $DBRUN -eq 0 ]]; then + echo "timeout. Can't run mysql server." + exit 1 +fi + +key="$1" +case $key in +up|upgrade) + VERSION="$2" + if [[ -z $VERSION ]]; then + VERSION="head" + echo "Version is not specified. Default version is head." + fi + echo "Performing upgrade ${VERSION}..." + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then + echo "table alembic_version does not exist. Trying to initial alembic_version." + mysql $DBCNF < ./alembic.sql + #compatible with version 0.1.0 and 0.1.1 + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='properties'") -eq 0 ]]; then + echo "table properties does not exist. The version of registry is 0.1.0" + else + echo "The version of registry is 0.1.1" + mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')" + fi + fi + alembic -c ./alembic.ini upgrade ${VERSION} + echo "Upgrade performed." + ;; +backup) + echo "Performing backup..." + mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql + echo "Backup performed." + ;; +restore) + echo "Performing restore..." + mysql $DBCNF < ./backup/registry.sql + echo "Restore performed." + ;; +*) + echo "unknown option" + exit 0 + ;; +esac