From 51d5df0849dec1207c73e5179b0836b3c82eaba3 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 2 Nov 2017 12:53:01 +0800 Subject: [PATCH] Update replication policy API to support trigger and filter --- docs/swagger.yaml | 44 ++++++++++- make/common/db/registry.sql | 2 + make/common/db/registry_sqlite.sql | 2 + src/common/dao/replication_job.go | 65 ++++++++++++----- src/common/models/replicate_test.go | 43 +++++++++++ src/common/models/replication_job.go | 93 +++++++++++++++++++----- src/ui/api/harborapi_test.go | 5 +- src/ui/api/replication_policy_test.go | 29 ++++++-- tests/apitests/apilib/rep_policy_post.go | 11 ++- tools/migration/changelog.md | 5 ++ 10 files changed, 252 insertions(+), 47 deletions(-) create mode 100644 src/common/models/replicate_test.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 1a4143473..f787b66df 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2397,9 +2397,18 @@ definitions: description: type: string description: The description of the policy. - cron_str: + trigger: type: string - description: The cron string for schedule job. + description: The trigger for schedule job. + filters: + type: array + description: >- + The replication policy filter array. + items: + $ref: '#/definitions/RepFilter' + replicate_deletion: + type: string + description: Whether replication deletion operation. start_time: type: string description: The start time of the policy. @@ -2428,6 +2437,18 @@ definitions: name: type: string description: The policy name. + trigger: + type: string + description: The trigger for schedule job. + filters: + type: array + description: >- + The replication policy filter array. + items: + $ref: '#/definitions/RepFilter' + replicate_deletion: + type: string + description: Whether replication deletion operation. enabled: type: integer format: int @@ -2449,9 +2470,26 @@ definitions: description: type: string description: The description of the policy. - cron_str: + trigger: type: string description: The cron string for schedule job. + filters: + type: array + description: The replication policy filter array. + items: + $ref: '#/definitions/RepFilter' + replicate_deletion: + type: string + description: Whether replication deletion operation. + RepFilter: + type: object + properties: + type: + type: string + description: The replication policy filter type. + value: + type: string + description: The replication policy filter value. RepPolicyEnablementReq: type: object properties: diff --git a/make/common/db/registry.sql b/make/common/db/registry.sql index 446627971..994b108d7 100644 --- a/make/common/db/registry.sql +++ b/make/common/db/registry.sql @@ -145,6 +145,8 @@ create table replication_policy ( description text, deleted tinyint (1) DEFAULT 0 NOT NULL, cron_str varchar(256), + filters varchar(1024), + replicate_deletion tinyint (1) DEFAULT 0 NOT NULL, start_time timestamp NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, diff --git a/make/common/db/registry_sqlite.sql b/make/common/db/registry_sqlite.sql index ce2cb9c20..2528a845d 100644 --- a/make/common/db/registry_sqlite.sql +++ b/make/common/db/registry_sqlite.sql @@ -141,6 +141,8 @@ create table replication_policy ( description text, deleted tinyint (1) DEFAULT 0 NOT NULL, cron_str varchar(256), + filters varchar(1024), + replicate_deletion tinyint (1) DEFAULT 0 NOT NULL, start_time timestamp NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP diff --git a/src/common/dao/replication_job.go b/src/common/dao/replication_job.go index eb6593259..b9997838e 100644 --- a/src/common/dao/replication_job.go +++ b/src/common/dao/replication_job.go @@ -104,29 +104,18 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) { // AddRepPolicy ... func AddRepPolicy(policy models.RepPolicy) (int64, error) { - o := GetOrmer() - sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)` - p, err := o.Raw(sql).Prepare() - if err != nil { + if err := policy.MarshalFilter(); err != nil { return 0, err } - - params := []interface{}{} - params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr) now := time.Now() + policy.CreationTime = now + policy.UpdateTime = now if policy.Enabled == 1 { - params = append(params, now) - } else { - params = append(params, nil) + policy.StartTime = now } - params = append(params, now, now) + policy.Deleted = 0 - r, err := p.Exec(params...) - if err != nil { - return 0, err - } - id, err := r.LastInsertId() - return id, err + return GetOrmer().Insert(&policy) } // GetRepPolicy ... @@ -143,6 +132,10 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) { return nil, err } + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + return &policy, nil } @@ -154,7 +147,8 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error sql := `select rp.id, rp.project_id, rp.target_id, rt.name as target_name, rp.name, rp.enabled, rp.description, - rp.cron_str, rp.start_time, rp.creation_time, rp.update_time, + rp.cron_str, rp.filters, rp.replicate_deletion,rp.start_time, + rp.creation_time, rp.update_time, count(rj.status) as error_job_count from replication_policy rp left join replication_target rt on rp.target_id=rt.id @@ -180,6 +174,13 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil { return nil, err } + + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } @@ -197,6 +198,10 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) { return nil, err } + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + return &policy, nil } @@ -211,6 +216,12 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) { return nil, err } + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } @@ -225,6 +236,12 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) { return nil, err } + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } @@ -239,14 +256,24 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol return nil, err } + for _, policy := range policies { + if err := policy.UnmarshalFilter(); err != nil { + return nil, err + } + } + return policies, nil } // UpdateRepPolicy ... func UpdateRepPolicy(policy *models.RepPolicy) error { + if err := policy.MarshalFilter(); err != nil { + return err + } o := GetOrmer() policy.UpdateTime = time.Now() - _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", "CronStr", "UpdateTime") + _, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", + "Trigger", "FiltersInDB", "ReplicateDeletion", "UpdateTime") return err } diff --git a/src/common/models/replicate_test.go b/src/common/models/replicate_test.go new file mode 100644 index 000000000..142242aee --- /dev/null +++ b/src/common/models/replicate_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMarshalAndUnmarshalFilter(t *testing.T) { + filters := []*RepFilter{ + &RepFilter{ + Type: "repository", + Value: "library/ubuntu*", + }, + } + policy := &RepPolicy{ + Filters: filters, + } + + err := policy.MarshalFilter() + require.Nil(t, err) + + policy.Filters = nil + err = policy.UnmarshalFilter() + require.Nil(t, err) + + assert.EqualValues(t, filters, policy.Filters) +} diff --git a/src/common/models/replication_job.go b/src/common/models/replication_job.go index 255fc1d9a..f17ec5875 100644 --- a/src/common/models/replication_job.go +++ b/src/common/models/replication_job.go @@ -15,10 +15,13 @@ package models import ( + "encoding/json" + "fmt" "time" "github.com/astaxie/beego/validation" "github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/replication" ) const ( @@ -38,21 +41,23 @@ const ( // RepPolicy is the model for a replication policy, which associate to a project and a target (destination) type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - ProjectID int64 `orm:"column(project_id)" json:"project_id"` - ProjectName string `json:"project_name,omitempty"` - TargetID int64 `orm:"column(target_id)" json:"target_id"` - TargetName string `json:"target_name,omitempty"` - Name string `orm:"column(name)" json:"name"` - // Target RepTarget `orm:"-" json:"target"` - Enabled int `orm:"column(enabled)" json:"enabled"` - Description string `orm:"column(description)" json:"description"` - CronStr string `orm:"column(cron_str)" json:"cron_str"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` - UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` - ErrorJobCount int `json:"error_job_count"` - Deleted int `orm:"column(deleted)" json:"deleted"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` + ProjectID int64 `orm:"column(project_id)" json:"project_id"` + ProjectName string `orm:"-" json:"project_name,omitempty"` + TargetID int64 `orm:"column(target_id)" json:"target_id"` + TargetName string `orm:"-" json:"target_name,omitempty"` + Name string `orm:"column(name)" json:"name"` + Enabled int `orm:"column(enabled)" json:"enabled"` + Description string `orm:"column(description)" json:"description"` + Trigger string `orm:"column(cron_str)" json:"trigger"` + Filters []*RepFilter `orm:"-" json:"filters"` + FiltersInDB string `orm:"column(filters)" json:"-"` + ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` + StartTime time.Time `orm:"column(start_time)" json:"start_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` + ErrorJobCount int `orm:"-" json:"error_job_count"` + Deleted int `orm:"column(deleted)" json:"deleted"` } // Valid ... @@ -77,8 +82,62 @@ func (r *RepPolicy) Valid(v *validation.Validation) { v.SetError("enabled", "must be 0 or 1") } - if len(r.CronStr) > 256 { - v.SetError("cron_str", "max length is 256") + if len(r.Trigger) > 256 { + v.SetError("trigger", "max length is 256") + } + + for _, filter := range r.Filters { + filter.Valid(v) + } + + if err := r.MarshalFilter(); err != nil { + v.SetError("filters", err.Error()) + } + if len(r.Filters) > 1024 { + v.SetError("filters", "max length is 1024") + } +} + +// MarshalFilter marshal RepFilter array to json string +func (r *RepPolicy) MarshalFilter() error { + if r.Filters != nil { + b, err := json.Marshal(r.Filters) + if err != nil { + return err + } + r.FiltersInDB = string(b) + } + return nil +} + +// UnmarshalFilter unmarshal json string to RepFilter array +func (r *RepPolicy) UnmarshalFilter() error { + if len(r.FiltersInDB) > 0 { + filter := []*RepFilter{} + if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil { + return err + } + r.Filters = filter + } + return nil +} + +// RepFilter holds information for the replication policy filter +type RepFilter struct { + Type string `json:"type"` + Value string `json:"value"` +} + +// Valid ... +func (r *RepFilter) Valid(v *validation.Validation) { + if !(r.Type == replication.FilterItemKindProject || + r.Type == replication.FilterItemKindRepository || + r.Type == replication.FilterItemKindTag) { + v.SetError("filter.type", fmt.Sprintf("invalid filter type: %s", r.Type)) + } + + if len(r.Value) == 0 { + v.SetError("filter.value", "can not be empty") } } diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index ef754bad0..fa60ec3c9 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -711,7 +711,10 @@ func (a testapi) AddPolicy(authInfo usrInfo, repPolicy apilib.RepPolicyPost) (in _sling = _sling.Path(path) _sling = _sling.BodyJSON(repPolicy) - httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo) + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) + if httpStatusCode != http.StatusCreated { + log.Println(string(body)) + } return httpStatusCode, err } diff --git a/src/ui/api/replication_policy_test.go b/src/ui/api/replication_policy_test.go index f518537a2..622cbecb3 100644 --- a/src/ui/api/replication_policy_test.go +++ b/src/ui/api/replication_policy_test.go @@ -15,10 +15,14 @@ package api import ( "fmt" - "github.com/stretchr/testify/assert" - "github.com/vmware/harbor/tests/apitests/apilib" "strconv" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/replication" + "github.com/vmware/harbor/tests/apitests/apilib" ) const ( @@ -37,7 +41,12 @@ func TestPoliciesPost(t *testing.T) { //add target CommonAddTarget() targetID := int64(CommonGetTarget()) - repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName} + repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{ + &models.RepFilter{ + Type: replication.FilterItemKindRepository, + Value: "library/ubuntu*", + }, + }} fmt.Println("Testing Policies Post API") @@ -52,7 +61,7 @@ func TestPoliciesPost(t *testing.T) { } //-------------------case 2 : response code = 409------------------------// - fmt.Println("case 1 : response code = 409:policy already exists") + fmt.Println("case 2 : response code = 409:policy already exists") httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) if err != nil { t.Error("Error while add policy", err.Error()) @@ -108,7 +117,7 @@ func TestPoliciesPost(t *testing.T) { } //-------------------case 7 : response code = 400------------------------// - fmt.Println("case 6 : response code = 400:target_id does not exist.") + fmt.Println("case 7 : response code = 400:target_id does not exist.") repPolicy.TargetId = int64(1111) httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) @@ -119,6 +128,16 @@ func TestPoliciesPost(t *testing.T) { assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") } + fmt.Println("case 8 : response code = 400: invalid filter") + repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{ + &models.RepFilter{ + Type: "replication", + Value: "", + }, + }} + httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy) + require.Nil(t, err) + assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400") } func TestPoliciesList(t *testing.T) { diff --git a/tests/apitests/apilib/rep_policy_post.go b/tests/apitests/apilib/rep_policy_post.go index 6b8d42731..a78b01aa6 100644 --- a/tests/apitests/apilib/rep_policy_post.go +++ b/tests/apitests/apilib/rep_policy_post.go @@ -1,10 +1,10 @@ -/* +/* * Harbor API * * These APIs provide services for manipulating Harbor project. * * OpenAPI spec version: 0.3.0 - * + * * Generated by: https://github.com/swagger-api/swagger-codegen.git * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,10 @@ package apilib +import ( + "github.com/vmware/harbor/src/common/models" +) + type RepPolicyPost struct { // The project ID. @@ -32,4 +36,7 @@ type RepPolicyPost struct { // The policy name. Name string `json:"name,omitempty"` + + // Filters + Filters []*models.RepFilter `json:"filters"` } diff --git a/tools/migration/changelog.md b/tools/migration/changelog.md index 8c40e5986..d2e9a0b70 100644 --- a/tools/migration/changelog.md +++ b/tools/migration/changelog.md @@ -56,3 +56,8 @@ Changelog for harbor database schema - insert data into table `project_metadata` - delete column `public` from table `project` - add column `insecure` to table `replication_target` + +## 1.3.1 + + - add column `filters` to table `replication_policy` + - add column `replicate_deletion` to table `replication_policy`