diff --git a/make/migrations/postgresql/0011_1.10.0_schema.up.sql b/make/migrations/postgresql/0011_1.10.0_schema.up.sql index ad4c5c35a..57f6ff4ff 100644 --- a/make/migrations/postgresql/0011_1.10.0_schema.up.sql +++ b/make/migrations/postgresql/0011_1.10.0_schema.up.sql @@ -19,12 +19,14 @@ CREATE TABLE scanner_registration CREATE TABLE scan_report ( id SERIAL PRIMARY KEY NOT NULL, + uuid VARCHAR(64) UNIQUE NOT NULL, digest VARCHAR(256) NOT NULL, registration_uuid VARCHAR(64) NOT NULL, mime_type VARCHAR(256) NOT NULL, job_id VARCHAR(32), status VARCHAR(16) NOT NULL, status_code INTEGER DEFAULT 0, + status_rev BIGINT DEFAULT 0, report JSON, start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, diff --git a/src/core/api/scanners.go b/src/core/api/scanners.go index 34a2ecb89..ee00888f8 100644 --- a/src/core/api/scanners.go +++ b/src/core/api/scanners.go @@ -76,7 +76,7 @@ func (sa *ScannerAPI) List() { } // Get query key words - kws := make(map[string]string) + kws := make(map[string]interface{}) properties := []string{"name", "description", "url"} for _, k := range properties { kw := sa.GetString(k) @@ -316,7 +316,7 @@ func (sa *ScannerAPI) get() *scanner.Registration { func (sa *ScannerAPI) checkDuplicated(property, value string) bool { // Explicitly check if conflict - kw := make(map[string]string) + kw := make(map[string]interface{}) kw[property] = value query := &q.Query{ diff --git a/src/core/api/scanners_test.go b/src/core/api/scanners_test.go index f0f7d3edd..02744788f 100644 --- a/src/core/api/scanners_test.go +++ b/src/core/api/scanners_test.go @@ -296,7 +296,7 @@ func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() { } func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) { - kw := make(map[string]string, 1) + kw := make(map[string]interface{}, 1) kw["name"] = r.Name query := &q.Query{ Keywords: kw, @@ -304,7 +304,7 @@ func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) { emptyL := make([]*scanner.Registration, 0) suite.mockC.On("ListRegistrations", query).Return(emptyL, nil) - kw2 := make(map[string]string, 1) + kw2 := make(map[string]interface{}, 1) kw2["url"] = r.URL query2 := &q.Query{ Keywords: kw2, diff --git a/src/pkg/q/query.go b/src/pkg/q/query.go index 048a25298..74c8e3da9 100644 --- a/src/pkg/q/query.go +++ b/src/pkg/q/query.go @@ -21,5 +21,5 @@ type Query struct { // Page size PageSize int64 // List of key words - Keywords map[string]string + Keywords map[string]interface{} } diff --git a/src/pkg/scan/dao/scan/report.go b/src/pkg/scan/dao/scan/report.go new file mode 100644 index 000000000..653f864df --- /dev/null +++ b/src/pkg/scan/dao/scan/report.go @@ -0,0 +1,140 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "fmt" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/pkg/errors" +) + +func init() { + orm.RegisterModel(new(Report)) +} + +// CreateReport creates new report +func CreateReport(r *Report) (int64, error) { + o := dao.GetOrmer() + return o.Insert(r) +} + +// DeleteReport deletes the given report +func DeleteReport(uuid string) error { + o := dao.GetOrmer() + qt := o.QueryTable(new(Report)) + + // Delete report with query way + count, err := qt.Filter("uuid", uuid).Delete() + if err != nil { + return err + } + + if count == 0 { + return errors.Errorf("no report with uuid %s deleted", uuid) + } + + return nil +} + +// ListReports lists the reports with given query parameters. +// Keywords in query here will be enforced with `exact` way. +func ListReports(query *q.Query) ([]*Report, error) { + o := dao.GetOrmer() + qt := o.QueryTable(new(Report)) + + if query != nil { + if len(query.Keywords) > 0 { + for k, v := range query.Keywords { + if vv, ok := v.([]interface{}); ok { + qt = qt.Filter(fmt.Sprintf("%s__in", k), vv...) + } + + qt = qt.Filter(k, v) + } + } + + if query.PageNumber > 0 && query.PageSize > 0 { + qt = qt.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize) + } + } + + l := make([]*Report, 0) + _, err := qt.All(&l) + + return l, err +} + +// UpdateReportData only updates the `report` column with conditions matched. +func UpdateReportData(uuid string, report string, statusRev int64) error { + o := dao.GetOrmer() + qt := o.QueryTable(new(Report)) + + data := make(orm.Params) + data["report"] = report + data["status_rev"] = statusRev + + count, err := qt.Filter("uuid", uuid). + Filter("status_rev__lte", statusRev).Update(data) + + if err != nil { + return err + } + + if count == 0 { + return errors.Errorf("no report with uuid %s updated", uuid) + } + + return nil +} + +// UpdateReportStatus updates the report `status` with conditions matched. +func UpdateReportStatus(uuid string, status string, statusCode int, statusRev int64) error { + o := dao.GetOrmer() + qt := o.QueryTable(new(Report)) + + data := make(orm.Params) + data["status"] = status + data["status_code"] = statusCode + data["status_rev"] = statusRev + + count, err := qt.Filter("uuid", uuid). + Filter("status_rev__lte", statusRev). + Filter("status_code__lte", statusCode).Update(data) + + if err != nil { + return err + } + + if count == 0 { + return errors.Errorf("no report with uuid %s updated", uuid) + } + + return nil +} + +// UpdateJobID updates the report `job_id` column +func UpdateJobID(uuid string, jobID string) error { + o := dao.GetOrmer() + qt := o.QueryTable(new(Report)) + + params := make(orm.Params, 1) + params["job_id"] = jobID + _, err := qt.Filter("uuid", uuid).Update(params) + + return err +} diff --git a/src/pkg/scan/dao/scan/report_test.go b/src/pkg/scan/dao/scan/report_test.go new file mode 100644 index 000000000..63d318128 --- /dev/null +++ b/src/pkg/scan/dao/scan/report_test.go @@ -0,0 +1,131 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/q" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ReportTestSuite is test suite of testing report DAO. +type ReportTestSuite struct { + suite.Suite +} + +// TestReport is the entry of ReportTestSuite. +func TestReport(t *testing.T) { + suite.Run(t, &ReportTestSuite{}) +} + +// SetupSuite prepares env for test suite. +func (suite *ReportTestSuite) SetupSuite() { + dao.PrepareTestForPostgresSQL() +} + +// SetupTest prepares env for test case. +func (suite *ReportTestSuite) SetupTest() { + r := &Report{ + UUID: "uuid", + Digest: "digest1001", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + Status: job.PendingStatus.String(), + StatusCode: job.PendingStatus.Code(), + } + + id, err := CreateReport(r) + require.NoError(suite.T(), err) + require.Condition(suite.T(), func() (success bool) { + success = id > 0 + return + }) +} + +// TearDownTest clears enf for test case. +func (suite *ReportTestSuite) TearDownTest() { + err := DeleteReport("uuid") + require.NoError(suite.T(), err) +} + +// TestReportList tests list reports with query parameters. +func (suite *ReportTestSuite) TestReportList() { + query1 := &q.Query{ + PageSize: 1, + PageNumber: 1, + Keywords: map[string]interface{}{ + "digest": "digest1001", + "registration_uuid": "ruuid", + "mime_type": v1.MimeTypeNativeReport, + }, + } + l, err := ListReports(query1) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + query2 := &q.Query{ + PageSize: 1, + PageNumber: 1, + Keywords: map[string]interface{}{ + "digest": "digest1002", + }, + } + l, err = ListReports(query2) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 0, len(l)) +} + +// TestReportUpdateJobID tests update job ID of the report. +func (suite *ReportTestSuite) TestReportUpdateJobID() { + err := UpdateJobID("uuid", "jobid001") + require.NoError(suite.T(), err) + + l, err := ListReports(nil) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + assert.Equal(suite.T(), "jobid001", l[0].JobID) +} + +// TestReportUpdateReportData tests update the report data. +func (suite *ReportTestSuite) TestReportUpdateReportData() { + err := UpdateReportData("uuid", "{}", 1000) + require.NoError(suite.T(), err) + + l, err := ListReports(nil) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + assert.Equal(suite.T(), "{}", l[0].Report) + + err = UpdateReportData("uuid", "{\"a\": 900}", 900) + require.Error(suite.T(), err) +} + +// TestReportUpdateStatus tests update the report status. +func (suite *ReportTestSuite) TestReportUpdateStatus() { + err := UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000) + require.NoError(suite.T(), err) + + err = UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900) + require.Error(suite.T(), err) + + err = UpdateReportStatus("uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000) + require.Error(suite.T(), err) +} diff --git a/src/pkg/scan/dao/scanner/registration_test.go b/src/pkg/scan/dao/scanner/registration_test.go index c76a2efd2..d7a228a5f 100644 --- a/src/pkg/scan/dao/scanner/registration_test.go +++ b/src/pkg/scan/dao/scanner/registration_test.go @@ -107,7 +107,7 @@ func (suite *RegistrationDAOTestSuite) TestList() { require.Equal(suite.T(), 1, len(l)) // with query and found items - keywords := make(map[string]string) + keywords := make(map[string]interface{}) keywords["description"] = "sample" l, err = ListRegistrations(&q.Query{ PageSize: 5, diff --git a/src/pkg/scan/job.go b/src/pkg/scan/job.go index f38dc53e5..aa9919977 100644 --- a/src/pkg/scan/job.go +++ b/src/pkg/scan/job.go @@ -129,43 +129,15 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { return errors.Wrap(err, "run scan job") } - // Exit signal - exit := make(chan bool, 1) - // Done signal - done := make(chan bool, 1) - // Collect errors - eChan := make(chan error, 1) - - go func() { - defer func() { - // Done! - done <- true - }() - - for { - select { - case e := <-eChan: - if err != nil { - err = errors.Wrap(e, err.Error()) - } else { - err = e - } - case <-exit: - // Gracefully exit - return - case <-ctx.SystemContext().Done(): - // Terminated by system - return - } - } - }() + // For collecting errors + errs := make([]error, len(mimes)) // Concurrently retrieving report by different mime types wg := &sync.WaitGroup{} wg.Add(len(mimes)) - for _, mt := range mimes { - go func(m string) { + for i, mt := range mimes { + go func(i int, m string) { defer wg.Done() // Log info @@ -191,13 +163,13 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { continue } - eChan <- errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m)) + errs[i] = errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m)) return } // Make sure the data is aligned with the v1 spec. if _, err = report.ResolveData(m, []byte(rawReport)); err != nil { - eChan <- errors.Wrap(err, "scan job: resolve report data") + errs[i] = errors.Wrap(err, "scan job: resolve report data") return } @@ -222,25 +194,33 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { } // Send error and exit - eChan <- errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m)) + errs[i] = errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m)) return case <-ctx.SystemContext().Done(): // Terminated by system return case <-time.After(checkTimeout): - eChan <- errors.New("check scan report timeout") + errs[i] = errors.New("check scan report timeout") return } } - }(mt) + }(i, mt) } // Wait for all the retrieving routines are completed wg.Wait() - // Stop error collection goroutine - exit <- true - // done! - <-done + + // Merge errors + for _, e := range errs { + if e != nil { + if err != nil { + err = errors.Wrap(e, err.Error()) + } else { + err = e + } + } + } + // Log error to the job log if err != nil { myLogger.Error(err) diff --git a/src/pkg/scan/report/base_manager.go b/src/pkg/scan/report/base_manager.go new file mode 100644 index 000000000..163e2707f --- /dev/null +++ b/src/pkg/scan/report/base_manager.go @@ -0,0 +1,169 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "time" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +// basicManager is a default implementation of report manager. +type basicManager struct{} + +// NewManager news basic manager. +func NewManager() Manager { + return &basicManager{} +} + +// Create ... +func (bm *basicManager) Create(r *scan.Report) (string, error) { + // Validate report object + if r == nil { + return "", errors.New("nil scan report object") + } + + if len(r.Digest) == 0 || len(r.RegistrationUUID) == 0 || len(r.MimeType) == 0 { + return "", errors.New("malformed scan report object") + } + + // Check if there is existing report copy + // Limit only one scanning performed by a given provider on the specified artifact can be there + kws := make(map[string]interface{}, 3) + kws["digest"] = r.Digest + kws["registration_uuid"] = r.RegistrationUUID + kws["mime_type"] = []interface{}{r.MimeType} + + existingCopies, err := scan.ListReports(&q.Query{ + PageNumber: 1, + PageSize: 1, + Keywords: kws, + }) + + if err != nil { + return "", errors.Wrap(err, "check existence of report") + } + + // Delete existing copy + if len(existingCopies) > 0 { + theCopy := existingCopies[0] + + // Status conflict + theStatus := job.Status(theCopy.Status) + if theStatus.Compare(job.RunningStatus) <= 0 { + return "", errors.Errorf("conflict: a previous scanning is %s", theCopy.Status) + } + + // Otherwise it will be a completed report + // Clear it before insert this new one + if err := scan.DeleteReport(theCopy.UUID); err != nil { + return "", errors.Wrap(err, "clear old scan report") + } + } + + // Assign uuid + UUID, err := uuid.NewUUID() + if err != nil { + return "", errors.Wrap(err, "create report: new UUID") + } + r.UUID = UUID.String() + + // Fill in / override the related properties + r.StartTime = time.Now().UTC() + r.Status = job.PendingStatus.String() + r.StatusCode = job.PendingStatus.Code() + + // Insert + if _, err = scan.CreateReport(r); err != nil { + return "", errors.Wrap(err, "create report") + } + + return r.UUID, nil +} + +// GetBy ... +func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) { + if len(digest) == 0 { + return nil, errors.New("empty digest to get report data") + } + + kws := make(map[string]interface{}) + kws["digest"] = digest + if len(registrationUUID) > 0 { + kws["registration_uuid"] = registrationUUID + } + if len(mimeTypes) > 0 { + kws["mime_type"] = mimeTypes + } + // Query all + query := &q.Query{ + PageNumber: 0, + Keywords: kws, + } + + return scan.ListReports(query) +} + +// UpdateScanJobID ... +func (bm *basicManager) UpdateScanJobID(uuid string, jobID string) error { + if len(uuid) == 0 || len(jobID) == 0 { + return errors.New("bad arguments") + } + + return scan.UpdateJobID(uuid, jobID) +} + +// UpdateStatus ... +func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) error { + if len(uuid) == 0 { + return errors.New("missing uuid") + } + + if rev <= 0 { + return errors.New("invalid data revision") + } + + stCode := job.ErrorStatus.Code() + st := job.Status(status) + // Check if it is job valid status. + // Probably an error happened before submitting jobs. + if st.Code() != -1 { + // Assign error code + stCode = st.Code() + } + + return scan.UpdateReportStatus(uuid, status, stCode, rev) +} + +// UpdateReportData ... +func (bm *basicManager) UpdateReportData(uuid string, report string, rev int64) error { + if len(uuid) == 0 { + return errors.New("missing uuid") + } + + if rev <= 0 { + return errors.New("invalid data revision") + } + + if len(report) == 0 { + return errors.New("missing report JSON data") + } + + return scan.UpdateReportData(uuid, report, rev) +} diff --git a/src/pkg/scan/report/base_manager_test.go b/src/pkg/scan/report/base_manager_test.go new file mode 100644 index 000000000..e4b881e0a --- /dev/null +++ b/src/pkg/scan/report/base_manager_test.go @@ -0,0 +1,156 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// TestManagerSuite is a test suite for the report manager. +type TestManagerSuite struct { + suite.Suite + + m Manager + rpUUID string +} + +// TestManager is an entry of suite TestManagerSuite. +func TestManager(t *testing.T) { + suite.Run(t, &TestManagerSuite{}) +} + +// SetupSuite prepares test env for suite TestManagerSuite. +func (suite *TestManagerSuite) SetupSuite() { + dao.PrepareTestForPostgresSQL() + + suite.m = NewManager() +} + +// SetupTest prepares env for test cases. +func (suite *TestManagerSuite) SetupTest() { + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + } + + uuid, err := suite.m.Create(rp) + require.NoError(suite.T(), err) + require.NotEmpty(suite.T(), uuid) + suite.rpUUID = uuid +} + +// TearDownTest clears test env for test cases. +func (suite *TestManagerSuite) TearDownTest() { + // No delete method defined in manager as no requirement, + // so, to clear env, call dao method here + err := scan.DeleteReport(suite.rpUUID) + require.NoError(suite.T(), err) +} + +// TestManagerCreateWithExisting tests the case that a copy already is there when creating report. +func (suite *TestManagerSuite) TestManagerCreateWithExisting() { + err := suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 2000) + require.NoError(suite.T(), err) + + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + } + + uuid, err := suite.m.Create(rp) + require.NoError(suite.T(), err) + require.NotEmpty(suite.T(), uuid) + + assert.NotEqual(suite.T(), suite.rpUUID, uuid) + suite.rpUUID = uuid +} + +// TestManagerGetBy tests the get by method. +func (suite *TestManagerSuite) TestManagerGetBy() { + l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + assert.Equal(suite.T(), suite.rpUUID, l[0].UUID) + + l, err = suite.m.GetBy("d1000", "ruuid", nil) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + assert.Equal(suite.T(), suite.rpUUID, l[0].UUID) + + l, err = suite.m.GetBy("d1000", "", nil) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + assert.Equal(suite.T(), suite.rpUUID, l[0].UUID) +} + +// TestManagerUpdateJobID tests update job ID method. +func (suite *TestManagerSuite) TestManagerUpdateJobID() { + l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + oldJID := l[0].JobID + + err = suite.m.UpdateScanJobID(suite.rpUUID, "jID1001") + require.NoError(suite.T(), err) + + l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + assert.NotEqual(suite.T(), oldJID, l[0].JobID) + assert.Equal(suite.T(), "jID1001", l[0].JobID) +} + +// TestManagerUpdateStatus tests update status method +func (suite *TestManagerSuite) TestManagerUpdateStatus() { + l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + oldSt := l[0].Status + + err = suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 10000) + require.NoError(suite.T(), err) + + l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + assert.NotEqual(suite.T(), oldSt, l[0].Status) + assert.Equal(suite.T(), job.SuccessStatus.String(), l[0].Status) +} + +// TestManagerUpdateReportData tests update job report data. +func (suite *TestManagerSuite) TestManagerUpdateReportData() { + err := suite.m.UpdateReportData(suite.rpUUID, "{\"a\":1000}", 1000) + require.NoError(suite.T(), err) + + l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + assert.Equal(suite.T(), "{\"a\":1000}", l[0].Report) +} diff --git a/src/pkg/scan/report/manager.go b/src/pkg/scan/report/manager.go new file mode 100644 index 000000000..4c4ca13a1 --- /dev/null +++ b/src/pkg/scan/report/manager.go @@ -0,0 +1,80 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + +// Manager is used to manage the scan reports. +type Manager interface { + // Create a new report record. + // + // Arguments: + // r *scan.Report : report model to be created + // + // Returns: + // string : uuid of the new report + // error : non nil error if any errors occurred + // + Create(r *scan.Report) (string, error) + + // Update the scan job ID of the given report. + // + // Arguments: + // uuid string : uuid to identify the report + // jobID string: scan job ID + // + // Returns: + // error : non nil error if any errors occurred + // + UpdateScanJobID(uuid string, jobID string) error + + // Update the status (mapping to the scan job status) of the given report. + // + // Arguments: + // uuid string : uuid to identify the report + // status string: status info + // rev int64 : data revision info + // + // Returns: + // error : non nil error if any errors occurred + // + UpdateStatus(uuid string, status string, rev int64) error + + // Update the report data (with JSON format) of the given report. + // + // Arguments: + // uuid string : uuid to identify the report + // report string: report JSON data + // rev int64 : data revision info + // + // Returns: + // error : non nil error if any errors occurred + // + UpdateReportData(uuid string, report string, rev int64) error + + // Get the reports for the given digest by other properties. + // + // Arguments: + // digest string : digest of the artifact + // registrationUUID string : [optional] the report generated by which registration. + // If it is empty, reports by all the registrations are retrieved. + // mimeTypes []string : [optional] mime types of the reports requiring + // If empty array is specified, reports with all the supported mimes are retrieved. + // + // Returns: + // []*scan.Report : report list + // error : non nil error if any errors occurred + GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) +} diff --git a/src/pkg/scan/rest/auth/bearer_auth.go b/src/pkg/scan/rest/auth/bearer_auth.go index d7b8f383c..a21eb1117 100644 --- a/src/pkg/scan/rest/auth/bearer_auth.go +++ b/src/pkg/scan/rest/auth/bearer_auth.go @@ -17,6 +17,8 @@ package auth import ( "fmt" "net/http" + + "github.com/pkg/errors" ) // bearerAuthorizer authorizes the request by adding `Authorization Bearer credential` header @@ -31,7 +33,7 @@ func (ba *bearerAuthorizer) Authorize(req *http.Request) error { req.Header.Add(authorization, fmt.Sprintf("%s %s", ba.typeID, ba.accessCred)) } - return nil + return errors.Errorf("%s: %s", ba.typeID, "missing data to authorize request") } // NewBearerAuth create bearer authorizer diff --git a/src/pkg/scan/rest/v1/client_pool.go b/src/pkg/scan/rest/v1/client_pool.go index 0084b0d76..bf1dc3aa2 100644 --- a/src/pkg/scan/rest/v1/client_pool.go +++ b/src/pkg/scan/rest/v1/client_pool.go @@ -15,16 +15,24 @@ package v1 import ( - "encoding/base64" "fmt" + "os" + "os/signal" "sync" + "syscall" + "time" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/pkg/errors" ) +const ( + defaultDeadCheckInterval = 1 * time.Minute + defaultExpireTime = 5 * time.Minute +) + // DefaultClientPool is a default client pool. -var DefaultClientPool = NewClientPool() +var DefaultClientPool = NewClientPool(nil) // ClientPool defines operations for the client pool which provides v1 client cache. type ClientPool interface { @@ -39,16 +47,47 @@ type ClientPool interface { Get(r *scanner.Registration) (Client, error) } +// PoolConfig provides configurations for the client pool. +type PoolConfig struct { + // Interval for checking dead instance. + DeadCheckInterval time.Duration + // Expire time for the instance to be marked as dead. + ExpireTime time.Duration +} + +// poolItem append timestamp for the caching client instance. +type poolItem struct { + c Client + timestamp time.Time +} + // basicClientPool is default implementation of client pool interface. type basicClientPool struct { - pool *sync.Map + pool *sync.Map + config *PoolConfig } // NewClientPool news a basic client pool. -func NewClientPool() ClientPool { - return &basicClientPool{ - pool: &sync.Map{}, +func NewClientPool(config *PoolConfig) ClientPool { + bcp := &basicClientPool{ + pool: &sync.Map{}, + config: config, } + + // Set config + if bcp.config == nil { + bcp.config = &PoolConfig{} + } + + if bcp.config.DeadCheckInterval == 0 { + bcp.config.DeadCheckInterval = defaultDeadCheckInterval + } + + if bcp.config.ExpireTime == 0 { + bcp.config.ExpireTime = defaultExpireTime + } + + return bcp } // Get client for the specified registration. @@ -57,38 +96,7 @@ func NewClientPool() ClientPool { // If one day, we have to clear unactivated client instances in the pool, // add the following func after the first time initializing the client. // pool item represents the client with a timestamp of last accessed. -// -// type poolItem struct { -// c Client -// timestamp time.Time -// } -// -// func (bcp *basicClientPool) deadCheck(key string, item *poolItem) { -// // Run in a separate goroutine -// go func() { -// // As we do not have a global context, let's watch the system signal to -// // exit the goroutine correctly. -// sig := make(chan os.Signal, 1) -// signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill) -// -// tk := time.NewTicker(bcp.config.DeadCheckInterval) -// defer tk.Stop() -// -// for { -// select { -// case t := <-tk.C: -// if item.timestamp.Add(bcp.config.ExpireTime).Before(t.UTC()) { -// // Expired -// bcp.pool.Delete(key) -// return -// } -// case <-sig: -// // Terminated by system -// return -// } -// } -// }() -// } + func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) { if r == nil { return nil, errors.New("nil scanner registration") @@ -100,7 +108,7 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) { k := key(r) - c, ok := bcp.pool.Load(k) + item, ok := bcp.pool.Load(k) if !ok { nc, err := NewClient(r) if err != nil { @@ -108,21 +116,54 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) { } // Cache it - bcp.pool.Store(k, nc) - c = nc + npi := &poolItem{ + c: nc, + timestamp: time.Now().UTC(), + } + + bcp.pool.Store(k, npi) + item = npi + + // dead check + bcp.deadCheck(k, npi) } - return c.(Client), nil + return item.(*poolItem).c, nil +} + +func (bcp *basicClientPool) deadCheck(key string, item *poolItem) { + // Run in a separate goroutine + go func() { + // As we do not have a global context, let's watch the system signal to + // exit the goroutine correctly. + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill) + + tk := time.NewTicker(bcp.config.DeadCheckInterval) + defer tk.Stop() + + for { + select { + case t := <-tk.C: + if item.timestamp.Add(bcp.config.ExpireTime).Before(t.UTC()) { + // Expired + bcp.pool.Delete(key) + return + } + case <-sig: + // Terminated by system + return + } + } + }() } func key(r *scanner.Registration) string { - raw := fmt.Sprintf("%s:%s:%s:%s:%v", + return fmt.Sprintf("%s:%s:%s:%s:%v", r.UUID, r.URL, r.Auth, r.AccessCredential, r.SkipCertVerify, ) - - return base64.StdEncoding.EncodeToString([]byte(raw)) } diff --git a/src/pkg/scan/rest/v1/client_pool_test.go b/src/pkg/scan/rest/v1/client_pool_test.go index 95e316d41..9666f4067 100644 --- a/src/pkg/scan/rest/v1/client_pool_test.go +++ b/src/pkg/scan/rest/v1/client_pool_test.go @@ -17,6 +17,7 @@ package v1 import ( "fmt" "testing" + "time" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/goharbor/harbor/src/pkg/scan/rest/auth" @@ -34,12 +35,16 @@ type ClientPoolTestSuite struct { // TestClientPool is the entry of ClientPoolTestSuite. func TestClientPool(t *testing.T) { - suite.Run(t, new(ClientPoolTestSuite)) + suite.Run(t, &ClientPoolTestSuite{}) } // SetupSuite sets up test suite env. func (suite *ClientPoolTestSuite) SetupSuite() { - suite.pool = NewClientPool() + cfg := &PoolConfig{ + DeadCheckInterval: 100 * time.Millisecond, + ExpireTime: 300 * time.Millisecond, + } + suite.pool = NewClientPool(cfg) } // TestClientPoolGet tests the get method of client pool. @@ -66,4 +71,12 @@ func (suite *ClientPoolTestSuite) TestClientPoolGet() { p2 := fmt.Sprintf("%p", client2.(*basicClient)) assert.Equal(suite.T(), p1, p2) + + <-time.After(400 * time.Millisecond) + client3, err := suite.pool.Get(r) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), client3) + + p3 := fmt.Sprintf("%p", client3.(*basicClient)) + assert.NotEqual(suite.T(), p2, p3) } diff --git a/src/pkg/scan/rest/v1/spec.go b/src/pkg/scan/rest/v1/spec.go index d03ddc29b..6d4f6bf0e 100644 --- a/src/pkg/scan/rest/v1/spec.go +++ b/src/pkg/scan/rest/v1/spec.go @@ -60,19 +60,17 @@ type Spec struct { // NewSpec news V1 spec func NewSpec(base string) *Spec { - baseRoute := "http://localhost" + s := &Spec{} if len(base) > 0 { if strings.HasSuffix(base, "/") { - baseRoute = base[:len(base)-1] + s.baseRoute = base[:len(base)-1] } else { - baseRoute = base + s.baseRoute = base } } - return &Spec{ - baseRoute: baseRoute, - } + return s } // Metadata API diff --git a/src/pkg/scan/scanner/manager_test.go b/src/pkg/scan/scanner/manager_test.go index 169a216f0..6f8a485d2 100644 --- a/src/pkg/scan/scanner/manager_test.go +++ b/src/pkg/scan/scanner/manager_test.go @@ -63,7 +63,7 @@ func (suite *BasicManagerTestSuite) TearDownSuite() { // TestList tests list registrations func (suite *BasicManagerTestSuite) TestList() { - m := make(map[string]string, 1) + m := make(map[string]interface{}, 1) m["name"] = "forUT" l, err := suite.mgr.List(&q.Query{