upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2024-09-20 14:45:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13910 from mmpei/official-feature-artifact-hub-replicate

Browse Source 
Support artifact hub replication using new API
This commit is contained in:
Wenkai Yin(尹文开) 2021-01-08 10:15:55 +08:00 committed by GitHub
commit 4580aeff3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 139 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/replication/adapter/adapter.go
View File

@ -68,7 +68,7 @@ type ArtifactRegistry interface {
type ChartRegistry interface {
FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
ChartExist(name, version string) (bool, error)
DownloadChart(name, version string) (io.ReadCloser, error)
DownloadChart(name, version, contentURL string) (io.ReadCloser, error)
UploadChart(name, version string, chart io.Reader) error
DeleteChart(name, version string) error
}

6
src/replication/adapter/artifacthub/adapter_test.go
View File

@ -82,7 +82,11 @@ func TestAdapter_DownloadChart(t *testing.T) {
URL: "https://artifacthub.io",
})
data, err := a.DownloadChart("harbor/harbor", "1.5.0")
data, err := a.DownloadChart("harbor/harbor", "1.5.0", "")
assert.NotNil(t, err)
assert.Nil(t, data)
data, err = a.DownloadChart("harbor/harbor", "1.5.0", "https://helm.goharbor.io/harbor-1.5.0.tgz")
assert.Nil(t, err)
assert.NotNil(t, data)
}

104
src/replication/adapter/artifacthub/chart_registry.go
View File

@ -17,7 +17,6 @@ package artifacthub
import (
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model"
"io"
@ -26,62 +25,92 @@ import (
)
func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
pkgs, err := a.client.getAllPackages(HelmChart)
charts, err := a.client.getReplicationInfo()
if err != nil {
return nil, err
}
resources := []*model.Resource{}
var repositories []*model.Repository
for _, pkg := range pkgs {
repositories = append(repositories, &model.Repository{
Name: fmt.Sprintf("%s/%s", pkg.Repository.Name, pkg.Name),
})
var artifacts []*model.Artifact
repoSet := map[string]interface{}{}
versionSet := map[string]interface{}{}
for _, chart := range charts {
name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package)
if _, ok := repoSet[name]; !ok {
repoSet[name] = nil
repositories = append(repositories, &model.Repository{
Name: name,
})
}
}
repositories, err = filter.DoFilterRepositories(repositories, filters)
if err != nil {
return nil, err
}
if len(repositories) == 0 {
return resources, nil
}
for _, repository := range repositories {
pkgDetail, err := a.client.getHelmPackageDetail(repository.Name)
if err != nil {
log.Errorf("fetch package detail: %v", err)
return nil, err
if len(repoSet) != len(repositories) {
repoSet = map[string]interface{}{}
for _, repo := range repositories {
repoSet[repo.Name] = nil
}
}
var artifacts []*model.Artifact
for _, version := range pkgDetail.AvailableVersions {
for _, chart := range charts {
name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package)
if _, ok := repoSet[name]; ok {
artifacts = append(artifacts, &model.Artifact{
Tags: []string{version.Version},
Tags: []string{chart.Version},
})
}
}
artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
if err != nil {
return nil, err
}
if len(artifacts) == 0 {
artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
if err != nil {
return nil, err
}
if len(artifacts) == 0 {
return resources, nil
}
for _, arti := range artifacts {
versionSet[arti.Tags[0]] = nil
}
for _, chart := range charts {
name := fmt.Sprintf("%s/%s", chart.Repository, chart.Package)
if _, ok := repoSet[name]; !ok {
continue
}
for _, artifact := range artifacts {
resources = append(resources, &model.Resource{
Type: model.ResourceTypeChart,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository.Name,
},
Artifacts: []*model.Artifact{artifact},
},
})
if _, ok := versionSet[chart.Version]; !ok {
continue
}
resources = append(resources, &model.Resource{
Type: model.ResourceTypeChart,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: name,
},
Artifacts: []*model.Artifact{
{
Tags: []string{chart.Version},
},
},
},
ExtendedInfo: map[string]interface{}{
"contentURL": chart.ContentURL,
},
})
}
return resources, nil
}
// ChartExist will never be used, for this function is only used when endpoint is destination
func (a *adapter) ChartExist(name, version string) (bool, error) {
_, err := a.client.getHelmChartVersion(name, version)
if err != nil {
@ -93,16 +122,11 @@ func (a *adapter) ChartExist(name, version string) (bool, error) {
return true, nil
}
func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
chartVersion, err := a.client.getHelmChartVersion(name, version)
if err != nil {
return nil, err
func (a *adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
if len(contentURL) == 0 {
return nil, errors.Errorf("empty chart content url, %s:%s", name, version)
}
if len(chartVersion.ContentURL) == 0 {
return nil, errors.Errorf("")
}
return a.download(chartVersion.ContentURL)
return a.download(contentURL)
}
func (a *adapter) download(contentURL string) (io.ReadCloser, error) {

132
src/replication/adapter/artifacthub/client.go
View File

@ -24,101 +24,6 @@ func newClient(registry *model.Registry) *Client {
}
}
// searchPackages query the artifact package list from artifact hub.
func (c *Client) searchPackages(kind, offset, limit int, queryString string) (*PackageResponse, error) {
request, err := http.NewRequest(http.MethodGet, baseURL+searchPackages(kind, offset, limit, queryString), nil)
if err != nil {
return nil, err
}
resp, err := c.do(request)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
msg := &Message{}
err = json.Unmarshal(body, msg)
if err != nil {
msg.Message = string(body)
}
return nil, fmt.Errorf("search package list error %d: %s", resp.StatusCode, msg.Message)
}
packageResp := &PackageResponse{}
err = json.Unmarshal(body, packageResp)
if err != nil {
return nil, fmt.Errorf("unmarshal package list response error: %v", err)
}
return packageResp, nil
}
// getAllPackages gets all of the specific kind of artifact packages from artifact hub.
func (c *Client) getAllPackages(kind int) (pkgs []*Package, err error) {
offset := 0
limit := 50
shouldContinue := true
// todo: rate limit
for shouldContinue {
pkgResp, err := c.searchPackages(HelmChart, offset, limit, "")
if err != nil {
return nil, err
}
pkgs = append(pkgs, pkgResp.Data.Packages...)
total := pkgResp.Metadata.Total
offset = offset + limit
if offset >= total {
shouldContinue = false
}
}
return pkgs, nil
}
// getHelmPackageDetail get the chart detail of a helm chart from artifact hub.
func (c *Client) getHelmPackageDetail(fullName string) (*PackageDetail, error) {
request, err := http.NewRequest(http.MethodGet, baseURL+getHelmPackageDetail(fullName), nil)
if err != nil {
return nil, err
}
resp, err := c.do(request)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusNotFound {
return nil, ErrHTTPNotFound
} else if resp.StatusCode != http.StatusOK {
msg := &Message{}
err = json.Unmarshal(body, msg)
if err != nil {
msg.Message = string(body)
}
return nil, fmt.Errorf("fetch package detail error %d: %s", resp.StatusCode, msg.Message)
}
pkgDetail := &PackageDetail{}
err = json.Unmarshal(body, pkgDetail)
if err != nil {
return nil, fmt.Errorf("unmarshal package detail response error: %v", err)
}
return pkgDetail, nil
}
// getHelmVersion get the package version of a helm chart from artifact hub.
func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, error) {
request, err := http.NewRequest(http.MethodGet, baseURL+getHelmVersion(fullName, version), nil)
@ -157,6 +62,43 @@ func (c *Client) getHelmChartVersion(fullName, version string) (*ChartVersion, e
return chartVersion, nil
}
// getReplicationInfo gets the brief info of all helm chart from artifact hub.
// see https://github.com/artifacthub/hub/issues/997
func (c *Client) getReplicationInfo() ([]*ChartInfo, error) {
request, err := http.NewRequest(http.MethodGet, baseURL+getReplicationInfo, nil)
if err != nil {
return nil, err
}
resp, err := c.do(request)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
msg := &Message{}
err = json.Unmarshal(body, msg)
if err != nil {
msg.Message = string(body)
}
return nil, fmt.Errorf("get chart replication info error %d: %s", resp.StatusCode, msg.Message)
}
var chartInfo []*ChartInfo
err = json.Unmarshal(body, &chartInfo)
if err != nil {
return nil, fmt.Errorf("unmarshal chart replication info error: %v", err)
}
return chartInfo, nil
}
func (c *Client) checkHealthy() error {
request, err := http.NewRequest(http.MethodGet, baseURL, nil)
if err != nil {

3
src/replication/adapter/artifacthub/consts.go
View File

@ -6,7 +6,8 @@ import (
)
const (
baseURL = "https://artifacthub.io"
baseURL = "https://artifacthub.io"
getReplicationInfo = "/api/v1/harborReplication"
)
const (

16
src/replication/adapter/artifacthub/model.go
View File

@ -13,9 +13,10 @@ type PackageData struct {
// Package ...
type Package struct {
PackageID string `json:"package_id"`
Name string `json:"name"`
Repository *Repository `json:"repository"`
PackageID string `json:"package_id"`
Name string `json:"name"`
NormalizedName string `json:"normalized_name"`
Repository *Repository `json:"repository"`
}
// Repository ...
@ -29,6 +30,7 @@ type Repository struct {
type PackageDetail struct {
PackageID string `json:"package_id"`
Name string `json:"name"`
NormalizedName string `json:"normalized_name"`
Version string `json:"version"`
AppVersion string `json:"app_version"`
Repository RepositoryDetail `json:"repository"`
@ -70,3 +72,11 @@ type Metadata struct {
type Message struct {
Message string `json:"message"`
}
// ChartInfo ...
type ChartInfo struct {
Repository string `json:"repository"`
Package string `json:"package"`
Version string `json:"version"`
ContentURL string `json:"url"`
}

2
src/replication/adapter/harbor/base/chart_registry.go
View File

@ -144,7 +144,7 @@ func (a *Adapter) getChartInfo(name, version string) (*chartVersionDetail, error
}
// DownloadChart downloads the specific chart
func (a *Adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
func (a *Adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
info, err := a.getChartInfo(name, version)
if err != nil {
return nil, err

2
src/replication/adapter/harbor/base/chart_registry_test.go
View File

@ -149,7 +149,7 @@ func TestDownloadChart(t *testing.T) {
}
adapter, err := New(registry)
require.Nil(t, err)
_, err = adapter.DownloadChart("library/harbor", "1.0")
_, err = adapter.DownloadChart("library/harbor", "1.0", "")
require.Nil(t, err)
}

2
src/replication/adapter/helmhub/chart_registry.go
View File

@ -101,7 +101,7 @@ func (a *adapter) ChartExist(name, version string) (bool, error) {
return false, nil
}
func (a *adapter) DownloadChart(name, version string) (io.ReadCloser, error) {
func (a *adapter) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
versionList, err := a.client.fetchChartDetail(name)
if err != nil {
return nil, err

2
src/replication/adapter/tencentcr/chart_registry.go
View File

@ -146,7 +146,7 @@ func (a *adapter) getChartInfo(name, version string) (info *tcrChartVersionDetai
return
}
func (a *adapter) DownloadChart(name, version string) (rc io.ReadCloser, err error) {
func (a *adapter) DownloadChart(name, version, contentURL string) (rc io.ReadCloser, err error) {
var info *tcrChartVersionDetail
info, err = a.getChartInfo(name, version)
if err != nil {

17
src/replication/transfer/chart/transfer.go
View File

@ -37,8 +37,9 @@ func factory(logger trans.Logger, stopFunc trans.StopFunc) (trans.Transfer, erro
}
type chart struct {
name string
version string
name string
version string
contentURL string
}
type transfer struct {
@ -62,9 +63,15 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
})
}
var contentURL string
if len(src.ExtendedInfo) > 0 {
contentURL = src.ExtendedInfo["contentURL"].(string)
}
srcChart := &chart{
name: src.Metadata.Repository.Name,
version: src.Metadata.Artifacts[0].Tags[0],
name: src.Metadata.Repository.Name,
version: src.Metadata.Artifacts[0].Tags[0],
contentURL: contentURL,
}
dstChart := &chart{
name: dst.Metadata.Repository.Name,
@ -151,7 +158,7 @@ func (t *transfer) copy(src, dst *chart, override bool) error {
}
// copy the chart between the source and destination registries
chart, err := t.src.DownloadChart(src.name, src.version)
chart, err := t.src.DownloadChart(src.name, src.version, src.contentURL)
if err != nil {
t.logger.Errorf("failed to download the chart %s:%s: %v", src.name, src.version, err)
return err

2
src/replication/transfer/chart/transfer_test.go
View File

@ -45,7 +45,7 @@ func (f *fakeRegistry) FetchCharts(filters []*model.Filter) ([]*model.Resource,
func (f *fakeRegistry) ChartExist(name, version string) (bool, error) {
return true, nil
}
func (f *fakeRegistry) DownloadChart(name, version string) (io.ReadCloser, error) {
func (f *fakeRegistry) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
r := ioutil.NopCloser(bytes.NewReader([]byte{'a'}))
return r, nil
}