Merge artifactInfo and ManifestInfo

This commit gets rid of middleware info middleware, and make artifact
info the single source of truth in terms of the artifact a request
handles.  Fixes #10574

Signed-off-by: Daniel Jiang <jiangd@vmware.com>
This commit is contained in:
Daniel Jiang 2020-02-23 03:23:19 +08:00
parent 1765abc985
commit 46c72ae372
13 changed files with 91 additions and 288 deletions

View File

@ -33,6 +33,7 @@ const (
blobFromQuery = "from"
blobMountDigest = "blob_mount_digest"
blobMountRepo = "blob_mount_repo"
tag = "tag"
)
var (
@ -70,7 +71,9 @@ func Middleware() func(http.Handler) http.Handler {
if ref, ok := m[middleware.ReferenceSubexp]; ok {
art.Reference = ref
}
if t, ok := m[tag]; ok {
art.Tag = t
}
if bmr, ok := m[blobMountRepo]; ok {
// Fail early for now, though in docker registry an invalid may return 202
// it's not clear in OCI spec how to handle invalid from parm
@ -118,6 +121,8 @@ func parse(url *url.URL) (map[string]string, bool) {
}
if digest.DigestRegexp.MatchString(m[middleware.ReferenceSubexp]) {
m[middleware.DigestSubexp] = m[middleware.ReferenceSubexp]
} else if ref, ok := m[middleware.ReferenceSubexp]; ok {
m[tag] = ref
}
return m, match
}

View File

@ -137,6 +137,7 @@ func TestPopulateArtifactInfo(t *testing.T) {
Repository: "library/hello-world",
Reference: "latest",
ProjectName: "library",
Tag: "latest",
},
},
{
@ -176,7 +177,7 @@ func TestPopulateArtifactInfo(t *testing.T) {
if tt.art != nil {
a, ok := middleware.ArtifactInfoFromContext(next.ctx)
assert.True(t, ok)
assert.Equal(t, *tt.art, *a)
assert.Equal(t, *tt.art, a)
}
}
}

View File

@ -43,54 +43,54 @@ func Middleware() func(http.Handler) http.Handler {
}
}
func validate(req *http.Request) (bool, *middleware.ManifestInfo) {
mf, ok := middleware.ManifestInfoFromContext(req.Context())
if !ok {
return false, nil
func validate(req *http.Request) (bool, middleware.ArtifactInfo) {
none := middleware.ArtifactInfo{}
if err := middleware.EnsureArtifactDigest(req.Context()); err != nil {
return false, none
}
_, err := mf.ManifestExists(req.Context())
if err != nil {
return false, mf
af, ok := middleware.ArtifactInfoFromContext(req.Context())
if !ok {
return false, none
}
if scannerPull, ok := middleware.ScannerPullFromContext(req.Context()); ok && scannerPull {
return false, mf
return false, none
}
if !middleware.GetPolicyChecker().ContentTrustEnabled(mf.ProjectName) {
return false, mf
if !middleware.GetPolicyChecker().ContentTrustEnabled(af.ProjectName) {
return false, af
}
return true, mf
return true, af
}
func matchNotaryDigest(mf *middleware.ManifestInfo) (bool, error) {
func matchNotaryDigest(af middleware.ArtifactInfo) (bool, error) {
if NotaryEndpoint == "" {
NotaryEndpoint = config.InternalNotaryEndpoint()
}
targets, err := notary.GetInternalTargets(NotaryEndpoint, util.TokenUsername, mf.Repository)
targets, err := notary.GetInternalTargets(NotaryEndpoint, util.TokenUsername, af.Repository)
if err != nil {
return false, err
}
for _, t := range targets {
if mf.Digest != "" {
if af.Digest != "" {
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if mf.Digest == d {
if af.Digest == d {
return true, nil
}
} else {
if t.Tag == mf.Tag {
log.Debugf("found reference: %s in notary, try to match digest.", mf.Tag)
if t.Tag == af.Tag {
log.Debugf("found reference: %s in notary, try to match digest.", af.Tag)
d, err := notary.DigestFromTarget(t)
if err != nil {
return false, err
}
if mf.Digest == d {
if af.Digest == d {
return true, nil
}
}
}
}
log.Debugf("image: %#v, not found in notary", mf)
log.Debugf("image: %#v, not found in notary", af)
return false, nil
}

View File

@ -33,12 +33,12 @@ func MiddlewareDelete() func(http.Handler) http.Handler {
// handleDelete ...
func handleDelete(req *http.Request) error {
mf, ok := middleware.ManifestInfoFromContext(req.Context())
art, ok := middleware.ArtifactInfoFromContext(req.Context())
if !ok {
return errors.New("cannot get the manifest information from request context")
}
af, err := artifact.Ctl.GetByReference(req.Context(), mf.Repository, mf.Digest, &artifact.Option{
af, err := artifact.Ctl.GetByReference(req.Context(), art.Repository, art.Digest, &artifact.Option{
WithTag: true,
TagOption: &artifact.TagOption{WithImmutableStatus: true},
})
@ -49,7 +49,7 @@ func handleDelete(req *http.Request) error {
return err
}
_, repoName := common_util.ParseRepository(mf.Repository)
_, repoName := common_util.ParseRepository(art.Repository)
for _, tag := range af.Tags {
if tag.Immutable {
return NewErrImmutable(repoName, tag.Name)

View File

@ -35,12 +35,12 @@ func MiddlewarePush() func(http.Handler) http.Handler {
// If the pushing image matched by any of immutable rule, will have to whether it is the first time to push it,
// as the immutable rule only impacts the existing tag.
func handlePush(req *http.Request) error {
mf, ok := middleware.ManifestInfoFromContext(req.Context())
art, ok := middleware.ArtifactInfoFromContext(req.Context())
if !ok {
return errors.New("cannot get the manifest information from request context")
}
af, err := artifact.Ctl.GetByReference(req.Context(), mf.Repository, mf.Tag, &artifact.Option{
af, err := artifact.Ctl.GetByReference(req.Context(), art.Repository, art.Tag, &artifact.Option{
WithTag: true,
TagOption: &artifact.TagOption{WithImmutableStatus: true},
})
@ -51,11 +51,11 @@ func handlePush(req *http.Request) error {
return err
}
_, repoName := common_util.ParseRepository(mf.Repository)
_, repoName := common_util.ParseRepository(art.Repository)
for _, tag := range af.Tags {
// push a existing immutable tag, reject th e request
if tag.Name == mf.Tag && tag.Immutable {
return NewErrImmutable(repoName, mf.Tag)
if tag.Name == art.Tag && tag.Immutable {
return NewErrImmutable(repoName, art.Tag)
}
}

View File

@ -36,11 +36,11 @@ func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, n
url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
req, _ := http.NewRequest("PUT", url, nil)
mfInfo := &middleware.ManifestInfo{
ProjectID: projectID,
Repository: repository,
Tag: tag,
Digest: dgt,
afInfo := &middleware.ArtifactInfo{
ProjectName: projectName,
Repository: repository,
Tag: tag,
Digest: dgt,
}
rr := httptest.NewRecorder()
@ -53,7 +53,7 @@ func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, n
}
}
*req = *(req.WithContext(internal_orm.NewContext(context.TODO(), dao.GetOrmer())))
*req = *(req.WithContext(middleware.NewManifestInfoContext(req.Context(), mfInfo)))
*req = *(req.WithContext(context.WithValue(req.Context(), middleware.ArtifactInfoKey, afInfo)))
h := MiddlewarePush()(n)
h.ServeHTTP(util.NewCustomResponseWriter(rr), req)
@ -66,11 +66,11 @@ func doDeleteManifestRequest(projectID int64, projectName, name, tag, dgt string
url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
req, _ := http.NewRequest("DELETE", url, nil)
mfInfo := &middleware.ManifestInfo{
ProjectID: projectID,
Repository: repository,
Tag: tag,
Digest: dgt,
afInfo := &middleware.ArtifactInfo{
ProjectName: projectName,
Repository: repository,
Tag: tag,
Digest: dgt,
}
rr := httptest.NewRecorder()
@ -83,7 +83,7 @@ func doDeleteManifestRequest(projectID int64, projectName, name, tag, dgt string
}
}
*req = *(req.WithContext(internal_orm.NewContext(context.TODO(), dao.GetOrmer())))
*req = *(req.WithContext(middleware.NewManifestInfoContext(req.Context(), mfInfo)))
*req = *(req.WithContext(context.WithValue(req.Context(), middleware.ArtifactInfoKey, afInfo)))
h := MiddlewareDelete()(n)
h.ServeHTTP(util.NewCustomResponseWriter(rr), req)

View File

@ -1,75 +0,0 @@
package manifestinfo
import (
"fmt"
"github.com/goharbor/harbor/src/common/utils"
ierror "github.com/goharbor/harbor/src/internal/error"
project2 "github.com/goharbor/harbor/src/pkg/project"
serror "github.com/goharbor/harbor/src/server/error"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/opencontainers/go-digest"
"net/http"
"regexp"
"strings"
)
var (
manifestURLRe = regexp.MustCompile(`^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`)
)
// Middleware gets the manifest information from request and inject it into the context
func Middleware() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
mf, err := parseManifestInfoFromPath(req)
if err != nil {
serror.SendError(rw, err)
return
}
*req = *(req.WithContext(middleware.NewManifestInfoContext(req.Context(), mf)))
next.ServeHTTP(rw, req)
})
}
}
// parseManifestInfoFromPath parse manifest from request path
func parseManifestInfoFromPath(req *http.Request) (*middleware.ManifestInfo, error) {
match, repository, reference := MatchManifestURL(req)
if !match {
return nil, fmt.Errorf("not match url %s for manifest", req.URL.Path)
}
projectName, _ := utils.ParseRepository(repository)
project, err := project2.Mgr.Get(projectName)
if err != nil {
return nil, fmt.Errorf("failed to get project %s, error: %v", projectName, err)
}
if project == nil {
return nil, ierror.NotFoundError(nil).WithMessage("project %s not found", projectName)
}
info := &middleware.ManifestInfo{
ProjectID: project.ProjectID,
ProjectName: projectName,
Repository: repository,
}
dgt, err := digest.Parse(reference)
if err != nil {
info.Tag = reference
} else {
info.Digest = dgt.String()
}
return info, nil
}
// MatchManifestURL ...
func MatchManifestURL(req *http.Request) (bool, string, string) {
s := manifestURLRe.FindStringSubmatch(req.URL.Path)
if len(s) == 3 {
s[1] = strings.TrimSuffix(s[1], "/")
return true, s[1], s[2]
}
return false, "", ""
}

View File

@ -1,104 +0,0 @@
package manifestinfo
import (
"fmt"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"net/http"
"net/http/httptest"
"reflect"
)
type mfinfoTestSuite struct {
suite.Suite
require *require.Assertions
assert *assert.Assertions
}
func (t *mfinfoTestSuite) SetupSuite() {
t.require = require.New(t.T())
t.assert = assert.New(t.T())
test.InitDatabaseFromEnv()
}
func (t *mfinfoTestSuite) TestParseManifestInfoFromPath() {
mustRequest := func(method, url string) *http.Request {
req, _ := http.NewRequest(method, url, nil)
return req
}
type args struct {
req *http.Request
}
tests := []struct {
name string
args args
want *middleware.ManifestInfo
wantErr bool
}{
{
"ok for digest",
args{mustRequest(http.MethodDelete, "/v2/library/photon/manifests/sha256:3e17b60ab9d92d953fb8ebefa25624c0d23fb95f78dde5572285d10158044059")},
&middleware.ManifestInfo{
ProjectID: 1,
Repository: "library/photon",
Digest: "sha256:3e17b60ab9d92d953fb8ebefa25624c0d23fb95f78dde5572285d10158044059",
},
false,
},
{
"ok for tag",
args{mustRequest(http.MethodDelete, "/v2/library/photon/manifests/latest")},
&middleware.ManifestInfo{
ProjectID: 1,
Repository: "library/photon",
Tag: "latest",
},
false,
},
{
"project not found",
args{mustRequest(http.MethodDelete, "/v2/notfound/photon/manifests/latest")},
nil,
true,
},
{
"url not match",
args{mustRequest(http.MethodDelete, "/v2/library/photon/manifest/latest")},
nil,
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func() {
got, err := parseManifestInfoFromPath(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf(err, fmt.Sprintf("ParseManifestInfoFromPath() error = %v, wantErr %v", err, tt.wantErr))
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf(err, fmt.Sprintf("ParseManifestInfoFromPath() = %v, want %v", got, tt.want))
}
})
}
}
func (t *mfinfoTestSuite) TestResolveManifest() {
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
})
req := httptest.NewRequest(http.MethodDelete, "/v2/library/hello-world/manifests/latest", nil)
rec := httptest.NewRecorder()
Middleware()(next).ServeHTTP(rec, req)
t.assert.Equal(rec.Code, http.StatusOK)
mf, ok := middleware.ManifestInfoFromContext(req.Context())
t.assert.True(ok)
t.assert.Equal(mf.Tag, "latest")
t.assert.Equal(mf.ProjectID, int64(1))
}

View File

@ -28,7 +28,7 @@ func Middleware() func(http.Handler) http.Handler {
}
func parseToken(req *http.Request) error {
mf, ok := middleware.ManifestInfoFromContext(req.Context())
art, ok := middleware.ArtifactInfoFromContext(req.Context())
if !ok {
return errors.New("cannot get the manifest information from request context")
}
@ -50,7 +50,7 @@ func parseToken(req *http.Request) error {
accessItems = append(accessItems, auth.Access{
Resource: auth.Resource{
Type: rbac.ResourceRepository.String(),
Name: mf.Repository,
Name: art.Repository,
},
Action: rbac.ActionScannerPull.String(),
})

View File

@ -1,6 +1,7 @@
package regtoken
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/goharbor/harbor/src/server/middleware"
@ -25,9 +26,9 @@ func doPullManifestRequest(projectName, name, tag string, next ...http.HandlerFu
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
rr := httptest.NewRecorder()
mfInfo := &middleware.ManifestInfo{
ProjectID: 1,
af := &middleware.ArtifactInfo{
Repository: name,
Reference: tag,
Tag: tag,
Digest: "",
}
@ -40,10 +41,9 @@ func doPullManifestRequest(projectName, name, tag string, next ...http.HandlerFu
w.WriteHeader(http.StatusNotFound)
}
}
*req = *(req.WithContext(middleware.NewManifestInfoContext(req.Context(), mfInfo)))
h := Middleware()(http.HandlerFunc(n))
h.ServeHTTP(util.NewCustomResponseWriter(rr), req)
ctx := context.WithValue(req.Context(), middleware.ArtifactInfoKey, af)
*req = *(req.WithContext(ctx))
n.ServeHTTP(util.NewCustomResponseWriter(rr), req)
return rr.Code
}

View File

@ -16,7 +16,6 @@ import (
"net/http"
"net/http/httptest"
"regexp"
"sync"
)
type contextKey string
@ -30,8 +29,6 @@ const (
DigestSubexp = "digest"
// ArtifactInfoKey the context key for artifact info
ArtifactInfoKey = contextKey("artifactInfo")
// manifestInfoKey the context key for manifest info
manifestInfoKey = contextKey("ManifestInfo")
// ScannerPullCtxKey the context key for robot account to bypass the pull policy check.
ScannerPullCtxKey = contextKey("ScannerPullCheck")
)
@ -49,60 +46,44 @@ var (
V2CatalogURLRe = regexp.MustCompile(`^/v2/_catalog$`)
)
// ManifestInfo ...
type ManifestInfo struct {
ProjectID int64
ProjectName string
Repository string
Tag string
Digest string
manifestExist bool
manifestExistErr error
manifestExistOnce sync.Once
}
// ManifestExists ...
func (info *ManifestInfo) ManifestExists(ctx context.Context) (bool, error) {
info.manifestExistOnce.Do(func() {
af, err := artifact.Ctl.GetByReference(ctx, info.Repository, info.Tag, nil)
if err != nil {
info.manifestExistErr = err
return
}
info.manifestExist = true
info.Digest = af.Digest
})
return info.manifestExist, info.manifestExistErr
}
// ArtifactInfo ...
type ArtifactInfo struct {
Repository string
Reference string
ProjectName string
Digest string
Tag string
BlobMountRepository string
BlobMountProjectName string
BlobMountDigest string
}
// ArtifactInfoFromContext returns the artifact info from context
func ArtifactInfoFromContext(ctx context.Context) (*ArtifactInfo, bool) {
// ArtifactInfoFromContext returns the artifact info from context, the returned value is a copied value, so updating
// the attributes of returned artifactInfo will not update the one in context.
func ArtifactInfoFromContext(ctx context.Context) (ArtifactInfo, bool) {
info, ok := ctx.Value(ArtifactInfoKey).(*ArtifactInfo)
return info, ok
var res ArtifactInfo
if ok {
res = *info
}
return res, ok
}
// NewManifestInfoContext returns context with manifest info
func NewManifestInfoContext(ctx context.Context, info *ManifestInfo) context.Context {
return context.WithValue(ctx, manifestInfoKey, info)
}
// ManifestInfoFromContext returns manifest info from context
func ManifestInfoFromContext(ctx context.Context) (*ManifestInfo, bool) {
info, ok := ctx.Value(manifestInfoKey).(*ManifestInfo)
return info, ok
// EnsureArtifactDigest get artifactInfo from context and set the digest for artifact that has project name repository and reference
func EnsureArtifactDigest(ctx context.Context) error {
info, ok := ctx.Value(ArtifactInfoKey).(*ArtifactInfo)
if !ok {
return fmt.Errorf("no artifact info in context")
}
if len(info.Digest) > 0 {
return nil
}
af, err := artifact.Ctl.GetByReference(ctx, info.Repository, info.Reference, nil)
if err != nil || af == nil {
return fmt.Errorf("failed to get artifact for populating digest, error: %v", err)
}
info.Digest = af.Digest
return nil
}
// NewScannerPullContext returns context with policy check info

View File

@ -92,27 +92,26 @@ func Middleware() func(http.Handler) http.Handler {
}
}
func validate(req *http.Request) (bool, *middleware.ManifestInfo, vuln.Severity, models.CVEWhitelist) {
func validate(req *http.Request) (bool, middleware.ArtifactInfo, vuln.Severity, models.CVEWhitelist) {
var vs vuln.Severity
var wl models.CVEWhitelist
var mf *middleware.ManifestInfo
mf, ok := middleware.ManifestInfoFromContext(req.Context())
if !ok {
return false, nil, vs, wl
var af middleware.ArtifactInfo
err := middleware.EnsureArtifactDigest(req.Context())
if err != nil {
return false, af, vs, wl
}
exist, err := mf.ManifestExists(req.Context())
if err != nil || !exist {
return false, nil, vs, wl
af, ok := middleware.ArtifactInfoFromContext(req.Context())
if !ok {
return false, af, vs, wl
}
if scannerPull, ok := middleware.ScannerPullFromContext(req.Context()); ok && scannerPull {
return false, mf, vs, wl
return false, af, vs, wl
}
// Is vulnerable policy set?
projectVulnerableEnabled, projectVulnerableSeverity, wl := middleware.GetPolicyChecker().VulnerablePolicy(mf.ProjectName)
projectVulnerableEnabled, projectVulnerableSeverity, wl := middleware.GetPolicyChecker().VulnerablePolicy(af.ProjectName)
if !projectVulnerableEnabled {
return false, mf, vs, wl
return false, af, vs, wl
}
return true, mf, projectVulnerableSeverity, wl
return true, af, projectVulnerableSeverity, wl
}

View File

@ -21,7 +21,6 @@ import (
"github.com/goharbor/harbor/src/server/middleware/blob"
"github.com/goharbor/harbor/src/server/middleware/contenttrust"
"github.com/goharbor/harbor/src/server/middleware/immutable"
"github.com/goharbor/harbor/src/server/middleware/manifestinfo"
"github.com/goharbor/harbor/src/server/middleware/readonly"
"github.com/goharbor/harbor/src/server/middleware/regtoken"
"github.com/goharbor/harbor/src/server/middleware/v2auth"
@ -49,7 +48,6 @@ func RegisterRoutes() {
root.NewRoute().
Method(http.MethodGet).
Path("/*/manifests/:reference").
Middleware(manifestinfo.Middleware()).
Middleware(regtoken.Middleware()).
Middleware(contenttrust.Middleware()).
Middleware(vulnerable.Middleware()).
@ -62,14 +60,12 @@ func RegisterRoutes() {
Method(http.MethodDelete).
Path("/*/manifests/:reference").
Middleware(readonly.Middleware()).
Middleware(manifestinfo.Middleware()).
Middleware(immutable.MiddlewareDelete()).
HandlerFunc(deleteManifest)
root.NewRoute().
Method(http.MethodPut).
Path("/*/manifests/:reference").
Middleware(readonly.Middleware()).
Middleware(manifestinfo.Middleware()).
Middleware(immutable.MiddlewarePush()).
Middleware(blob.PutManifestMiddleware()).
HandlerFunc(putManifest)