fix(quota/sync) #8886

The foreign layer won't be counted into project quota
NOTE: the foreign layer will be dumped from the registry in the migration

Signed-off-by: wang yan <wangyan@vmware.com>

src/common/const.go

@@ -154,4 +154,7 @@ const (
 	QuotaPerProjectEnable = "quota_per_project_enable"
 	CountPerProject       = "count_per_project"
 	StoragePerProject     = "storage_per_project"
+
+	// ForeignLayer
+	ForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
 )

src/common/dao/project_blob.go

@@ -20,6 +20,8 @@ import (
 	"time"
 
 	"github.com/goharbor/harbor/src/common/models"
+
+	"github.com/goharbor/harbor/src/common"
 )
 
 // AddBlobToProject ...
@@ -114,6 +116,7 @@ func GetBlobsNotInProject(projectID int64, blobDigests ...string) ([]*models.Blo
 }
 
 // CountSizeOfProject ...
+// foreign blob won't be calculated
 func CountSizeOfProject(pid int64) (int64, error) {
 	var blobs []models.Blob
 
@@ -130,8 +133,9 @@ JOIN artifact_blob afnb
 JOIN BLOB bb
     ON afnb.digest_blob = bb.digest
 WHERE af.project_id = ? 
+AND bb.content_type != ?
 `
-	_, err := GetOrmer().Raw(sql, pid).QueryRows(&blobs)
+	_, err := GetOrmer().Raw(sql, pid, common.ForeignLayer).QueryRows(&blobs)
 	if err != nil {
 		return 0, err
 	}

src/common/dao/project_blob_test.go

@@ -81,20 +81,31 @@ func TestHasBlobInProject(t *testing.T) {
 
 func TestCountSizeOfProject(t *testing.T) {
 	_, err := AddBlob(&models.Blob{
-		Digest: "CountSizeOfProject_blob1",
-		Size:   101,
+		Digest:      "CountSizeOfProject_blob1",
+		ContentType: "application/vnd.docker.distribution.manifest.v2+json",
+		Size:        101,
 	})
 	require.Nil(t, err)
 
 	_, err = AddBlob(&models.Blob{
-		Digest: "CountSizeOfProject_blob2",
-		Size:   202,
+		Digest:      "CountSizeOfProject_blob2",
+		ContentType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+		Size:        202,
 	})
 	require.Nil(t, err)
 
 	_, err = AddBlob(&models.Blob{
-		Digest: "CountSizeOfProject_blob3",
-		Size:   303,
+		Digest:      "CountSizeOfProject_blob3",
+		ContentType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+		Size:        303,
+	})
+	require.Nil(t, err)
+
+	// this blob won't be calculated into project size
+	_, err = AddBlob(&models.Blob{
+		Digest:      "CountSizeOfProject_blob4",
+		ContentType: "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+		Size:        404,
 	})
 	require.Nil(t, err)
 
@@ -128,11 +139,16 @@ func TestCountSizeOfProject(t *testing.T) {
 		DigestAF:   "CountSizeOfProject_af1",
 		DigestBlob: "CountSizeOfProject_blob3",
 	}
+	afnb4 := &models.ArtifactAndBlob{
+		DigestAF:   "CountSizeOfProject_af1",
+		DigestBlob: "CountSizeOfProject_blob4",
+	}
 
 	var afnbs []*models.ArtifactAndBlob
 	afnbs = append(afnbs, afnb1)
 	afnbs = append(afnbs, afnb2)
 	afnbs = append(afnbs, afnb3)
+	afnbs = append(afnbs, afnb4)
 
 	// add
 	err = AddArtifactNBlobs(afnbs)

src/core/api/quota/registry/registry.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 
+	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
 	common_quota "github.com/goharbor/harbor/src/common/quota"
@@ -157,7 +158,8 @@ func (rm *Migrator) Usage(projects []quota.ProjectInfo) ([]quota.ProjectUsage, e
 			// Because that there are some shared blobs between repositories, it needs to remove the duplicate items.
 			for _, blob := range repo.Blobs {
 				_, exist := blobs[blob.Digest]
-				if !exist {
+				// foreign blob won't be calculated
+				if !exist && blob.ContentType != common.ForeignLayer {
 					blobs[blob.Digest] = blob.Size
 				}
 			}