Fix the multipart form data content missing issue by refill the content again before passing to the backend server

refill the multipart form data at harbor API layer
export write error functions from the chartserver package
refactor the error format to be compatiable with push plugin

src/chartserver/chart_operator.go

@@ -16,6 +16,7 @@ import (
 
 const (
 	readmeFileName = "README.md"
+	valuesFileName = "values.yaml"
 )
 
 //ChartVersionDetails keeps the detailed data info of the chart version
@@ -72,7 +73,7 @@ func (cho *ChartOperator) GetChartDetails(content []byte) (*ChartVersionDetails,
 		values = parseRawValues([]byte(chartData.Values.GetRaw()))
 		if len(values) > 0 {
 			//Append values.yaml file
-			files["values.yaml"] = chartData.Values.Raw
+			files[valuesFileName] = chartData.Values.Raw
 		}
 	}
 

src/chartserver/client.go

@@ -75,6 +75,10 @@ func (cc *ChartClient) GetContent(addr string) ([]byte, error) {
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusOK {
+		if err := extractError(content); err != nil {
+			return nil, err
+		}
+
 		return nil, fmt.Errorf("failed to retrieve content from '%s' with error: %s", fullURI.Path, content)
 	}
 

src/chartserver/manipulation_handler.go

@@ -49,19 +49,19 @@ func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Reque
 
 	content, err := mh.apiClient.GetContent(url)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
 	chartList, err := mh.chartOperator.GetChartList(content)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
 	jsonData, err := json.Marshal(chartList)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
@@ -79,7 +79,7 @@ func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request
 func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) {
 	chartV, err := mh.getChartVersion(req.URL.String())
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
@@ -97,20 +97,20 @@ func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.
 		}
 
 		if len(strings.TrimSpace(namespace)) == 0 {
-			writeInternalError(w, errors.New("failed to extract namespace from the request"))
+			WriteInternalError(w, errors.New("failed to extract namespace from the request"))
 			return
 		}
 
 		content, err := mh.getChartVersionContent(namespace, chartV.URLs[0])
 		if err != nil {
-			writeInternalError(w, err)
+			WriteInternalError(w, err)
 			return
 		}
 
 		//Process bytes and get more details of chart version
 		chartDetails, err = mh.chartOperator.GetChartDetails(content)
 		if err != nil {
-			writeInternalError(w, err)
+			WriteInternalError(w, err)
 			return
 		}
 		chartDetails.Metadata = chartV
@@ -127,7 +127,7 @@ func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.
 
 	bytes, err := json.Marshal(chartDetails)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 

src/chartserver/repo_handler.go

@@ -60,14 +60,14 @@ func (rh *RepositoryHandler) GetIndexFile(w http.ResponseWriter, req *http.Reque
 	//Get project manager references
 	projectMgr, err := filter.GetProjectManager(req)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
 	//Get all the projects
 	results, err := projectMgr.List(nil)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
@@ -179,7 +179,7 @@ LOOP:
 	//All the threads are done
 	//Met an error
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 
@@ -195,7 +195,7 @@ LOOP:
 
 	bytes, err := yaml.Marshal(mergedIndexFile)
 	if err != nil {
-		writeInternalError(w, err)
+		WriteInternalError(w, err)
 		return
 	}
 

src/chartserver/utils.go

@@ -14,8 +14,8 @@ const (
 	contentTypeJSON   = "application/json"
 )
 
-//Write error to http client
-func writeError(w http.ResponseWriter, code int, err error) {
+//WriteError writes error to http client
+func WriteError(w http.ResponseWriter, code int, err error) {
 	errorObj := make(map[string]string)
 	errorObj["error"] = err.Error()
 	errorContent, _ := json.Marshal(errorObj)
@@ -24,9 +24,9 @@ func writeError(w http.ResponseWriter, code int, err error) {
 	w.Write(errorContent)
 }
 
-//StatusCode == 500
-func writeInternalError(w http.ResponseWriter, err error) {
-	writeError(w, http.StatusInternalServerError, err)
+//WriteInternalError writes error with statusCode == 500
+func WriteInternalError(w http.ResponseWriter, err error) {
+	WriteError(w, http.StatusInternalServerError, err)
 }
 
 //Write JSON data to http client
@@ -36,6 +36,26 @@ func writeJSONData(w http.ResponseWriter, data []byte) {
 	w.Write(data)
 }
 
+//Extract error object '{"error": "****---***"}' from the content if existing
+//nil error will be returned if it does exist
+func extractError(content []byte) error {
+	if len(content) == 0 {
+		return nil
+	}
+
+	errorObj := make(map[string]string)
+	err := json.Unmarshal(content, &errorObj)
+	if err != nil {
+		return nil
+	}
+
+	if errText, ok := errorObj["error"]; ok {
+		return errors.New(errText)
+	}
+
+	return nil
+}
+
 //Parse the redis configuration to the beego cache pattern
 //Config pattern is "address:port[,weight,password,db_index]"
 func parseRedisConfig(redisConfigV string) (string, error) {

src/ui/api/chart_repository.go

@@ -1,9 +1,13 @@
 package api
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
 	"net/http"
 	"net/url"
 	"strings"
@@ -25,6 +29,10 @@ const (
 	accessLevelWrite
 	accessLevelAll
 	accessLevelSystem
+
+	formFieldNameForChart = "chart"
+	formFiledNameForProv  = "prov"
+	headerContentType     = "Content-Type"
 )
 
 //chartController is a singleton instance
@@ -162,6 +170,21 @@ func (cra *ChartRepositoryAPI) UploadChartVersion() {
 		return
 	}
 
+	//Rewrite file content
+	formFiles := make([]formFile, 0)
+	formFiles = append(formFiles,
+		formFile{
+			formField: formFieldNameForChart,
+			mustHave:  true,
+		},
+		formFile{
+			formField: formFiledNameForProv,
+		})
+	if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
+		chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
+		return
+	}
+
 	chartController.GetManipulationHandler().UploadChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
 }
 
@@ -172,6 +195,18 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() {
 		return
 	}
 
+	//Rewrite file content
+	formFiles := make([]formFile, 0)
+	formFiles = append(formFiles,
+		formFile{
+			formField: formFiledNameForProv,
+			mustHave:  true,
+		})
+	if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
+		chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
+		return
+	}
+
 	chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
 }
 
@@ -310,3 +345,63 @@ func initializeChartController() (*chartserver.Controller, error) {
 
 	return controller, nil
 }
+
+//formFile is used to represent the uploaded files in the form
+type formFile struct {
+	//form field key contains the form file
+	formField string
+
+	//flag to indicate if the file identified by the 'formField'
+	//must exist
+	mustHave bool
+}
+
+//If the files are uploaded with multipart/form-data mimetype, beego will extract the data
+//from the request automatically. Then the request passed to the backend server with proxying
+//way will have empty content.
+//This method will refill the requests with file content.
+func (cra *ChartRepositoryAPI) rewriteFileContent(files []formFile, request *http.Request) error {
+	if len(files) == 0 {
+		return nil //no files, early return
+	}
+
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+	defer func() {
+		if err := w.Close(); err != nil {
+			//Just log it
+			hlog.Errorf("Failed to defer close multipart writer with error: %s", err.Error())
+		}
+	}()
+
+	//Process files by key one by one
+	for _, f := range files {
+		mFile, mHeader, err := cra.GetFile(f.formField)
+		//Handle error case by case
+		if err != nil {
+			formatedErr := fmt.Errorf("Get file content with multipart header from key '%s' failed with error: %s", f.formField, err.Error())
+			if f.mustHave || err != http.ErrMissingFile {
+				return formatedErr
+			}
+
+			//Error can be ignored, just log it
+			hlog.Warning(formatedErr.Error())
+			continue
+		}
+
+		fw, err := w.CreateFormFile(f.formField, mHeader.Filename)
+		if err != nil {
+			return fmt.Errorf("Create form file with multipart header failed with error: %s", err.Error())
+		}
+
+		_, err = io.Copy(fw, mFile)
+		if err != nil {
+			return fmt.Errorf("Copy file stream in multipart form data failed with error: %s", err.Error())
+		}
+	}
+
+	request.Header.Set(headerContentType, w.FormDataContentType())
+	request.Body = ioutil.NopCloser(&body)
+
+	return nil
+}