优化文件传输占用内存问题

build/cmd/docker_publish.sh

@@ -9,7 +9,7 @@ Version=$(genVersion $1)
 echo ${Version}
 
 Username="eolinker"
-if [[ "$1" != "" ]]
+if [[ "$2" != "" ]]
 then
 		Username=$2
 fi

drivers/app/extend-param.go

@@ -31,10 +31,10 @@ func (a *additionalParam) Execute(ctx http_service.IHttpContext) error {
 		return nil
 	}
 	contentType, _, _ := mime.ParseMediaType(ctx.Proxy().Body().ContentType())
-	bodyParams, formParams, err := parseBodyParams(ctx)
-	if err != nil {
-		return fmt.Errorf(`fail to parse body! [err]: %v`, err)
-	}
+	var bodyParams interface{}
+	var formParams map[string][]string
+	var err error
+
 	for _, p := range a.params {
 		conflict := p.Conflict
 		if conflict == "" {
@@ -45,6 +45,12 @@ func (a *additionalParam) Execute(ctx http_service.IHttpContext) error {
 			if ctx.Proxy().Method() != http.MethodPost && ctx.Proxy().Method() != http.MethodPut && ctx.Proxy().Method() != http.MethodPatch {
 				continue
 			}
+			if bodyParams == nil && formParams == nil {
+				bodyParams, formParams, err = parseBodyParams(ctx)
+				if err != nil {
+					return fmt.Errorf(`fail to parse body! [err]: %v`, err)
+				}
+			}
 			switch contentType {
 			case http_context.FormData, http_context.MultipartForm:
 				switch p.Conflict {

node/http-context/body.go

@@ -2,6 +2,7 @@ package http_context
 
 import (
 	"bytes"
+	"github.com/eolinker/eosc/log"
 	"io"
 	"strings"
 
@@ -32,7 +33,8 @@ const (
 type BodyRequestHandler struct {
 	request *fasthttp.Request
 
-	formdata *multipart.Form
+	formdata    *multipart.Form
+	isResetFile bool
 }
 
 func (b *BodyRequestHandler) MultipartForm() (*multipart.Form, error) {
@@ -52,7 +54,8 @@ func (b *BodyRequestHandler) MultipartForm() (*multipart.Form, error) {
 		File:  form.File,
 	}
 
-	return form, b.resetFile()
+	//return form, b.resetFile()
+	return form, nil
 }
 func (b *BodyRequestHandler) Files() (map[string][]*multipart.FileHeader, error) {
 	form, err := b.MultipartForm()
@@ -121,6 +124,18 @@ func (b *BodyRequestHandler) BodyForm() (url.Values, error) {
 
 // RawBody 获取raw数据
 func (b *BodyRequestHandler) RawBody() ([]byte, error) {
+	if b.isResetFile == false {
+		contentType, _, _ := mime.ParseMediaType(string(b.request.Header.ContentType()))
+		if contentType == MultipartForm {
+			err := b.resetFile()
+			if err != nil {
+				log.Errorf("reset file error: %v", err)
+				return nil, err
+			}
+			b.isResetFile = true
+		}
+	}
+
 	return b.request.Body(), nil
 }
 
@@ -148,7 +163,9 @@ func (b *BodyRequestHandler) SetToForm(key, value string) error {
 			return err
 		}
 		multipartForm.Value[key] = []string{value}
-		return b.resetFile()
+		b.isResetFile = false
+		//return b.resetFile()
+		return nil
 	default:
 		return ErrorNotForm
 	}
@@ -169,7 +186,9 @@ func (b *BodyRequestHandler) AddForm(key, value string) error {
 			return err
 		}
 		multipartForm.Value[key] = append(multipartForm.Value[key], value)
-		return b.resetFile()
+		b.isResetFile = false
+		//return b.resetFile()
+		return nil
 	default:
 		return ErrorNotForm
 	}
@@ -187,8 +206,9 @@ func (b *BodyRequestHandler) AddFile(key string, file *multipart.FileHeader) err
 		return err
 	}
 	multipartForm.File[key] = append(multipartForm.File[key], file)
-
-	return b.resetFile()
+	b.isResetFile = false
+	//return b.resetFile()
+	return nil
 }
 
 // SetFile 设置文件参数
@@ -200,7 +220,8 @@ func (b *BodyRequestHandler) SetFile(files map[string][]*multipart.FileHeader) e
 	}
 	multipartForm.File = files
 
-	return b.resetFile()
+	//return b.resetFile()
+	return nil
 }
 
 func (b *BodyRequestHandler) resetFile() error {
@@ -218,11 +239,6 @@ func (b *BodyRequestHandler) resetFile() error {
 			if err != nil {
 				return err
 			}
-			//part, err := writer.CreateFormFile(name, f.Filename)
-			//if err != nil {
-			//	fio.Close()
-			//	return err
-			//}
 			part, err := writer.CreatePart(f.Header)
 			if err != nil {
 				return err
@@ -279,7 +295,8 @@ func (b *BodyRequestHandler) SetForm(values url.Values) error {
 			return err
 		}
 		multipartForm.Value = values
-		return b.resetFile()
+		//return b.resetFile()
+		return nil
 	}
 
 	return ErrorNotForm

node/http-context/context.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -183,7 +184,7 @@ func (ctx *HttpContext) SendTo(scheme string, node eoscContext.INode, timeout ti
 
 	host := node.Addr()
 	request := ctx.proxyRequest.Request()
-	request.CloseBodyStream()
+	//request.CloseBodyStream()
 	rewriteHost := string(request.Host())
 	upstreamHost := ctx.GetUpstreamHostHandler()
 	if upstreamHost != nil {
@@ -203,6 +204,9 @@ func (ctx *HttpContext) SendTo(scheme string, node eoscContext.INode, timeout ti
 		rewriteHost = host
 		request.URI().SetHost(host)
 	}
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	log.DebugF("After: HeapAlloc=%.2f MB", float64(m.HeapAlloc)/(1024.0*1024.0))
 
 	beginTime := time.Now()
 	response := fasthttp.AcquireResponse()
@@ -363,6 +367,9 @@ func (ctx *HttpContext) Clone() (eoscContext.EoContext, error) {
 
 // NewContext 创建Context
 func NewContext(ctx *fasthttp.RequestCtx, port int) *HttpContext {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	log.DebugF("Before: HeapAlloc=%.2f MB", float64(m.HeapAlloc)/(1024.0*1024.0))
 
 	remoteAddr := ctx.RemoteAddr().String()
 
@@ -374,9 +381,9 @@ func NewContext(ctx *fasthttp.RequestCtx, port int) *HttpContext {
 	// 原始请求最大读取body为8k，使用clone request
 	request := fasthttp.AcquireRequest()
 
-	if ctx.Request.IsBodyStream() && ctx.Request.Header.ContentLength() > 8*1024 {
-		ctx.Request.Body()
-	}
+	//if ctx.Request.IsBodyStream() && ctx.Request.Header.ContentLength() > 8*1024 {
+	//	ctx.Request.Body()
+	//}
 	ctx.Request.CopyTo(request)
 	httpContext.requestReader.reset(request, remoteAddr)