From 0a7443cae7e49d570d4d44eca519a53fe5774404 Mon Sep 17 00:00:00 2001 From: wklken Date: Mon, 24 Jan 2022 00:49:51 +0800 Subject: [PATCH 01/11] feat(headers): set user-agent by s.UserAgent("") --- gorequest.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gorequest.go b/gorequest.go index f9c2ce0..d784309 100644 --- a/gorequest.go +++ b/gorequest.go @@ -365,6 +365,18 @@ func (s *SuperAgent) AppendHeader(param string, value string) *SuperAgent { return s } +// UserAgent is used for setting User-Agent into headers +// Example. To set `User-Agent` as `Custom user agent` +// +// gorequest.New(). +// Post("https://httpbin.org/post"). +// UserAgent("Custom user agent"). +// End() +func (s *SuperAgent) UserAgent(ua string) *SuperAgent { + s.Header.Add("User-Agent", ua) + return s +} + // Retry is used for setting a Retryer policy // Example. To set Retryer policy with 5 seconds between each attempt. // 3 max attempt. From b6763a22871e972c352188be680308c3659bc2eb Mon Sep 17 00:00:00 2001 From: wklken Date: Mon, 24 Jan 2022 20:06:09 +0800 Subject: [PATCH 02/11] feat(feature/stats): add Stats for SuperAgent --- gorequest.go | 24 ++++++++++++++---------- stats.go | 10 ++++++++++ util.go | 15 +++++++++++++++ 3 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 stats.go diff --git a/gorequest.go b/gorequest.go index f9c2ce0..0aaee43 100644 --- a/gorequest.go +++ b/gorequest.go @@ -65,6 +65,7 @@ type SuperAgent struct { DoNotClearSuperAgent bool isClone bool ctx context.Context + Stats Stats } var DisableTransportSwap = false @@ -98,22 +99,13 @@ func New() *SuperAgent { logger: log.New(os.Stderr, "[gorequest]", log.LstdFlags), isClone: false, ctx: nil, + Stats: Stats{}, } // disable keep alives by default, see this issue https://github.com/parnurzeal/gorequest/issues/75 s.Transport.DisableKeepAlives = true return s } -// just need to change the array pointer? -func copyRetryable(old superAgentRetryable) superAgentRetryable { - newRetryable := old - newRetryable.RetryableStatus = make([]int, len(old.RetryableStatus)) - for i := range old.RetryableStatus { - newRetryable.RetryableStatus[i] = old.RetryableStatus[i] - } - return newRetryable -} - // Clone returns a copy of this superagent. Useful if you want to reuse the client/settings // concurrently. // Note: This does a shallow copy of the parent. So you will need to be @@ -149,6 +141,7 @@ func (s *SuperAgent) Clone() *SuperAgent { DoNotClearSuperAgent: true, isClone: true, ctx: s.ctx, + Stats: copyStats(s.Stats), } return clone } @@ -202,6 +195,7 @@ func (s *SuperAgent) ClearSuperAgent() { s.Cookies = make([]*http.Cookie, 0) s.Errors = nil s.ctx = nil + s.Stats = Stats{} } // CustomMethod is just a wrapper to initialize SuperAgent instance by method string. @@ -1201,6 +1195,10 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { } } + startTime := time.Now() + // stats collect the requestBytes + s.Stats.RequestBytes = req.ContentLength + // Send request resp, err = s.Client.Do(req) if err != nil { @@ -1209,6 +1207,9 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { } defer resp.Body.Close() + // stats collect the RequestDuration + s.Stats.RequestDuration = time.Since(startTime) + // Log details of this response if s.Debug { dump, err := httputil.DumpResponse(resp, true) @@ -1225,6 +1226,9 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { if err != nil { return nil, nil, []error{err} } + + // stats collect the responseBytes + s.Stats.ResponseBytes = int64(len(body)) return resp, body, nil } diff --git a/stats.go b/stats.go new file mode 100644 index 0000000..0337d2b --- /dev/null +++ b/stats.go @@ -0,0 +1,10 @@ +package gorequest + +import "time" + +type Stats struct { + RequestBytes int64 + ResponseBytes int64 + + RequestDuration time.Duration +} diff --git a/util.go b/util.go index 8472e52..16489c1 100644 --- a/util.go +++ b/util.go @@ -16,6 +16,21 @@ func cloneMapArray(old map[string][]string) map[string][]string { return newMap } +// just need to change the array pointer? +func copyRetryable(old superAgentRetryable) superAgentRetryable { + newRetryable := old + newRetryable.RetryableStatus = make([]int, len(old.RetryableStatus)) + for i := range old.RetryableStatus { + newRetryable.RetryableStatus[i] = old.RetryableStatus[i] + } + return newRetryable +} + +func copyStats(old Stats) Stats { + newStats := old + return newStats +} + func shallowCopyData(old map[string]interface{}) map[string]interface{} { if old == nil { return nil From c963406f610af99a11927011239a9e7efd9f99d9 Mon Sep 17 00:00:00 2001 From: wklken Date: Tue, 25 Jan 2022 10:21:32 +0800 Subject: [PATCH 03/11] feat(ft/disablecompression): add DisableCompression() --- gorequest.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gorequest.go b/gorequest.go index f9c2ce0..11af0c2 100644 --- a/gorequest.go +++ b/gorequest.go @@ -182,6 +182,12 @@ func (s *SuperAgent) SetLogger(logger Logger) *SuperAgent { return s } +// DisableCompression disable the compression of http.Client. +func (s *SuperAgent) DisableCompression() *SuperAgent { + s.Transport.DisableCompression = true + return s +} + // ClearSuperAgent clear SuperAgent data for another new request. func (s *SuperAgent) ClearSuperAgent() { if s.DoNotClearSuperAgent { From 1372257697a6518dcfead65039126e84b76c7920 Mon Sep 17 00:00:00 2001 From: wklken Date: Tue, 25 Jan 2022 14:37:59 +0800 Subject: [PATCH 04/11] feat(mock): add Mock() support HTTP mocking --- README.md | 27 +++++++++++++++++++++++++++ go.mod | 1 + go.sum | 6 ++++++ gorequest.go | 13 ++++++++++++- gorequest_test.go | 22 ++++++++++++++++++++++ 5 files changed, 68 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8122e64..421a9f0 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,33 @@ baseRequest.Timeout(10 * time.Millisecond). resp, body, errs := baseRequest.Clone().Get("http://exmaple.com/").End() ``` +## Mock + +You can mock the response of gorequest via [gock](https://github.com/h2non/gock) + +```go +func TestMock(t *testing.T) { + defer gock.Off() + + gock.New("http://foo.com"). + Get("/bar"). + Reply(200). + JSON(map[string]string{"foo": "bar"}) + + resp, body, errs := New().Mock().Get("http://foo.com/bar").SetDebug(true).End() + if len(errs) != 0 { + t.Fatalf("Expected no error, got error") + } + if resp.StatusCode != 200 { + t.Fatalf("Expected status code 200, got %d", resp.StatusCode) + } + + if strings.Trim(body, " \n") != `{"foo":"bar"}` { + t.Fatalf("Expected body `{\"foo\":\"bar\"}`, got `%s`", body) + } +} +``` + ## Debug For debugging, GoRequest leverages `httputil` to dump details of every request/response. (Thanks to @dafang) diff --git a/go.mod b/go.mod index 1ea7a75..63a5a68 100644 --- a/go.mod +++ b/go.mod @@ -7,5 +7,6 @@ require ( github.com/smartystreets/goconvey v1.7.2 // indirect github.com/spf13/cast v1.4.1 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f + gopkg.in/h2non/gock.v1 v1.1.2 moul.io/http2curl v1.0.0 ) diff --git a/go.sum b/go.sum index f77ede7..296757e 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,12 @@ github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy0 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= @@ -31,5 +35,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= diff --git a/gorequest.go b/gorequest.go index f9c2ce0..98600f1 100644 --- a/gorequest.go +++ b/gorequest.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/cast" "golang.org/x/net/publicsuffix" + "gopkg.in/h2non/gock.v1" "moul.io/http2curl" ) @@ -65,6 +66,7 @@ type SuperAgent struct { DoNotClearSuperAgent bool isClone bool ctx context.Context + isMock bool } var DisableTransportSwap = false @@ -98,6 +100,7 @@ func New() *SuperAgent { logger: log.New(os.Stderr, "[gorequest]", log.LstdFlags), isClone: false, ctx: nil, + isMock: false, } // disable keep alives by default, see this issue https://github.com/parnurzeal/gorequest/issues/75 s.Transport.DisableKeepAlives = true @@ -149,6 +152,7 @@ func (s *SuperAgent) Clone() *SuperAgent { DoNotClearSuperAgent: true, isClone: true, ctx: s.ctx, + isMock: s.isMock, } return clone } @@ -158,6 +162,13 @@ func (s *SuperAgent) Context(ctx context.Context) *SuperAgent { return s } +// Mock will enable gock, http mocking for net/http +func (s *SuperAgent) Mock() *SuperAgent { + gock.InterceptClient(s.Client) + s.isMock = true + return s +} + // SetDebug enable the debug mode which logs request/response detail. func (s *SuperAgent) SetDebug(enable bool) *SuperAgent { s.Debug = enable @@ -1175,7 +1186,7 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { } // Set Transport - if !DisableTransportSwap { + if !DisableTransportSwap && !s.isMock { s.Client.Transport = s.Transport } diff --git a/gorequest_test.go b/gorequest_test.go index 4757f52..545d44a 100644 --- a/gorequest_test.go +++ b/gorequest_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/elazarl/goproxy" + "gopkg.in/h2non/gock.v1" ) type ( @@ -2688,3 +2689,24 @@ func TestContext(t *testing.T) { t.Fatalf("Expected context deadline exceeded error, got %v", errs[0].Error()) } } + +func TestMock(t *testing.T) { + defer gock.Off() + + gock.New("http://foo.com"). + Get("/bar"). + Reply(200). + JSON(map[string]string{"foo": "bar"}) + + resp, body, errs := New().Mock().Get("http://foo.com/bar").SetDebug(true).End() + if len(errs) != 0 { + t.Fatalf("Expected no error, got error") + } + if resp.StatusCode != 200 { + t.Fatalf("Expected status code 200, got %d", resp.StatusCode) + } + + if strings.Trim(body, " \n") != `{"foo":"bar"}` { + t.Fatalf("Expected body `{\"foo\":\"bar\"}`, got `%s`", body) + } +} From f2eb2e0f614e9d484da8a9cd4d54b35492fd87be Mon Sep 17 00:00:00 2001 From: wklken Date: Tue, 25 Jan 2022 14:52:54 +0800 Subject: [PATCH 05/11] fix(transport): update safeModifyTransport() copy transport to support go 1.16 --- gorequest.go | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/gorequest.go b/gorequest.go index f9c2ce0..aa3f93e 100644 --- a/gorequest.go +++ b/gorequest.go @@ -1448,22 +1448,30 @@ func (s *SuperAgent) safeModifyTransport() { } oldTransport := s.Transport s.Transport = &http.Transport{ - Proxy: oldTransport.Proxy, - DialContext: oldTransport.DialContext, - Dial: oldTransport.Dial, - DialTLS: oldTransport.DialTLS, - TLSClientConfig: oldTransport.TLSClientConfig, - TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout, - DisableKeepAlives: oldTransport.DisableKeepAlives, - DisableCompression: oldTransport.DisableCompression, - MaxIdleConns: oldTransport.MaxIdleConns, - MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost, - IdleConnTimeout: oldTransport.IdleConnTimeout, - ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout, - ExpectContinueTimeout: oldTransport.ExpectContinueTimeout, - TLSNextProto: oldTransport.TLSNextProto, + Proxy: oldTransport.Proxy, + DialContext: oldTransport.DialContext, + Dial: oldTransport.Dial, + DialTLSContext: oldTransport.DialTLSContext, + DialTLS: oldTransport.DialTLS, + TLSClientConfig: oldTransport.TLSClientConfig, + TLSHandshakeTimeout: oldTransport.TLSHandshakeTimeout, + DisableKeepAlives: oldTransport.DisableKeepAlives, + DisableCompression: oldTransport.DisableCompression, + MaxIdleConns: oldTransport.MaxIdleConns, + MaxIdleConnsPerHost: oldTransport.MaxIdleConnsPerHost, + MaxConnsPerHost: oldTransport.MaxConnsPerHost, + IdleConnTimeout: oldTransport.IdleConnTimeout, + ResponseHeaderTimeout: oldTransport.ResponseHeaderTimeout, + ExpectContinueTimeout: oldTransport.ExpectContinueTimeout, + TLSNextProto: oldTransport.TLSNextProto, + ProxyConnectHeader: oldTransport.ProxyConnectHeader, + + // new from go 1.16 + GetProxyConnectHeader: oldTransport.GetProxyConnectHeader, + MaxResponseHeaderBytes: oldTransport.MaxResponseHeaderBytes, - // new in go1.8 - ProxyConnectHeader: oldTransport.ProxyConnectHeader, + WriteBufferSize: oldTransport.WriteBufferSize, + ReadBufferSize: oldTransport.ReadBufferSize, + ForceAttemptHTTP2: oldTransport.ForceAttemptHTTP2, } } From f25a6c1865bf589fd920a3f5eac21262297d0218 Mon Sep 17 00:00:00 2001 From: wklken Date: Tue, 25 Jan 2022 15:30:01 +0800 Subject: [PATCH 06/11] fix(force_type): forceType not working while it's only be changed in getResponseBytes fix https://github.com/parnurzeal/gorequest/issues/217 and https://github.com/parnurzeal/gorequest/issues/204 --- gorequest.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/gorequest.go b/gorequest.go index f9c2ce0..ec37a77 100644 --- a/gorequest.go +++ b/gorequest.go @@ -1147,25 +1147,6 @@ func (s *SuperAgent) getResponseBytes() (Response, []byte, []error) { if len(s.Errors) != 0 { return nil, nil, s.Errors } - // check if there is forced type - switch s.ForceType { - case TypeJSON, TypeForm, TypeXML, TypeText, TypeMultipart: - s.TargetType = s.ForceType - // If forcetype is not set, check whether user set Content-Type header. - // If yes, also bounce to the correct supported TargetType automatically. - default: - contentType := s.Header.Get("Content-Type") - for k, v := range Types { - if contentType == v { - s.TargetType = k - } - } - } - - // if slice and map get mixed, let's bounce to rawstring - if len(s.Data) != 0 && len(s.SliceData) != 0 { - s.BounceToRawString = true - } // Make Request req, err = s.MakeRequest() @@ -1236,6 +1217,26 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) { err error ) + // check if there is forced type + switch s.ForceType { + case TypeJSON, TypeForm, TypeXML, TypeText, TypeMultipart: + s.TargetType = s.ForceType + // If forcetype is not set, check whether user set Content-Type header. + // If yes, also bounce to the correct supported TargetType automatically. + default: + contentType := s.Header.Get("Content-Type") + for k, v := range Types { + if contentType == v { + s.TargetType = k + } + } + } + + // if slice and map get mixed, let's bounce to rawstring + if len(s.Data) != 0 && len(s.SliceData) != 0 { + s.BounceToRawString = true + } + if s.Method == "" { return nil, fmt.Errorf("no method specified") } From e87ba00764b3eb61c73a29aa8debd0980e3e3ca5 Mon Sep 17 00:00:00 2001 From: wklken Date: Thu, 10 Feb 2022 23:01:57 +0800 Subject: [PATCH 07/11] feat(sendfile): enable custom Content-Type for SendFile 1) code copy and change from https://github.com/parnurzeal/gorequest/pull/213 2)fix issue https://github.com/parnurzeal/gorequest/issues/208 --- gorequest.go | 44 ++++++++++++++++++++++++++++++++++++++++++-- gorequest_test.go | 27 +++++++++++++++++++++++++++ util.go | 22 ++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/gorequest.go b/gorequest.go index f9c2ce0..37c203b 100644 --- a/gorequest.go +++ b/gorequest.go @@ -6,6 +6,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -808,10 +809,11 @@ func (s *SuperAgent) SendString(content string) *SuperAgent { type File struct { Filename string Fieldname string + MimeType string Data []byte } -// SendFile function works only with type "multipart". The function accepts one mandatory and up to two optional arguments. The mandatory (first) argument is the file. +// SendFile function works only with type "multipart". The function accepts one mandatory and up to three optional arguments. The mandatory (first) argument is the file. // The function accepts a path to a file as string: // // gorequest.New(). @@ -859,11 +861,32 @@ type File struct { // SendFile(b, "", "my_custom_fieldname"). // filename left blank, will become "example_file.ext" // End() // +// The third optional argument (fourth argument overall) is a bool value skipFileNumbering. It defaults to "false", +// if fieldname is "file" and skipFileNumbering is set to "false", the fieldname will be automatically set to +// fileNUMBER, where number is the greatest existing number+1. +// +// b, _ := ioutil.ReadFile("./example_file.ext") +// gorequest.New(). +// Post("http://example.com"). +// Type("multipart"). +// SendFile(b, "filename", "my_custom_fieldname", false). +// End() +// +// The fourth optional argument (fifth argument overall) is the mimetype request form-data part. It defaults to "application/octet-stream". +// +// b, _ := ioutil.ReadFile("./example_file.ext") +// gorequest.New(). +// Post("http://example.com"). +// Type("multipart"). +// SendFile(b, "filename", "my_custom_fieldname", false, "mime_type"). +// End() +// func (s *SuperAgent) SendFile(file interface{}, args ...interface{}) *SuperAgent { filename := "" fieldname := "file" skipFileNumbering := false + fileType := "application/octet-stream" if len(args) >= 1 { argFilename := fmt.Sprintf("%v", args[0]) @@ -886,6 +909,17 @@ func (s *SuperAgent) SendFile(file interface{}, args ...interface{}) *SuperAgent } } + if len(args) >= 4 { + argFileType := fmt.Sprintf("%v", args[3]) + if len(argFileType) > 0 { + fileType = strings.TrimSpace(argFileType) + } + if fileType == "" { + s.Errors = append(s.Errors, errors.New("the fifth SendFile method argument for MIME type cannot be an empty string")) + return s + } + } + if (fieldname == "file" && !skipFileNumbering) || fieldname == "" { fieldname = "file" + strconv.Itoa(len(s.FileData)+1) } @@ -908,6 +942,7 @@ func (s *SuperAgent) SendFile(file interface{}, args ...interface{}) *SuperAgent s.FileData = append(s.FileData, File{ Filename: filename, Fieldname: fieldname, + MimeType: fileType, Data: data, }) case reflect.Slice: @@ -918,6 +953,7 @@ func (s *SuperAgent) SendFile(file interface{}, args ...interface{}) *SuperAgent f := File{ Filename: filename, Fieldname: fieldname, + MimeType: fileType, Data: make([]byte, len(slice)), } for i := range slice { @@ -934,6 +970,9 @@ func (s *SuperAgent) SendFile(file interface{}, args ...interface{}) *SuperAgent if len(args) == 3 { return s.SendFile(v.Elem().Interface(), args[0], args[1], args[2]) } + if len(args) == 4 { + return s.SendFile(v.Elem().Interface(), args[0], args[1], args[2], args[3]) + } return s.SendFile(v.Elem().Interface()) default: if v.Type() == reflect.TypeOf(os.File{}) { @@ -949,6 +988,7 @@ func (s *SuperAgent) SendFile(file interface{}, args ...interface{}) *SuperAgent s.FileData = append(s.FileData, File{ Filename: filename, Fieldname: fieldname, + MimeType: fileType, Data: data, }) return s @@ -1340,7 +1380,7 @@ func (s *SuperAgent) MakeRequest() (*http.Request, error) { // add the files if len(s.FileData) != 0 { for _, file := range s.FileData { - fw, _ := mw.CreateFormFile(file.Fieldname, file.Filename) + fw, _ := CreateFormFile(mw, file.Fieldname, file.Filename, file.MimeType) fw.Write(file.Data) } contentReader = buf diff --git a/gorequest_test.go b/gorequest_test.go index 4757f52..0460734 100644 --- a/gorequest_test.go +++ b/gorequest_test.go @@ -1203,6 +1203,7 @@ func TestMultipartRequest(t *testing.T) { const case10b_send_file_by_path_pointer = "/send_file_by_path_pointer" const case11_send_file_by_path_without_name = "/send_file_by_path_without_name" const case12_send_file_by_path_without_name_but_with_fieldname = "/send_file_by_path_without_name_but_with_fieldname" + const case121_send_file_by_path_with_name_and_fieldname_and_mimetype = "/send_file_by_path_with_name_and_fieldname_and_mimetype" const case13_send_file_by_content_without_name = "/send_file_by_content_without_name" const case13a_send_file_by_content_without_name_pointer = "/send_file_by_content_without_name_pointer" @@ -1210,6 +1211,7 @@ func TestMultipartRequest(t *testing.T) { const case15_send_file_by_content_without_name_but_with_fieldname = "/send_file_by_content_without_name_but_with_fieldname" const case16_send_file_by_content_with_name_and_with_fieldname = "/send_file_by_content_with_name_and_with_fieldname" + const case161_send_file_by_content_with_name_and_fieldname_and_mimetype = "/send_file_by_content_with_name_and_fieldname_and_mimetype" const case17_send_file_multiple_by_path_and_content_without_name = "/send_file_multiple_by_path_and_content_without_name" const case18_send_file_multiple_by_path_and_content_with_name = "/send_file_multiple_by_path_and_content_with_name" @@ -1441,6 +1443,21 @@ func TestMultipartRequest(t *testing.T) { t.Error("Expected Header:Content-Type:application/octet-stream", "| but got", r.MultipartForm.File["my_fieldname"][0].Header["Content-Type"]) } checkFile(t, r.MultipartForm.File["my_fieldname"][0]) + case case161_send_file_by_content_with_name_and_fieldname_and_mimetype, case121_send_file_by_path_with_name_and_fieldname_and_mimetype: + if len(r.MultipartForm.File) != 1 { + t.Error("Expected length of files:[] == 1", "| but got", len(r.MultipartForm.File)) + } + if _, ok := r.MultipartForm.File["my_fieldname"]; !ok { + keys := reflect.ValueOf(r.MultipartForm.File).MapKeys() + t.Error("Expected Fieldname:my_fieldname", "| but got", keys) + } + if r.MultipartForm.File["my_fieldname"][0].Filename != "MY_LICENSE" { + t.Error("Expected Filename:MY_LICENSE", "| but got", r.MultipartForm.File["my_fieldname"][0].Filename) + } + if r.MultipartForm.File["my_fieldname"][0].Header["Content-Type"][0] != "application/json" { + t.Error("Expected Header:Content-Type:application/json", "| but got", r.MultipartForm.File["my_fieldname"][0].Header["Content-Type"]) + } + checkFile(t, r.MultipartForm.File["my_fieldname"][0]) case case17_send_file_multiple_by_path_and_content_without_name: if len(r.MultipartForm.File) != 2 { t.Error("Expected length of files:[] == 2", "| but got", len(r.MultipartForm.File)) @@ -1661,6 +1678,11 @@ func TestMultipartRequest(t *testing.T) { SendFile(fileByPath, "", "my_fieldname"). End() + New().Post(ts.URL+case121_send_file_by_path_with_name_and_fieldname_and_mimetype). + Type("multipart"). + SendFile(fileByPath, "MY_LICENSE", "my_fieldname", false, "application/json"). + End() + b, _ := ioutil.ReadFile("./LICENSE") New().Post(ts.URL + case13_send_file_by_content_without_name). Type("multipart"). @@ -1687,6 +1709,11 @@ func TestMultipartRequest(t *testing.T) { SendFile(b, "MY_LICENSE", "my_fieldname"). End() + New().Post(ts.URL+case161_send_file_by_content_with_name_and_fieldname_and_mimetype). + Type("multipart"). + SendFile(b, "MY_LICENSE", "my_fieldname", false, "application/json"). + End() + New().Post(ts.URL + case17_send_file_multiple_by_path_and_content_without_name). Type("multipart"). SendFile("./LICENSE"). diff --git a/util.go b/util.go index 8472e52..d0065b8 100644 --- a/util.go +++ b/util.go @@ -1,7 +1,12 @@ package gorequest import ( + "fmt" + "io" + "mime/multipart" "net/http" + "net/textproto" + "strings" "unsafe" ) @@ -104,3 +109,20 @@ func filterFlags(content string) string { } return content } + +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +// CreateFormFile is a convenience wrapper around CreatePart. It creates +// a new form-data header with the provided field name and file name. +func CreateFormFile(w *multipart.Writer, fieldname, filename string, contenttype string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + escapeQuotes(fieldname), escapeQuotes(filename))) + h.Set("Content-Type", contenttype) + return w.CreatePart(h) +} From 3784d51f2ecac7c2bf4ed8afb63365793346ac9f Mon Sep 17 00:00:00 2001 From: wklken Date: Sat, 12 Feb 2022 01:31:52 +0800 Subject: [PATCH 08/11] feat(timeout): support http client timeout details --- gorequest.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/gorequest.go b/gorequest.go index f9c2ce0..a031fb7 100644 --- a/gorequest.go +++ b/gorequest.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "log" "mime/multipart" + "net" "net/http" "net/http/cookiejar" "net/http/httputil" @@ -1441,6 +1442,39 @@ func (s *SuperAgent) Timeout(timeout time.Duration) *SuperAgent { return s } +type Timeouts struct { + Dial time.Duration + KeepAlive time.Duration + + TlsHandshake time.Duration + ResponseHeader time.Duration + ExpectContinue time.Duration + IdleConn time.Duration +} + +func (s *SuperAgent) Timeouts(timeouts *Timeouts) *SuperAgent { + s.safeModifyHttpClient() + + transport, ok := s.Client.Transport.(*http.Transport) + if !ok { + return s + } + + transport.DialContext = (&net.Dialer{ + Timeout: timeouts.Dial, + KeepAlive: timeouts.KeepAlive, + }).DialContext + + transport.TLSHandshakeTimeout = timeouts.TlsHandshake + transport.ResponseHeaderTimeout = timeouts.ResponseHeader + transport.ExpectContinueTimeout = timeouts.ExpectContinue + transport.ExpectContinueTimeout = timeouts.IdleConn + + s.Client.Transport = transport + + return s +} + // does a shallow clone of the transport func (s *SuperAgent) safeModifyTransport() { if !s.isClone { From c5f3b34514caef8c21560840d70978047311e6d6 Mon Sep 17 00:00:00 2001 From: wklken Date: Sat, 19 Feb 2022 23:38:50 +0800 Subject: [PATCH 09/11] test(useragent): add unittest for UserAgent() --- gorequest_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/gorequest_test.go b/gorequest_test.go index 4757f52..2beb490 100644 --- a/gorequest_test.go +++ b/gorequest_test.go @@ -2668,6 +2668,32 @@ func TestSetHeaders(t *testing.T) { End() } +func TestUserAgent(t *testing.T) { + text := "hi" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // check method is PATCH before going to check other features + if r.Method != POST { + t.Errorf("Expected method %q; got %q", POST, r.Method) + } + if r.Header == nil { + t.Error("Expected non-nil request Header") + } + if r.Header.Get("User-Agent") != "gorequest" { + t.Error("Expected Header User-Agent-> gorequest", "| but got", r.Header.Get("User-Agent")) + } + return + })) + + defer ts.Close() + + New().Post(ts.URL). + UserAgent("gorequest"). + Type("text"). + Send(text). + End() +} + func TestContext(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // check method is GET before going to check other features From 715896fced076bddd44f10e4645f22d42c8a3401 Mon Sep 17 00:00:00 2001 From: wklken Date: Sat, 19 Feb 2022 23:42:21 +0800 Subject: [PATCH 10/11] Create go.yml --- .github/workflows/go.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..b7d02e7 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,22 @@ +name: Go + +on: + push: + branches: [ master, develop ] + pull_request: + branches: [ master, develop ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Test + run: go test -v ./... From 3c00f6e9c3abb7330c4b1981ec3235c4d1b761d4 Mon Sep 17 00:00:00 2001 From: wklken Date: Sun, 20 Feb 2022 00:02:46 +0800 Subject: [PATCH 11/11] chore(version): 1.0.1 --- CHANGELOG.md | 19 +++++++++++++++++++ README.md | 10 ++++++---- gorequest_test.go | 1 - 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f38505..18d0b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,25 @@ GoRequest Changelog ========= +## GoRequest v1.0.1 (2022-02-19) + +### BUGFIXES + +- forceType not working while it's only be changed in getResponseBytes https://github.com/wklken/gorequest/pull/25 + +### ENHANCEMENTS + +- add: UserAgent(), set user-agent by s.UserAgent("") https://github.com/wklken/gorequest/pull/20 +- add: Stats, collecte statistics for SuperAgent request https://github.com/wklken/gorequest/pull/21 +- add: DisableCompression() https://github.com/wklken/gorequest/pull/22 +- add: Mock() support HTTP mocking https://github.com/wklken/gorequest/pull/23 +- add: Timeouts() support http client timeout details https://github.com/wklken/gorequest/pull/27 +- enable custom Content-Type for SendFile https://github.com/wklken/gorequest/pull/26 + +### OTHERS + +- upgrade safeModifyTransport() copy transport to support go 1.16 https://github.com/wklken/gorequest/pull/24 + ## GoRequest v1.0.0 (2022-01-19) ### BUGFIXES diff --git a/README.md b/README.md index 421a9f0..aeefb40 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,13 @@ $ go get github.com/wklken/gorequest ``` ## Documentation -See [Go Doc](http://godoc.org/github.com/wklken/gorequest) or [Go Walker](http://gowalker.org/github.com/wklken/gorequest) for usage and details. + +See [Go Doc](http://godoc.org/github.com/wklken/gorequest) for usage and details. ## Status -[![Drone Build Status](https://drone.io/github.com/jmcvetta/restclient/status.png)](https://drone.io/github.com/parnurzeal/gorequest/latest) -[![Travis Build Status](https://travis-ci.org/parnurzeal/gorequest.svg?branch=master)](https://travis-ci.org/parnurzeal/gorequest) +[![Drone Build Status](https://drone.io/github.com/jmcvetta/restclient/status.png)](https://drone.io/github.com/wklken/gorequest/latest) +[![Travis Build Status](https://travis-ci.org/wklken/gorequest.svg?branch=master)](https://travis-ci.org/wklken/gorequest) ## Why should you use GoRequest? @@ -384,9 +385,10 @@ Thanks to all contributors thus far: | https://github.com/xild | | https://github.com/yangmls | | https://github.com/6david9 | +| https://github.com/wklken | -Also, co-maintainer is needed here. If anyone is interested, please email me (parnurzeal at gmail.com) +Also, co-maintainer is needed here. If anyone is interested, please email me (wklken at gmail.com) ## Credits diff --git a/gorequest_test.go b/gorequest_test.go index 2900550..5a9d3c9 100644 --- a/gorequest_test.go +++ b/gorequest_test.go @@ -2710,7 +2710,6 @@ func TestUserAgent(t *testing.T) { if r.Header.Get("User-Agent") != "gorequest" { t.Error("Expected Header User-Agent-> gorequest", "| but got", r.Header.Get("User-Agent")) } - return })) defer ts.Close()