dump_test.go

Documentation: net/http/httputil

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package httputil
     6  
     7  import (
     8  	"bufio"
     9  	"bytes"
    10  	"context"
    11  	"fmt"
    12  	"io"
    13  	"math/rand"
    14  	"net/http"
    15  	"net/url"
    16  	"runtime"
    17  	"runtime/pprof"
    18  	"strings"
    19  	"testing"
    20  	"time"
    21  )
    22  
    23  type eofReader struct{}
    24  
    25  func (n eofReader) Close() error { return nil }
    26  
    27  func (n eofReader) Read([]byte) (int, error) { return 0, io.EOF }
    28  
    29  type dumpTest struct {
    30  	// Either Req or GetReq can be set/nil but not both.
    31  	Req    *http.Request
    32  	GetReq func() *http.Request
    33  
    34  	Body any // optional []byte or func() io.ReadCloser to populate Req.Body
    35  
    36  	WantDump    string
    37  	WantDumpOut string
    38  	MustError   bool // if true, the test is expected to throw an error
    39  	NoBody      bool // if true, set DumpRequest{,Out} body to false
    40  }
    41  
    42  var dumpTests = []dumpTest{
    43  	// HTTP/1.1 => chunked coding; body; empty trailer
    44  	{
    45  		Req: &http.Request{
    46  			Method: "GET",
    47  			URL: &url.URL{
    48  				Scheme: "http",
    49  				Host:   "www.google.com",
    50  				Path:   "/search",
    51  			},
    52  			ProtoMajor:       1,
    53  			ProtoMinor:       1,
    54  			TransferEncoding: []string{"chunked"},
    55  		},
    56  
    57  		Body: []byte("abcdef"),
    58  
    59  		WantDump: "GET /search HTTP/1.1\r\n" +
    60  			"Host: www.google.com\r\n" +
    61  			"Transfer-Encoding: chunked\r\n\r\n" +
    62  			chunk("abcdef") + chunk(""),
    63  	},
    64  
    65  	// Verify that DumpRequest preserves the HTTP version number, doesn't add a Host,
    66  	// and doesn't add a User-Agent.
    67  	{
    68  		Req: &http.Request{
    69  			Method:     "GET",
    70  			URL:        mustParseURL("/foo"),
    71  			ProtoMajor: 1,
    72  			ProtoMinor: 0,
    73  			Header: http.Header{
    74  				"X-Foo": []string{"X-Bar"},
    75  			},
    76  		},
    77  
    78  		WantDump: "GET /foo HTTP/1.0\r\n" +
    79  			"X-Foo: X-Bar\r\n\r\n",
    80  	},
    81  
    82  	{
    83  		Req: mustNewRequest("GET", "http://example.com/foo", nil),
    84  
    85  		WantDumpOut: "GET /foo HTTP/1.1\r\n" +
    86  			"Host: example.com\r\n" +
    87  			"User-Agent: Go-http-client/1.1\r\n" +
    88  			"Accept-Encoding: gzip\r\n\r\n",
    89  	},
    90  
    91  	// Test that an https URL doesn't try to do an SSL negotiation
    92  	// with a bytes.Buffer and hang with all goroutines not
    93  	// runnable.
    94  	{
    95  		Req: mustNewRequest("GET", "https://example.com/foo", nil),
    96  		WantDumpOut: "GET /foo HTTP/1.1\r\n" +
    97  			"Host: example.com\r\n" +
    98  			"User-Agent: Go-http-client/1.1\r\n" +
    99  			"Accept-Encoding: gzip\r\n\r\n",
   100  	},
   101  
   102  	// Request with Body, but Dump requested without it.
   103  	{
   104  		Req: &http.Request{
   105  			Method: "POST",
   106  			URL: &url.URL{
   107  				Scheme: "http",
   108  				Host:   "post.tld",
   109  				Path:   "/",
   110  			},
   111  			ContentLength: 6,
   112  			ProtoMajor:    1,
   113  			ProtoMinor:    1,
   114  		},
   115  
   116  		Body: []byte("abcdef"),
   117  
   118  		WantDumpOut: "POST / HTTP/1.1\r\n" +
   119  			"Host: post.tld\r\n" +
   120  			"User-Agent: Go-http-client/1.1\r\n" +
   121  			"Content-Length: 6\r\n" +
   122  			"Accept-Encoding: gzip\r\n\r\n",
   123  
   124  		NoBody: true,
   125  	},
   126  
   127  	// Request with Body > 8196 (default buffer size)
   128  	{
   129  		Req: &http.Request{
   130  			Method: "POST",
   131  			URL: &url.URL{
   132  				Scheme: "http",
   133  				Host:   "post.tld",
   134  				Path:   "/",
   135  			},
   136  			Header: http.Header{
   137  				"Content-Length": []string{"8193"},
   138  			},
   139  
   140  			ContentLength: 8193,
   141  			ProtoMajor:    1,
   142  			ProtoMinor:    1,
   143  		},
   144  
   145  		Body: bytes.Repeat([]byte("a"), 8193),
   146  
   147  		WantDumpOut: "POST / HTTP/1.1\r\n" +
   148  			"Host: post.tld\r\n" +
   149  			"User-Agent: Go-http-client/1.1\r\n" +
   150  			"Content-Length: 8193\r\n" +
   151  			"Accept-Encoding: gzip\r\n\r\n" +
   152  			strings.Repeat("a", 8193),
   153  		WantDump: "POST / HTTP/1.1\r\n" +
   154  			"Host: post.tld\r\n" +
   155  			"Content-Length: 8193\r\n\r\n" +
   156  			strings.Repeat("a", 8193),
   157  	},
   158  
   159  	{
   160  		GetReq: func() *http.Request {
   161  			return mustReadRequest("GET http://foo.com/ HTTP/1.1\r\n" +
   162  				"User-Agent: blah\r\n\r\n")
   163  		},
   164  		NoBody: true,
   165  		WantDump: "GET http://foo.com/ HTTP/1.1\r\n" +
   166  			"User-Agent: blah\r\n\r\n",
   167  	},
   168  
   169  	// Issue #7215. DumpRequest should return the "Content-Length" when set
   170  	{
   171  		GetReq: func() *http.Request {
   172  			return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" +
   173  				"Host: passport.myhost.com\r\n" +
   174  				"Content-Length: 3\r\n" +
   175  				"\r\nkey1=name1&key2=name2")
   176  		},
   177  		WantDump: "POST /v2/api/?login HTTP/1.1\r\n" +
   178  			"Host: passport.myhost.com\r\n" +
   179  			"Content-Length: 3\r\n" +
   180  			"\r\nkey",
   181  	},
   182  	// Issue #7215. DumpRequest should return the "Content-Length" in ReadRequest
   183  	{
   184  		GetReq: func() *http.Request {
   185  			return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" +
   186  				"Host: passport.myhost.com\r\n" +
   187  				"Content-Length: 0\r\n" +
   188  				"\r\nkey1=name1&key2=name2")
   189  		},
   190  		WantDump: "POST /v2/api/?login HTTP/1.1\r\n" +
   191  			"Host: passport.myhost.com\r\n" +
   192  			"Content-Length: 0\r\n\r\n",
   193  	},
   194  
   195  	// Issue #7215. DumpRequest should not return the "Content-Length" if unset
   196  	{
   197  		GetReq: func() *http.Request {
   198  			return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" +
   199  				"Host: passport.myhost.com\r\n" +
   200  				"\r\nkey1=name1&key2=name2")
   201  		},
   202  		WantDump: "POST /v2/api/?login HTTP/1.1\r\n" +
   203  			"Host: passport.myhost.com\r\n\r\n",
   204  	},
   205  
   206  	// Issue 18506: make drainBody recognize NoBody. Otherwise
   207  	// this was turning into a chunked request.
   208  	{
   209  		Req: mustNewRequest("POST", "http://example.com/foo", http.NoBody),
   210  		WantDumpOut: "POST /foo HTTP/1.1\r\n" +
   211  			"Host: example.com\r\n" +
   212  			"User-Agent: Go-http-client/1.1\r\n" +
   213  			"Content-Length: 0\r\n" +
   214  			"Accept-Encoding: gzip\r\n\r\n",
   215  	},
   216  
   217  	// Issue 34504: a non-nil Body without ContentLength set should be chunked
   218  	{
   219  		Req: &http.Request{
   220  			Method: "PUT",
   221  			URL: &url.URL{
   222  				Scheme: "http",
   223  				Host:   "post.tld",
   224  				Path:   "/test",
   225  			},
   226  			ContentLength: 0,
   227  			Proto:         "HTTP/1.1",
   228  			ProtoMajor:    1,
   229  			ProtoMinor:    1,
   230  			Body:          &eofReader{},
   231  		},
   232  		NoBody: true,
   233  		WantDumpOut: "PUT /test HTTP/1.1\r\n" +
   234  			"Host: post.tld\r\n" +
   235  			"User-Agent: Go-http-client/1.1\r\n" +
   236  			"Transfer-Encoding: chunked\r\n" +
   237  			"Accept-Encoding: gzip\r\n\r\n",
   238  	},
   239  
   240  	// Issue 54616: request with Connection header doesn't result in duplicate header.
   241  	{
   242  		GetReq: func() *http.Request {
   243  			return mustReadRequest("GET / HTTP/1.1\r\n" +
   244  				"Host: example.com\r\n" +
   245  				"Connection: close\r\n\r\n")
   246  		},
   247  		NoBody: true,
   248  		WantDump: "GET / HTTP/1.1\r\n" +
   249  			"Host: example.com\r\n" +
   250  			"Connection: close\r\n\r\n",
   251  	},
   252  }
   253  
   254  func TestDumpRequest(t *testing.T) {
   255  	// Make a copy of dumpTests and add 10 new cases with an empty URL
   256  	// to test that no goroutines are leaked. See golang.org/issue/32571.
   257  	// 10 seems to be a decent number which always triggers the failure.
   258  	dumpTests := dumpTests[:]
   259  	for i := 0; i < 10; i++ {
   260  		dumpTests = append(dumpTests, dumpTest{
   261  			Req:       mustNewRequest("GET", "", nil),
   262  			MustError: true,
   263  		})
   264  	}
   265  	numg0 := runtime.NumGoroutine()
   266  	for i, tt := range dumpTests {
   267  		if tt.Req != nil && tt.GetReq != nil || tt.Req == nil && tt.GetReq == nil {
   268  			t.Errorf("#%d: either .Req(%p) or .GetReq(%p) can be set/nil but not both", i, tt.Req, tt.GetReq)
   269  			continue
   270  		}
   271  
   272  		freshReq := func(ti dumpTest) *http.Request {
   273  			req := ti.Req
   274  			if req == nil {
   275  				req = ti.GetReq()
   276  			}
   277  
   278  			if req.Header == nil {
   279  				req.Header = make(http.Header)
   280  			}
   281  
   282  			if ti.Body == nil {
   283  				return req
   284  			}
   285  			switch b := ti.Body.(type) {
   286  			case []byte:
   287  				req.Body = io.NopCloser(bytes.NewReader(b))
   288  			case func() io.ReadCloser:
   289  				req.Body = b()
   290  			default:
   291  				t.Fatalf("Test %d: unsupported Body of %T", i, ti.Body)
   292  			}
   293  			return req
   294  		}
   295  
   296  		if tt.WantDump != "" {
   297  			req := freshReq(tt)
   298  			dump, err := DumpRequest(req, !tt.NoBody)
   299  			if err != nil {
   300  				t.Errorf("DumpRequest #%d: %s\nWantDump:\n%s", i, err, tt.WantDump)
   301  				continue
   302  			}
   303  			if string(dump) != tt.WantDump {
   304  				t.Errorf("DumpRequest %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDump, string(dump))
   305  				continue
   306  			}
   307  		}
   308  
   309  		if tt.MustError {
   310  			req := freshReq(tt)
   311  			_, err := DumpRequestOut(req, !tt.NoBody)
   312  			if err == nil {
   313  				t.Errorf("DumpRequestOut #%d: expected an error, got nil", i)
   314  			}
   315  			continue
   316  		}
   317  
   318  		if tt.WantDumpOut != "" {
   319  			req := freshReq(tt)
   320  			dump, err := DumpRequestOut(req, !tt.NoBody)
   321  			if err != nil {
   322  				t.Errorf("DumpRequestOut #%d: %s", i, err)
   323  				continue
   324  			}
   325  			if string(dump) != tt.WantDumpOut {
   326  				t.Errorf("DumpRequestOut %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDumpOut, string(dump))
   327  				continue
   328  			}
   329  		}
   330  	}
   331  
   332  	// Validate we haven't leaked any goroutines.
   333  	var dg int
   334  	dl := deadline(t, 5*time.Second, time.Second)
   335  	for time.Now().Before(dl) {
   336  		if dg = runtime.NumGoroutine() - numg0; dg <= 4 {
   337  			// No unexpected goroutines.
   338  			return
   339  		}
   340  
   341  		// Allow goroutines to schedule and die off.
   342  		runtime.Gosched()
   343  	}
   344  
   345  	buf := make([]byte, 4096)
   346  	buf = buf[:runtime.Stack(buf, true)]
   347  	t.Errorf("Unexpectedly large number of new goroutines: %d new: %s", dg, buf)
   348  }
   349  
   350  // deadline returns the time which is needed before t.Deadline()
   351  // if one is configured and it is s greater than needed in the future,
   352  // otherwise defaultDelay from the current time.
   353  func deadline(t *testing.T, defaultDelay, needed time.Duration) time.Time {
   354  	if dl, ok := t.Deadline(); ok {
   355  		if dl = dl.Add(-needed); dl.After(time.Now()) {
   356  			// Allow an arbitrarily long delay.
   357  			return dl
   358  		}
   359  	}
   360  
   361  	// No deadline configured or its closer than needed from now
   362  	// so just use the default.
   363  	return time.Now().Add(defaultDelay)
   364  }
   365  
   366  func chunk(s string) string {
   367  	return fmt.Sprintf("%x\r\n%s\r\n", len(s), s)
   368  }
   369  
   370  func mustParseURL(s string) *url.URL {
   371  	u, err := url.Parse(s)
   372  	if err != nil {
   373  		panic(fmt.Sprintf("Error parsing URL %q: %v", s, err))
   374  	}
   375  	return u
   376  }
   377  
   378  func mustNewRequest(method, url string, body io.Reader) *http.Request {
   379  	req, err := http.NewRequest(method, url, body)
   380  	if err != nil {
   381  		panic(fmt.Sprintf("NewRequest(%q, %q, %p) err = %v", method, url, body, err))
   382  	}
   383  	return req
   384  }
   385  
   386  func mustReadRequest(s string) *http.Request {
   387  	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(s)))
   388  	if err != nil {
   389  		panic(err)
   390  	}
   391  	return req
   392  }
   393  
   394  var dumpResTests = []struct {
   395  	res  *http.Response
   396  	body bool
   397  	want string
   398  }{
   399  	{
   400  		res: &http.Response{
   401  			Status:        "200 OK",
   402  			StatusCode:    200,
   403  			Proto:         "HTTP/1.1",
   404  			ProtoMajor:    1,
   405  			ProtoMinor:    1,
   406  			ContentLength: 50,
   407  			Header: http.Header{
   408  				"Foo": []string{"Bar"},
   409  			},
   410  			Body: io.NopCloser(strings.NewReader("foo")), // shouldn't be used
   411  		},
   412  		body: false, // to verify we see 50, not empty or 3.
   413  		want: `HTTP/1.1 200 OK
   414  Content-Length: 50
   415  Foo: Bar`,
   416  	},
   417  
   418  	{
   419  		res: &http.Response{
   420  			Status:        "200 OK",
   421  			StatusCode:    200,
   422  			Proto:         "HTTP/1.1",
   423  			ProtoMajor:    1,
   424  			ProtoMinor:    1,
   425  			ContentLength: 3,
   426  			Body:          io.NopCloser(strings.NewReader("foo")),
   427  		},
   428  		body: true,
   429  		want: `HTTP/1.1 200 OK
   430  Content-Length: 3
   431  
   432  foo`,
   433  	},
   434  
   435  	{
   436  		res: &http.Response{
   437  			Status:           "200 OK",
   438  			StatusCode:       200,
   439  			Proto:            "HTTP/1.1",
   440  			ProtoMajor:       1,
   441  			ProtoMinor:       1,
   442  			ContentLength:    -1,
   443  			Body:             io.NopCloser(strings.NewReader("foo")),
   444  			TransferEncoding: []string{"chunked"},
   445  		},
   446  		body: true,
   447  		want: `HTTP/1.1 200 OK
   448  Transfer-Encoding: chunked
   449  
   450  3
   451  foo
   452  0`,
   453  	},
   454  	{
   455  		res: &http.Response{
   456  			Status:        "200 OK",
   457  			StatusCode:    200,
   458  			Proto:         "HTTP/1.1",
   459  			ProtoMajor:    1,
   460  			ProtoMinor:    1,
   461  			ContentLength: 0,
   462  			Header: http.Header{
   463  				// To verify if headers are not filtered out.
   464  				"Foo1": []string{"Bar1"},
   465  				"Foo2": []string{"Bar2"},
   466  			},
   467  			Body: nil,
   468  		},
   469  		body: false, // to verify we see 0, not empty.
   470  		want: `HTTP/1.1 200 OK
   471  Foo1: Bar1
   472  Foo2: Bar2
   473  Content-Length: 0`,
   474  	},
   475  }
   476  
   477  func TestDumpResponse(t *testing.T) {
   478  	for i, tt := range dumpResTests {
   479  		gotb, err := DumpResponse(tt.res, tt.body)
   480  		if err != nil {
   481  			t.Errorf("%d. DumpResponse = %v", i, err)
   482  			continue
   483  		}
   484  		got := string(gotb)
   485  		got = strings.TrimSpace(got)
   486  		got = strings.ReplaceAll(got, "\r", "")
   487  
   488  		if got != tt.want {
   489  			t.Errorf("%d.\nDumpResponse got:\n%s\n\nWant:\n%s\n", i, got, tt.want)
   490  		}
   491  	}
   492  }
   493  
   494  // Issue 38352: Check for deadlock on canceled requests.
   495  func TestDumpRequestOutIssue38352(t *testing.T) {
   496  	if testing.Short() {
   497  		return
   498  	}
   499  	t.Parallel()
   500  
   501  	timeout := 10 * time.Second
   502  	if deadline, ok := t.Deadline(); ok {
   503  		timeout = time.Until(deadline)
   504  		timeout -= time.Second * 2 // Leave 2 seconds to report failures.
   505  	}
   506  	for i := 0; i < 1000; i++ {
   507  		delay := time.Duration(rand.Intn(5)) * time.Millisecond
   508  		ctx, cancel := context.WithTimeout(context.Background(), delay)
   509  		defer cancel()
   510  
   511  		r := bytes.NewBuffer(make([]byte, 10000))
   512  		req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://example.com", r)
   513  		if err != nil {
   514  			t.Fatal(err)
   515  		}
   516  
   517  		out := make(chan error)
   518  		go func() {
   519  			_, err = DumpRequestOut(req, true)
   520  			out <- err
   521  		}()
   522  
   523  		select {
   524  		case <-out:
   525  		case <-time.After(timeout):
   526  			b := &strings.Builder{}
   527  			fmt.Fprintf(b, "deadlock detected on iteration %d after %s with delay: %v\n", i, timeout, delay)
   528  			pprof.Lookup("goroutine").WriteTo(b, 1)
   529  			t.Fatal(b.String())
   530  		}
   531  	}
   532  }
   533
View as plain text