Goのnet/http.Client.Doの内部実装をたどったメモ

golang
package main

import (
        "fmt"
        "net/http"
        "io/ioutil"
)

var client = http.Client{}

func main() {

        req, err := http.NewRequest("GET", "http://example.com", nil)
        if err != nil{
                panic(err)
        }

        resp, err := client.Do(req)
        if err != nil{
                panic(err)
        }
        defer resp.Body.Close()

        body, err := ioutil.ReadAll(resp.Body)
        if err != nil{
                panic(err)
        }

        fmt.Println(string(body))
}

Client

TransportがTCPコネクションをキャッシュするのでClientは使い回すべき。複数のgoroutineでコンカレントに使っても大丈夫。

https://github.com/golang/go/blob/964639cc338db650ccadeafb7424bc8ebb2c0f6c/src/net/http/client.go#L36

type Client struct {

        // nilならDefaultTransportが使われる        
    	Transport RoundTripper

        // nilなら10回で止まる
        CheckRedirect func(req *Request, via []*Request) error

        // nilならcookieは無視される
        Jar CookieJar

        Timeout time.Duration
}

MaxIdleConnsとは別に、ホストごとの制限がある。

Goのnet/httpとKeep-Alive - sambaiz.net

https://github.com/golang/go/blob/d986daec1375527ef78cd59d81d42be7406a9803/src/net/http/transport.go#L39

var DefaultTransport RoundTripper = &Transport{
	Proxy: ProxyFromEnvironment,
	DialContext: (&net.Dialer{
		Timeout:   30 * time.Second,
		KeepAlive: 30 * time.Second,
		DualStack: true,
	}).DialContext,
	MaxIdleConns:          100,
	IdleConnTimeout:       90 * time.Second,
	TLSHandshakeTimeout:   10 * time.Second,
	ExpectContinueTimeout: 1 * time.Second,
}

const DefaultMaxIdleConnsPerHost = 2

Request

https://github.com/golang/go/blob/964639cc338db650ccadeafb7424bc8ebb2c0f6c/src/net/http/request.go#L690

func NewRequest(method, urlStr string, body io.Reader) (*Request, error) {
    ...
    
    u, err := url.Parse(urlStr)
    
    ...
    
    rc, ok := body.(io.ReadCloser)
    req := &Request{
            Method:     method,
            URL:        u,
            Proto:      "HTTP/1.1",
            ProtoMajor: 1,
            ProtoMinor: 1,
            Header:     make(Header),
            Body:       rc,
            Host:       u.Host,
        }

Do

https://github.com/golang/go/blob/964639cc338db650ccadeafb7424bc8ebb2c0f6c/src/net/http/client.go#L181

Requestを渡してResponseを受け取る。

func (c *Client) Do(req *Request) (*Response, error) {
    method := valueOrDefault(req.Method, "GET")
    if method == "GET" || method == "HEAD" {
        return c.doFollowingRedirects(req, shouldRedirectGet)
    }
    if method == "POST" || method == "PUT" {
        return c.doFollowingRedirects(req, shouldRedirectPost)
    }
    return c.send(req, c.deadline())
}

doFollowingRedirects

リクエストを送り、リダイレクトする場合はして、そうでない場合はレスポンスを返す。

https://github.com/golang/go/blob/964639cc338db650ccadeafb7424bc8ebb2c0f6c/src/net/http/client.go#L440

func (c *Client) doFollowingRedirects(req *Request, shouldRedirect func(int) bool) (*Response, error) {
    ...

    for{

        ...
        
        if resp, err = c.send(req, deadline); err != nil {
            if !deadline.IsZero() && !time.Now().Before(deadline) {
                err = &httpError{
                    err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
                    timeout: true,
                }
            }
            return nil, uerr(err)
        }

        if !shouldRedirect(resp.StatusCode) {
            return resp, nil
        }
    }

send

ClientのTransportのRoundTripを呼ぶ。ここからはTransport(RoundTripper)の仕事。

https://github.com/golang/go/blob/964639cc338db650ccadeafb7424bc8ebb2c0f6c/src/net/http/client.go#L140

func (c *Client) send(req *Request, deadline time.Time) (*Response, error) {
    
    ...
    
    resp, err := send(req, c.transport(), deadline)

https://github.com/golang/go/blob/964639cc338db650ccadeafb7424bc8ebb2c0f6c/src/net/http/client.go#L208

func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) {
   
    ...
    
    resp, err := rt.RoundTrip(req)

RoundTrip

チャネルを通して接続先とやりとりする。

https://github.com/golang/go/blob/d986daec1375527ef78cd59d81d42be7406a9803/src/net/http/transport.go#L319

func (t *Transport) RoundTrip(req *Request) (*Response, error) {
    
    ...
    
    treq := &transportRequest{Request: req, trace: trace}
    
    ...
	
    cm, err := t.connectMethodForRequest(treq)
    pconn, err := t.getConn(treq, cm)
   
    ...
    
    resp, err = pconn.roundTrip(treq)

writeとreadを同時に行っているのはサーバーがbodyのすべてを読む前にレスポンスを返すときのため。

https://github.com/golang/go/blob/d986daec1375527ef78cd59d81d42be7406a9803/src/net/http/transport.go#L1823

func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
    
    ...
    
    pc.writech <- writeRequest{req, writeErrCh, continueCh} // pc.writeLoopで読まれる

    resc := make(chan responseAndError)
	pc.reqch <- requestAndChan{ // pc.readLoopで読まれる
		req:        req.Request,
		ch:         resc,
		addedGzip:  requestedGzip,
		continueCh: continueCh,
		callerGone: gone,
	}
    var re responseAndError
    
    ...
    
    case re = <-resc: // pc.readLoopで書き込まれる
		re.err = pc.mapRoundTripErrorFromReadLoop(req.Request, startBytesWritten, re.err)
		break WaitResponse

pconn.getConn/dialConn

接続し、チャネルを読むループ(readLoop, writeLoop)を回す。

https://github.com/golang/go/blob/d986daec1375527ef78cd59d81d42be7406a9803/src/net/http/transport.go#L865

func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) {
    ...
    type dialRes struct {
		pc  *persistConn
		err error
	}
    dialc := make(chan dialRes)
    
    ...
    
    go func() {
		pc, err := t.dialConn(ctx, cm)
		dialc <- dialRes{pc, err}
	}()
    
    ...
    
    select {
	case v := <-dialc:
        if v.pc != nil {
            
            ...
			
            return v.pc, nil
        }

https://github.com/golang/go/blob/d986daec1375527ef78cd59d81d42be7406a9803/src/net/http/transport.go#L967

func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) {
    pconn := &persistConn{
		t:             t,
		cacheKey:      cm.key(),
		reqch:         make(chan requestAndChan, 1), // roundTripで書かれて、readLoopで読まれる
		writech:       make(chan writeRequest, 1), // roundTripで書かれて、writeLoopで読まれる
		closech:       make(chan struct{}),
		writeErrCh:    make(chan error, 1),
		writeLoopDone: make(chan struct{}),
	}

    ...
    
    conn, err := t.dial(ctx, "tcp", cm.addr())
	pconn.conn = conn
    
    ...
    
    go pconn.readLoop()
	go pconn.writeLoop()
	return pconn, nil
}