checkpoint fixing data race to process h2 and grpc requests

transport/http_transport.go

@@ -33,6 +33,8 @@ type httpTransportClient struct {
 	once     sync.Once
 
 	sync.RWMutex
+
+	// request must be stored for response processing
 	r    chan *http.Request
 	bl   []*http.Request
 	buff *bufio.Reader
@@ -48,10 +50,18 @@ type httpTransportSocket struct {
 	r  *http.Request
 	rw *bufio.ReadWriter
 
+	mtx sync.RWMutex
+
+	// the hijacked when using http 1
 	conn net.Conn
 	// for the first request
 	ch chan *http.Request
 
+	// h2 things
+	buf *bufio.Reader
+	// indicate if socket is closed
+	closed chan bool
+
 	// local/remote ip
 	local  string
 	remote string
@@ -161,14 +171,13 @@ func (h *httpTransportClient) Recv(m *Message) error {
 }
 
 func (h *httpTransportClient) Close() error {
-	err := h.conn.Close()
 	h.once.Do(func() {
 		h.Lock()
 		h.buff.Reset(nil)
 		h.Unlock()
 		close(h.r)
 	})
-	return err
+	return h.conn.Close()
 }
 
 func (h *httpTransportSocket) Local() string {
@@ -232,14 +241,23 @@ func (h *httpTransportSocket) Recv(m *Message) error {
 		return nil
 	}
 
+	// only process if the socket is open
+	select {
+	case <-h.closed:
+		return io.EOF
+	default:
+		// no op
+	}
+
 	// processing http2 request
 	// read streaming body
 
 	// set max buffer size
-	buf := make([]byte, 4*1024)
+	// TODO: adjustable buffer size
+	buf := make([]byte, 4*1024*1024)
 
 	// read the request body
-	n, err := h.r.Body.Read(buf)
+	n, err := h.buf.Read(buf)
 	// not an eof error
 	if err != nil {
 		return err
@@ -290,7 +308,13 @@ func (h *httpTransportSocket) Send(m *Message) error {
 		return rsp.Write(h.conn)
 	}
 
-	// http2 request
+	// only process if the socket is open
+	select {
+	case <-h.closed:
+		return io.EOF
+	default:
+		// no op
+	}
 
 	// set headers
 	for k, v := range m.Header {
@@ -299,6 +323,7 @@ func (h *httpTransportSocket) Send(m *Message) error {
 
 	// write request
 	_, err := h.w.Write(m.Body)
+
 	return err
 }
 
@@ -321,13 +346,26 @@ func (h *httpTransportSocket) error(m *Message) error {
 
 		return rsp.Write(h.conn)
 	}
+
 	return nil
 }
 
 func (h *httpTransportSocket) Close() error {
-	if h.r.ProtoMajor == 1 {
-		return h.conn.Close()
+	h.mtx.Lock()
+	defer h.mtx.Unlock()
+	select {
+	case <-h.closed:
+		return nil
+	default:
+		// close the channel
+		close(h.closed)
+
+		// close the connection
+		if h.r.ProtoMajor == 1 {
+			return h.conn.Close()
+		}
 	}
+
 	return nil
 }
 
@@ -374,20 +412,32 @@ func (h *httpTransportListener) Accept(fn func(Socket)) error {
 			con = conn
 		}
 
+		// buffered reader
+		bufr := bufio.NewReader(r.Body)
+
 		// save the request
 		ch := make(chan *http.Request, 1)
 		ch <- r
 
-		fn(&httpTransportSocket{
+		// create a new transport socket
+		sock := &httpTransportSocket{
 			ht:     h.ht,
 			w:      w,
 			r:      r,
 			rw:     buf,
+			buf:    bufr,
 			ch:     ch,
 			conn:   con,
 			local:  h.Addr(),
 			remote: r.RemoteAddr,
-		})
+			closed: make(chan bool),
+		}
+
+		// cleanup
+		//defer sock.Close()
+
+		// execute the socket
+		fn(sock)
 	})
 
 	// get optional handlers