HTTP Transport auth improvement in progress, Basic and NTLM working, Digest pending

psiphon/upstreamproxy/auth_basic.go

@@ -33,19 +33,23 @@ const (
 )
 
 type BasicHttpAuthenticator struct {
-	state BasicHttpAuthState
+	state    BasicHttpAuthState
+	username string
+	password string
 }
 
-func newBasicAuthenticator() *BasicHttpAuthenticator {
-	return &BasicHttpAuthenticator{state: BASIC_HTTP_AUTH_STATE_CHALLENGE_RECEIVED}
+func newBasicAuthenticator(username, password string) *BasicHttpAuthenticator {
+	return &BasicHttpAuthenticator{
+		state:    BASIC_HTTP_AUTH_STATE_CHALLENGE_RECEIVED,
+		username: username,
+		password: password,
+	}
 }
 
-func (a *BasicHttpAuthenticator) Authenticate(req *http.Request, resp *http.Response, username, password string) error {
+func (a *BasicHttpAuthenticator) Authenticate(req *http.Request, resp *http.Response) error {
 	if a.state == BASIC_HTTP_AUTH_STATE_CHALLENGE_RECEIVED {
-		auth := username + ":" + password
-		req.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
 		a.state = BASIC_HTTP_AUTH_STATE_RESPONSE_GENERATED
-		return nil
+		return a.PreAuthenticate(req)
 	} else {
 		return proxyError(fmt.Errorf("Authorization is not accepted by the proxy server"))
 	}
@@ -54,3 +58,17 @@ func (a *BasicHttpAuthenticator) Authenticate(req *http.Request, resp *http.Resp
 func (a *BasicHttpAuthenticator) IsConnectionBased() bool {
 	return false
 }
+
+func (a *BasicHttpAuthenticator) IsComplete() bool {
+	return a.state == BASIC_HTTP_AUTH_STATE_RESPONSE_GENERATED
+}
+
+func (a *BasicHttpAuthenticator) Reset() {
+	a.state = BASIC_HTTP_AUTH_STATE_CHALLENGE_RECEIVED
+}
+
+func (a *BasicHttpAuthenticator) PreAuthenticate(req *http.Request) error {
+	auth := a.username + ":" + a.password
+	req.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
+	return nil
+}

psiphon/upstreamproxy/auth_digest.go

@@ -36,11 +36,17 @@ const (
 )
 
 type DigestHttpAuthenticator struct {
-	state DigestHttpAuthState
+	state    DigestHttpAuthState
+	username string
+	password string
 }
 
-func newDigestAuthenticator() *DigestHttpAuthenticator {
-	return &DigestHttpAuthenticator{state: DIGEST_HTTP_AUTH_STATE_CHALLENGE_RECEIVED}
+func newDigestAuthenticator(username, password string) *DigestHttpAuthenticator {
+	return &DigestHttpAuthenticator{
+		state:    DIGEST_HTTP_AUTH_STATE_CHALLENGE_RECEIVED,
+		username: username,
+		password: password,
+	}
 }
 
 /* Adapted from https://github.com/ryanjdew/http-digest-auth-client */
@@ -124,7 +130,7 @@ func h(data string) string {
 	return fmt.Sprintf("%x", digest.Sum(nil))
 }
 
-func (a *DigestHttpAuthenticator) Authenticate(req *http.Request, resp *http.Response, username, password string) error {
+func (a *DigestHttpAuthenticator) Authenticate(req *http.Request, resp *http.Response) error {
 	if a.state != DIGEST_HTTP_AUTH_STATE_CHALLENGE_RECEIVED {
 		return proxyError(fmt.Errorf("Authorization is not accepted by the proxy server"))
 	}
@@ -168,8 +174,8 @@ func (a *DigestHttpAuthenticator) Authenticate(req *http.Request, resp *http.Res
 		d.Algorithm = digestParams["algorithm"]
 	}
 	d.Nc = 0x0
-	d.Username = username
-	d.Password = password
+	d.Username = a.username
+	d.Password = a.password
 	d.ApplyAuth(req)
 	a.state = DIGEST_HTTP_AUTH_STATE_RESPONSE_GENERATED
 	return nil
@@ -178,3 +184,15 @@ func (a *DigestHttpAuthenticator) Authenticate(req *http.Request, resp *http.Res
 func (a *DigestHttpAuthenticator) IsConnectionBased() bool {
 	return false
 }
+
+func (a *DigestHttpAuthenticator) IsComplete() bool {
+	return a.state == DIGEST_HTTP_AUTH_STATE_RESPONSE_GENERATED
+}
+
+func (a *DigestHttpAuthenticator) Reset() {
+	a.state = DIGEST_HTTP_AUTH_STATE_CHALLENGE_RECEIVED
+}
+
+func (a *DigestHttpAuthenticator) PreAuthenticate(req *http.Request) error {
+	return nil
+}

psiphon/upstreamproxy/auth_ntlm.go

@@ -36,14 +36,20 @@ const (
 )
 
 type NTLMHttpAuthenticator struct {
-	state NTLMHttpAuthState
+	state    NTLMHttpAuthState
+	username string
+	password string
 }
 
-func newNTLMAuthenticator() *NTLMHttpAuthenticator {
-	return &NTLMHttpAuthenticator{state: NTLM_HTTP_AUTH_STATE_CHALLENGE_RECEIVED}
+func newNTLMAuthenticator(username, password string) *NTLMHttpAuthenticator {
+	return &NTLMHttpAuthenticator{
+		state:    NTLM_HTTP_AUTH_STATE_CHALLENGE_RECEIVED,
+		username: username,
+		password: password,
+	}
 }
 
-func (a *NTLMHttpAuthenticator) Authenticate(req *http.Request, resp *http.Response, username, password string) error {
+func (a *NTLMHttpAuthenticator) Authenticate(req *http.Request, resp *http.Response) error {
 	if a.state == NTLM_HTTP_AUTH_STATE_RESPONSE_TYPE3_GENERATED {
 		return proxyError(fmt.Errorf("Authorization is not accepted by the proxy server"))
 	}
@@ -78,19 +84,19 @@ func (a *NTLMHttpAuthenticator) Authenticate(req *http.Request, resp *http.Respo
 	} else if a.state == NTLM_HTTP_AUTH_STATE_RESPONSE_TYPE1_GENERATED {
 		// Parse username for domain in form DOMAIN\username
 		var NTDomain, NTUser string
-		parts := strings.SplitN(username, "\\", 2)
+		parts := strings.SplitN(a.username, "\\", 2)
 		if len(parts) == 2 {
 			NTDomain = parts[0]
 			NTUser = parts[1]
 		} else {
 			NTDomain = ""
-			NTUser = username
+			NTUser = a.username
 		}
 		challengeBytes, err := base64.StdEncoding.DecodeString(challenge)
 		if err != nil {
 			return proxyError(fmt.Errorf("NTLM challeenge base 64 decoding: %v", err))
 		}
-		session.SetUserInfo(NTUser, password, NTDomain)
+		session.SetUserInfo(NTUser, a.password, NTDomain)
 		ntlmChallenge, err := ntlm.ParseChallengeMessage(challengeBytes)
 		if err != nil {
 			return proxyError(err)
@@ -112,3 +118,15 @@ func (a *NTLMHttpAuthenticator) Authenticate(req *http.Request, resp *http.Respo
 func (a *NTLMHttpAuthenticator) IsConnectionBased() bool {
 	return true
 }
+
+func (a *NTLMHttpAuthenticator) IsComplete() bool {
+	return a.state == NTLM_HTTP_AUTH_STATE_RESPONSE_TYPE3_GENERATED
+}
+
+func (a *NTLMHttpAuthenticator) Reset() {
+	a.state = NTLM_HTTP_AUTH_STATE_CHALLENGE_RECEIVED
+}
+
+func (a *NTLMHttpAuthenticator) PreAuthenticate(req *http.Request) error {
+	return nil
+}

psiphon/upstreamproxy/http_authenticator.go

@@ -35,8 +35,11 @@ const (
 )
 
 type HttpAuthenticator interface {
-	Authenticate(req *http.Request, resp *http.Response, username, pasword string) error
+	PreAuthenticate(req *http.Request) error
+	Authenticate(req *http.Request, resp *http.Response) error
 	IsConnectionBased() bool
+	IsComplete() bool
+	Reset()
 }
 
 func parseAuthChallenge(resp *http.Response) (map[string]string, error) {
@@ -58,7 +61,7 @@ func parseAuthChallenge(resp *http.Response) (map[string]string, error) {
 	return challenges, nil
 }
 
-func NewHttpAuthenticator(resp *http.Response) (HttpAuthenticator, error) {
+func NewHttpAuthenticator(resp *http.Response, username, password string) (HttpAuthenticator, error) {
 
 	challenges, err := parseAuthChallenge(resp)
 	if err != nil {
@@ -68,11 +71,11 @@ func NewHttpAuthenticator(resp *http.Response) (HttpAuthenticator, error) {
 
 	// NTLM > Digest > Basic
 	if _, ok := challenges["NTLM"]; ok {
-		return newNTLMAuthenticator(), nil
+		return newNTLMAuthenticator(username, password), nil
 	} else if _, ok := challenges["Digest"]; ok {
-		return newDigestAuthenticator(), nil
+		return newDigestAuthenticator(username, password), nil
 	} else if _, ok := challenges["Basic"]; ok {
-		return newBasicAuthenticator(), nil
+		return newBasicAuthenticator(username, password), nil
 	}
 
 	//Unsupported scheme

psiphon/upstreamproxy/proxy_http.go

@@ -145,7 +145,7 @@ func (pc *proxyConn) handshake(addr, username, password string) error {
 	req.Header.Set("User-Agent", "")
 
 	if pc.authState == HTTP_AUTH_STATE_CHALLENGED {
-		err := pc.authenticator.Authenticate(req, pc.authResponse, username, password)
+		err := pc.authenticator.Authenticate(req, pc.authResponse)
 		if err != nil {
 			pc.authState = HTTP_AUTH_STATE_FAILURE
 			//Already wrapped in proxyError
@@ -169,7 +169,7 @@ func (pc *proxyConn) handshake(addr, username, password string) error {
 	if resp.StatusCode == 407 {
 		if pc.authState == HTTP_AUTH_STATE_UNCHALLENGED {
 			var auth_err error = nil
-			pc.authenticator, auth_err = NewHttpAuthenticator(resp)
+			pc.authenticator, auth_err = NewHttpAuthenticator(resp, username, password)
 			if auth_err != nil {
 				pc.httpClientConn.Close()
 				pc.authState = HTTP_AUTH_STATE_FAILURE

psiphon/upstreamproxy/transport_proxy_auth.go

@@ -28,6 +28,7 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"sync"
 )
 
 const HTTP_STAT_LINE_LENGTH = 12
@@ -37,9 +38,12 @@ const HTTP_STAT_LINE_LENGTH = 12
 // when requested by server
 type ProxyAuthTransport struct {
 	*http.Transport
-	Dial     DialFunc
-	Username string
-	Password string
+	Dial          DialFunc
+	Username      string
+	Password      string
+	Authenticator HttpAuthenticator
+	authState     HttpAuthState
+	mu            sync.Mutex
 }
 
 func NewProxyAuthTransport(rawTransport *http.Transport) (*ProxyAuthTransport, error) {
@@ -47,10 +51,11 @@ func NewProxyAuthTransport(rawTransport *http.Transport) (*ProxyAuthTransport, e
 	if dialFn == nil {
 		dialFn = net.Dial
 	}
-	tr := &ProxyAuthTransport{Dial: dialFn}
+	tr := &ProxyAuthTransport{Dial: dialFn, authState: HTTP_AUTH_STATE_UNCHALLENGED}
 	proxyUrlFn := rawTransport.Proxy
 	if proxyUrlFn != nil {
 		wrappedDialFn := tr.wrapTransportDial()
+		rawTransport.Dial = wrappedDialFn
 		proxyUrl, err := proxyUrlFn(nil)
 		if err != nil {
 			return nil, err
@@ -60,18 +65,79 @@ func NewProxyAuthTransport(rawTransport *http.Transport) (*ProxyAuthTransport, e
 		}
 		tr.Username = proxyUrl.User.Username()
 		tr.Password, _ = proxyUrl.User.Password()
-		rawTransport.Dial = wrappedDialFn
+		// strip username and password from the proxyURL because
+		// we do not want the wrapped transport to handle authentication
+		proxyUrl.User = nil
+		rawTransport.Proxy = http.ProxyURL(proxyUrl)
 	}
 
 	tr.Transport = rawTransport
 	return tr, nil
 }
 
+func (tr *ProxyAuthTransport) preAuthenticateRequest(req *http.Request) error {
+	if tr.Authenticator == nil {
+		return nil
+	}
+	return tr.Authenticator.PreAuthenticate(req)
+}
+
 func (tr *ProxyAuthTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
 	if req.URL.Scheme != "http" {
 		return nil, fmt.Errorf("Only plain HTTP supported, for HTTPS use http.Transport with DialTLS & upstreamproxy.NewProxyDialFunc")
 	}
-	return tr.Transport.RoundTrip(req)
+	err = tr.preAuthenticateRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	//Clone request early because RoundTrip will destroy request Body
+	var ha HttpAuthenticator = nil
+	newReq := cloneRequest(req)
+	//authState := HTTP_AUTH_STATE_UNCHALLENGED
+
+	resp, err = tr.Transport.RoundTrip(newReq)
+
+	if err != nil {
+		return resp, proxyError(err)
+	}
+
+	if resp.StatusCode == 407 {
+		fmt.Println("407!")
+		tr.mu.Lock()
+		defer tr.mu.Unlock()
+		if tr.Authenticator == nil {
+			ha, err = NewHttpAuthenticator(resp, tr.Username, tr.Password)
+			if err != nil {
+				return nil, err
+			}
+			if ha.IsConnectionBased() {
+				return nil, proxyError(fmt.Errorf("Connection based auth was not handled by transportConn"))
+			}
+			tr.Authenticator = ha
+		}
+	authenticationLoop:
+		for {
+			newReq = cloneRequest(req)
+			err = tr.Authenticator.Authenticate(newReq, resp)
+			if err != nil {
+				return nil, err
+			}
+			resp, err = tr.Transport.RoundTrip(newReq)
+
+			if err != nil {
+				return resp, proxyError(err)
+			}
+			if resp.StatusCode != 407 {
+				if tr.Authenticator != nil && tr.Authenticator.IsComplete() {
+					tr.Authenticator.Reset()
+				}
+				break authenticationLoop
+			}
+		}
+	}
+	return resp, err
+
 }
 
 // wrapTransportDial wraps original transport Dial function
@@ -89,6 +155,28 @@ func (tr *ProxyAuthTransport) wrapTransportDial() DialFunc {
 	}
 }
 
+// cloneRequest returns a clone of the provided *http.Request. The clone is a
+// shallow copy of the struct and its Header map.
+func cloneRequest(r *http.Request) *http.Request {
+	// shallow copy of the struct
+	r2 := new(http.Request)
+	*r2 = *r
+	// deep copy of the Header
+	r2.Header = make(http.Header)
+	for k, s := range r.Header {
+		r2.Header[k] = s
+	}
+
+	if r.Body != nil {
+		body, _ := ioutil.ReadAll(r.Body)
+		defer r.Body.Close()
+		//restore original request Body
+		r.Body = ioutil.NopCloser(bytes.NewReader(body))
+		r2.Body = ioutil.NopCloser(bytes.NewReader(body))
+	}
+	return r2
+}
+
 type transportConn struct {
 	net.Conn
 	requestInterceptor io.Writer
@@ -98,8 +186,8 @@ type transportConn struct {
 	lastRequest   *http.Request
 	authenticator HttpAuthenticator
 	authState     HttpAuthState
-	authCache     string
 	transport     *ProxyAuthTransport
+	//mutex         *sync.Mutex
 }
 
 func newTransportConn(c net.Conn, tr *ProxyAuthTransport) *transportConn {
@@ -139,8 +227,8 @@ func newTransportConn(c net.Conn, tr *ProxyAuthTransport) *transportConn {
 
 // Read peeks into the new response and checks if the proxy requests authentication
 // If so, the last intercepted request is authenticated against the response
-func (tc *transportConn) Read(p []byte) (n int, err error) {
-	n, err = tc.Conn.Read(p)
+func (tc *transportConn) Read(p []byte) (n int, read_err error) {
+	n, read_err = tc.Conn.Read(p)
 	if n < HTTP_STAT_LINE_LENGTH {
 		return
 	}
@@ -150,24 +238,39 @@ func (tc *transportConn) Read(p []byte) (n int, err error) {
 		//This is a new response
 		//Let's see if proxy requests authentication
 		f := strings.SplitN(line, " ", 2)
-		readBufferReader := bytes.NewReader(p)
-		responseReader := io.MultiReader(readBufferReader, tc.Conn)
+
+		readBufferReader := io.NewSectionReader(bytes.NewReader(p), 0, int64(n))
+		responseReader := bufio.NewReader(readBufferReader)
 		if (f[0] == "HTTP/1.0" || f[0] == "HTTP/1.1") && f[1] == "407" {
-			resp, err := http.ReadResponse(bufio.NewReader(responseReader), nil)
+			resp, err := http.ReadResponse(responseReader, nil)
 			if err != nil {
 				return 0, err
 			}
-			// make sure we read the body of the response so that
-			// we don't block the reader
+			ha, err := NewHttpAuthenticator(resp, tc.transport.Username, tc.transport.Password)
+			if err != nil {
+				return 0, err
+			}
+			// If connection based auth is requested, we are going to
+			// authenticate this very connection
+
+			if !ha.IsConnectionBased() {
+				return
+			}
+
+			// Drain the rest of the response
+			// in order to perform auth handshake
+			readBufferReader.Seek(0, 0)
+			responseReader = bufio.NewReader(io.MultiReader(readBufferReader, tc.Conn))
+			resp, err = http.ReadResponse(responseReader, nil)
+			if err != nil {
+				return 0, err
+			}
+
 			ioutil.ReadAll(resp.Body)
 			resp.Body.Close()
 
-			if tc.authState == HTTP_AUTH_STATE_UNCHALLENGED {
-				tc.authenticator, err = NewHttpAuthenticator(resp)
-				if err != nil {
-					return 0, err
-				}
-				tc.authState = HTTP_AUTH_STATE_CHALLENGED
+			if tc.authenticator == nil {
+				tc.authenticator = ha
 			}
 
 			if resp.Close == true {
@@ -181,15 +284,15 @@ func (tc *transportConn) Read(p []byte) (n int, err error) {
 				}
 			}
 
-			// Authenticate and replay the request
-			err = tc.authenticator.Authenticate(tc.lastRequest, resp, tc.transport.Username, tc.transport.Password)
+			// Authenticate and replay the request on the connection
+			err = tc.authenticator.Authenticate(tc.lastRequest, resp)
 			if err != nil {
 				return 0, err
 			}
 			tc.lastRequest.WriteProxy(tc)
 			return tc.Read(p)
 		}
-	case err = <-tc.errChannel:
+	case err := <-tc.errChannel:
 		return 0, err
 	default:
 	}