meek.go 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538
  1. /*
  2. * Copyright (c) 2016, Psiphon Inc.
  3. * All rights reserved.
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation, either version 3 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. *
  18. */
  19. package server
  20. import (
  21. "bytes"
  22. "context"
  23. "crypto/rand"
  24. "crypto/tls"
  25. "encoding/base64"
  26. "encoding/hex"
  27. "encoding/json"
  28. std_errors "errors"
  29. "hash/crc64"
  30. "io"
  31. "net"
  32. "net/http"
  33. "runtime"
  34. "strconv"
  35. "strings"
  36. "sync"
  37. "sync/atomic"
  38. "time"
  39. "github.com/Psiphon-Labs/goarista/monotime"
  40. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
  41. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
  42. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/obfuscator"
  43. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
  44. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
  45. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/values"
  46. tris "github.com/Psiphon-Labs/tls-tris"
  47. "golang.org/x/crypto/nacl/box"
  48. )
  49. // MeekServer is based on meek-server.go from Tor and Psiphon:
  50. //
  51. // https://gitweb.torproject.org/pluggable-transports/meek.git/blob/HEAD:/meek-client/meek-client.go
  52. // CC0 1.0 Universal
  53. //
  54. // https://bitbucket.org/psiphon/psiphon-circumvention-system/src/default/go/meek-client/meek-client.go
  55. const (
  56. // Protocol version 1 clients can handle arbitrary length response bodies. Older clients
  57. // report no version number and expect at most 64K response bodies.
  58. MEEK_PROTOCOL_VERSION_1 = 1
  59. // Protocol version 2 clients initiate a session by sending an encrypted and obfuscated meek
  60. // cookie with their initial HTTP request. Connection information is contained within the
  61. // encrypted cookie payload. The server inspects the cookie and establishes a new session and
  62. // returns a new random session ID back to client via Set-Cookie header. The client uses this
  63. // session ID on all subsequent requests for the remainder of the session.
  64. MEEK_PROTOCOL_VERSION_2 = 2
  65. // Protocol version 3 clients include resiliency enhancements and will add a Range header
  66. // when retrying a request for a partially downloaded response payload.
  67. MEEK_PROTOCOL_VERSION_3 = 3
  68. MEEK_MAX_REQUEST_PAYLOAD_LENGTH = 65536
  69. MEEK_TURN_AROUND_TIMEOUT = 20 * time.Millisecond
  70. MEEK_EXTENDED_TURN_AROUND_TIMEOUT = 100 * time.Millisecond
  71. MEEK_MAX_SESSION_STALENESS = 45 * time.Second
  72. MEEK_HTTP_CLIENT_IO_TIMEOUT = 45 * time.Second
  73. MEEK_MIN_SESSION_ID_LENGTH = 8
  74. MEEK_MAX_SESSION_ID_LENGTH = 20
  75. MEEK_DEFAULT_RESPONSE_BUFFER_LENGTH = 65536
  76. MEEK_DEFAULT_POOL_BUFFER_LENGTH = 65536
  77. MEEK_DEFAULT_POOL_BUFFER_COUNT = 2048
  78. )
  79. // MeekServer implements the meek protocol, which tunnels TCP traffic (in the case of Psiphon,
  80. // Obfuscated SSH traffic) over HTTP. Meek may be fronted (through a CDN) or direct and may be
  81. // HTTP or HTTPS.
  82. //
  83. // Upstream traffic arrives in HTTP request bodies and downstream traffic is sent in response
  84. // bodies. The sequence of traffic for a given flow is associated using a session ID that's
  85. // set as a HTTP cookie for the client to submit with each request.
  86. //
  87. // MeekServer hooks into TunnelServer via the net.Conn interface by transforming the
  88. // HTTP payload traffic for a given session into net.Conn conforming Read()s and Write()s via
  89. // the meekConn struct.
  90. type MeekServer struct {
  91. support *SupportServices
  92. listener net.Listener
  93. listenerTunnelProtocol string
  94. listenerPort int
  95. passthroughAddress string
  96. tlsConfig *tris.Config
  97. obfuscatorSeedHistory *obfuscator.SeedHistory
  98. clientHandler func(clientTunnelProtocol string, clientConn net.Conn)
  99. openConns *common.Conns
  100. stopBroadcast <-chan struct{}
  101. sessionsLock sync.RWMutex
  102. sessions map[string]*meekSession
  103. checksumTable *crc64.Table
  104. bufferPool *CachedResponseBufferPool
  105. rateLimitLock sync.Mutex
  106. rateLimitHistory map[string][]time.Time
  107. rateLimitCount int
  108. rateLimitSignalGC chan struct{}
  109. }
  110. // NewMeekServer initializes a new meek server.
  111. func NewMeekServer(
  112. support *SupportServices,
  113. listener net.Listener,
  114. listenerTunnelProtocol string,
  115. listenerPort int,
  116. useTLS, isFronted, useObfuscatedSessionTickets bool,
  117. clientHandler func(clientTunnelProtocol string, clientConn net.Conn),
  118. stopBroadcast <-chan struct{}) (*MeekServer, error) {
  119. checksumTable := crc64.MakeTable(crc64.ECMA)
  120. bufferLength := MEEK_DEFAULT_POOL_BUFFER_LENGTH
  121. if support.Config.MeekCachedResponsePoolBufferSize != 0 {
  122. bufferLength = support.Config.MeekCachedResponsePoolBufferSize
  123. }
  124. bufferCount := MEEK_DEFAULT_POOL_BUFFER_COUNT
  125. if support.Config.MeekCachedResponsePoolBufferCount != 0 {
  126. bufferCount = support.Config.MeekCachedResponsePoolBufferCount
  127. }
  128. bufferPool := NewCachedResponseBufferPool(bufferLength, bufferCount)
  129. passthroughAddress := support.Config.TunnelProtocolPassthroughAddresses[listenerTunnelProtocol]
  130. meekServer := &MeekServer{
  131. support: support,
  132. listener: listener,
  133. listenerTunnelProtocol: listenerTunnelProtocol,
  134. listenerPort: listenerPort,
  135. passthroughAddress: passthroughAddress,
  136. obfuscatorSeedHistory: obfuscator.NewSeedHistory(nil),
  137. clientHandler: clientHandler,
  138. openConns: common.NewConns(),
  139. stopBroadcast: stopBroadcast,
  140. sessions: make(map[string]*meekSession),
  141. checksumTable: checksumTable,
  142. bufferPool: bufferPool,
  143. rateLimitHistory: make(map[string][]time.Time),
  144. rateLimitSignalGC: make(chan struct{}, 1),
  145. }
  146. if useTLS {
  147. tlsConfig, err := meekServer.makeMeekTLSConfig(
  148. isFronted, useObfuscatedSessionTickets)
  149. if err != nil {
  150. return nil, errors.Trace(err)
  151. }
  152. meekServer.tlsConfig = tlsConfig
  153. }
  154. return meekServer, nil
  155. }
  156. type meekContextKey struct {
  157. key string
  158. }
  159. var meekNetConnContextKey = &meekContextKey{"net.Conn"}
  160. // Run runs the meek server; this function blocks while serving HTTP or
  161. // HTTPS connections on the specified listener. This function also runs
  162. // a goroutine which cleans up expired meek client sessions.
  163. //
  164. // To stop the meek server, both Close() the listener and set the stopBroadcast
  165. // signal specified in NewMeekServer.
  166. func (server *MeekServer) Run() error {
  167. waitGroup := new(sync.WaitGroup)
  168. waitGroup.Add(1)
  169. go func() {
  170. defer waitGroup.Done()
  171. ticker := time.NewTicker(MEEK_MAX_SESSION_STALENESS / 2)
  172. defer ticker.Stop()
  173. for {
  174. select {
  175. case <-ticker.C:
  176. server.deleteExpiredSessions()
  177. case <-server.stopBroadcast:
  178. return
  179. }
  180. }
  181. }()
  182. waitGroup.Add(1)
  183. go func() {
  184. defer waitGroup.Done()
  185. server.rateLimitWorker()
  186. }()
  187. // Serve HTTP or HTTPS
  188. //
  189. // - WriteTimeout may include time awaiting request, as per:
  190. // https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts
  191. // - Legacy meek-server wrapped each client HTTP connection with an explicit idle
  192. // timeout net.Conn and didn't use http.Server timeouts. We could do the same
  193. // here (use ActivityMonitoredConn) but the stock http.Server timeouts should
  194. // now be sufficient.
  195. httpServer := &http.Server{
  196. ReadTimeout: MEEK_HTTP_CLIENT_IO_TIMEOUT,
  197. WriteTimeout: MEEK_HTTP_CLIENT_IO_TIMEOUT,
  198. Handler: server,
  199. ConnState: server.httpConnStateCallback,
  200. ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
  201. return context.WithValue(ctx, meekNetConnContextKey, conn)
  202. },
  203. // Disable auto HTTP/2 (https://golang.org/doc/go1.6)
  204. TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
  205. }
  206. // Note: Serve() will be interrupted by listener.Close() call
  207. var err error
  208. if server.tlsConfig != nil {
  209. httpsServer := HTTPSServer{Server: httpServer}
  210. err = httpsServer.ServeTLS(server.listener, server.tlsConfig)
  211. } else {
  212. err = httpServer.Serve(server.listener)
  213. }
  214. // Can't check for the exact error that Close() will cause in Accept(),
  215. // (see: https://code.google.com/p/go/issues/detail?id=4373). So using an
  216. // explicit stop signal to stop gracefully.
  217. select {
  218. case <-server.stopBroadcast:
  219. err = nil
  220. default:
  221. }
  222. // deleteExpiredSessions calls deleteSession which may block waiting
  223. // for active request handlers to complete; timely shutdown requires
  224. // stopping the listener and closing all existing connections before
  225. // awaiting the reaperWaitGroup.
  226. server.listener.Close()
  227. server.openConns.CloseAll()
  228. waitGroup.Wait()
  229. return err
  230. }
  231. // ServeHTTP handles meek client HTTP requests, where the request body
  232. // contains upstream traffic and the response will contain downstream
  233. // traffic.
  234. func (server *MeekServer) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) {
  235. // Note: no longer requiring that the request method is POST
  236. // Check for the expected meek/session ID cookie.
  237. // Also check for prohibited HTTP headers.
  238. var meekCookie *http.Cookie
  239. for _, c := range request.Cookies() {
  240. meekCookie = c
  241. break
  242. }
  243. if meekCookie == nil || len(meekCookie.Value) == 0 {
  244. log.WithTrace().Warning("missing meek cookie")
  245. common.TerminateHTTPConnection(responseWriter, request)
  246. return
  247. }
  248. if len(server.support.Config.MeekProhibitedHeaders) > 0 {
  249. for _, header := range server.support.Config.MeekProhibitedHeaders {
  250. value := request.Header.Get(header)
  251. if header != "" {
  252. log.WithTraceFields(LogFields{
  253. "header": header,
  254. "value": value,
  255. }).Warning("prohibited meek header")
  256. common.TerminateHTTPConnection(responseWriter, request)
  257. return
  258. }
  259. }
  260. }
  261. // A valid meek cookie indicates which class of request this is:
  262. //
  263. // 1. A new meek session. Create a new session ID and proceed with
  264. // relaying tunnel traffic.
  265. //
  266. // 2. An existing meek session. Resume relaying tunnel traffic.
  267. //
  268. // 3. A request to an endpoint. This meek connection is not for relaying
  269. // tunnel traffic. Instead, the request is handed off to a custom handler.
  270. sessionID, session, endPoint, clientIP, err := server.getSessionOrEndpoint(request, meekCookie)
  271. if err != nil {
  272. // Debug since session cookie errors commonly occur during
  273. // normal operation.
  274. log.WithTraceFields(LogFields{"error": err}).Debug("session lookup failed")
  275. common.TerminateHTTPConnection(responseWriter, request)
  276. return
  277. }
  278. if endPoint != "" {
  279. // Endpoint mode. Currently, this means it's handled by the tactics
  280. // request handler.
  281. geoIPData := server.support.GeoIPService.Lookup(clientIP)
  282. handled := server.support.TacticsServer.HandleEndPoint(
  283. endPoint, common.GeoIPData(geoIPData), responseWriter, request)
  284. if !handled {
  285. log.WithTraceFields(LogFields{"endPoint": endPoint}).Info("unhandled endpoint")
  286. common.TerminateHTTPConnection(responseWriter, request)
  287. }
  288. return
  289. }
  290. // Tunnel relay mode.
  291. // Ensure that there's only one concurrent request handler per client
  292. // session. Depending on the nature of a network disruption, it can
  293. // happen that a client detects a failure and retries while the server
  294. // is still streaming response in the handler for the _previous_ client
  295. // request.
  296. //
  297. // Even if the session.cachedResponse were safe for concurrent
  298. // use (it is not), concurrent handling could lead to loss of session
  299. // since upstream data read by the first request may not reach the
  300. // cached response before the second request reads the cached data.
  301. //
  302. // The existing handler will stream response data, holding the lock,
  303. // for no more than MEEK_EXTENDED_TURN_AROUND_TIMEOUT.
  304. //
  305. // TODO: interrupt an existing handler? The existing handler will be
  306. // sending data to the cached response, but if that buffer fills, the
  307. // session will be lost.
  308. requestNumber := atomic.AddInt64(&session.requestCount, 1)
  309. // Wait for the existing request to complete.
  310. session.lock.Lock()
  311. defer session.lock.Unlock()
  312. // If a newer request has arrived while waiting, discard this one.
  313. // Do not delay processing the newest request.
  314. //
  315. // If the session expired and was deleted while this request was waiting,
  316. // discard this request. The session is no longer valid, and the final call
  317. // to session.cachedResponse.Reset may have already occured, so any further
  318. // session.cachedResponse access may deplete resources (fail to refill the pool).
  319. if atomic.LoadInt64(&session.requestCount) > requestNumber || session.deleted {
  320. common.TerminateHTTPConnection(responseWriter, request)
  321. return
  322. }
  323. // pumpReads causes a TunnelServer/SSH goroutine blocking on a Read to
  324. // read the request body as upstream traffic.
  325. // TODO: run pumpReads and pumpWrites concurrently?
  326. // pumpReads checksums the request payload and skips relaying it when
  327. // it matches the immediately previous request payload. This allows
  328. // clients to resend request payloads, when retrying due to connection
  329. // interruption, without knowing whether the server has received or
  330. // relayed the data.
  331. err = session.clientConn.pumpReads(request.Body)
  332. if err != nil {
  333. if err != io.EOF {
  334. // Debug since errors such as "i/o timeout" occur during normal operation;
  335. // also, golang network error messages may contain client IP.
  336. log.WithTraceFields(LogFields{"error": err}).Debug("read request failed")
  337. }
  338. common.TerminateHTTPConnection(responseWriter, request)
  339. // Note: keep session open to allow client to retry
  340. return
  341. }
  342. // Set cookie before writing the response.
  343. if session.meekProtocolVersion >= MEEK_PROTOCOL_VERSION_2 && !session.sessionIDSent {
  344. // Replace the meek cookie with the session ID.
  345. // SetCookie for the the session ID cookie is only set once, to reduce overhead. This
  346. // session ID value replaces the original meek cookie value.
  347. http.SetCookie(responseWriter, &http.Cookie{Name: meekCookie.Name, Value: sessionID})
  348. session.sessionIDSent = true
  349. }
  350. // When streaming data into the response body, a copy is
  351. // retained in the cachedResponse buffer. This allows the
  352. // client to retry and request that the response be resent
  353. // when the HTTP connection is interrupted.
  354. //
  355. // If a Range header is present, the client is retrying,
  356. // possibly after having received a partial response. In
  357. // this case, use any cached response to attempt to resend
  358. // the response, starting from the resend position the client
  359. // indicates.
  360. //
  361. // When the resend position is not available -- because the
  362. // cachedResponse buffer could not hold it -- the client session
  363. // is closed, as there's no way to resume streaming the payload
  364. // uninterrupted.
  365. //
  366. // The client may retry before a cached response is prepared,
  367. // so a cached response is not always used when a Range header
  368. // is present.
  369. //
  370. // TODO: invalid Range header is ignored; should it be otherwise?
  371. position, isRetry := checkRangeHeader(request)
  372. if isRetry {
  373. atomic.AddInt64(&session.metricClientRetries, 1)
  374. }
  375. hasCompleteCachedResponse := session.cachedResponse.HasPosition(0)
  376. // The client is not expected to send position > 0 when there is
  377. // no cached response; let that case fall through to the next
  378. // HasPosition check which will fail and close the session.
  379. var responseSize int
  380. var responseError error
  381. if isRetry && (hasCompleteCachedResponse || position > 0) {
  382. if !session.cachedResponse.HasPosition(position) {
  383. greaterThanSwapInt64(&session.metricCachedResponseMissPosition, int64(position))
  384. common.TerminateHTTPConnection(responseWriter, request)
  385. session.delete(true)
  386. return
  387. }
  388. responseWriter.WriteHeader(http.StatusPartialContent)
  389. // TODO:
  390. // - enforce a max extended buffer count per client, for
  391. // fairness? Throttling may make this unnecessary.
  392. // - cachedResponse can now start releasing extended buffers,
  393. // as response bytes before "position" will never be requested
  394. // again?
  395. responseSize, responseError = session.cachedResponse.CopyFromPosition(position, responseWriter)
  396. greaterThanSwapInt64(&session.metricPeakCachedResponseHitSize, int64(responseSize))
  397. // The client may again fail to receive the payload and may again
  398. // retry, so not yet releasing cachedResponse buffers.
  399. } else {
  400. // _Now_ we release buffers holding data from the previous
  401. // response. And then immediately stream the new response into
  402. // newly acquired buffers.
  403. session.cachedResponse.Reset()
  404. // Note: this code depends on an implementation detail of
  405. // io.MultiWriter: a Write() to the MultiWriter writes first
  406. // to the cache, and then to the response writer. So if the
  407. // write to the response writer fails, the payload is cached.
  408. multiWriter := io.MultiWriter(session.cachedResponse, responseWriter)
  409. // The client expects 206, not 200, whenever it sets a Range header,
  410. // which it may do even when no cached response is prepared.
  411. if isRetry {
  412. responseWriter.WriteHeader(http.StatusPartialContent)
  413. }
  414. // pumpWrites causes a TunnelServer/SSH goroutine blocking on a Write to
  415. // write its downstream traffic through to the response body.
  416. responseSize, responseError = session.clientConn.pumpWrites(multiWriter)
  417. greaterThanSwapInt64(&session.metricPeakResponseSize, int64(responseSize))
  418. greaterThanSwapInt64(&session.metricPeakCachedResponseSize, int64(session.cachedResponse.Available()))
  419. }
  420. // responseError is the result of writing the body either from CopyFromPosition or pumpWrites
  421. if responseError != nil {
  422. if responseError != io.EOF {
  423. // Debug since errors such as "i/o timeout" occur during normal operation;
  424. // also, golang network error messages may contain client IP.
  425. log.WithTraceFields(LogFields{"error": responseError}).Debug("write response failed")
  426. }
  427. common.TerminateHTTPConnection(responseWriter, request)
  428. // Note: keep session open to allow client to retry
  429. return
  430. }
  431. }
  432. func checkRangeHeader(request *http.Request) (int, bool) {
  433. rangeHeader := request.Header.Get("Range")
  434. if rangeHeader == "" {
  435. return 0, false
  436. }
  437. prefix := "bytes="
  438. suffix := "-"
  439. if !strings.HasPrefix(rangeHeader, prefix) ||
  440. !strings.HasSuffix(rangeHeader, suffix) {
  441. return 0, false
  442. }
  443. rangeHeader = strings.TrimPrefix(rangeHeader, prefix)
  444. rangeHeader = strings.TrimSuffix(rangeHeader, suffix)
  445. position, err := strconv.Atoi(rangeHeader)
  446. if err != nil {
  447. return 0, false
  448. }
  449. return position, true
  450. }
  451. // getSessionOrEndpoint checks if the cookie corresponds to an existing tunnel
  452. // relay session ID. If no session is found, the cookie must be an obfuscated
  453. // meek cookie. A new session is created when the meek cookie indicates relay
  454. // mode; or the endpoint is returned when the meek cookie indicates endpoint
  455. // mode.
  456. func (server *MeekServer) getSessionOrEndpoint(
  457. request *http.Request, meekCookie *http.Cookie) (string, *meekSession, string, string, error) {
  458. // Check for an existing session.
  459. server.sessionsLock.RLock()
  460. existingSessionID := meekCookie.Value
  461. session, ok := server.sessions[existingSessionID]
  462. server.sessionsLock.RUnlock()
  463. if ok {
  464. // TODO: can multiple http client connections using same session cookie
  465. // cause race conditions on session struct?
  466. session.touch()
  467. return existingSessionID, session, "", "", nil
  468. }
  469. // Determine the client remote address, which is used for geolocation
  470. // and stats. When an intermediate proxy or CDN is in use, we may be
  471. // able to determine the original client address by inspecting HTTP
  472. // headers such as X-Forwarded-For.
  473. clientIP := strings.Split(request.RemoteAddr, ":")[0]
  474. if len(server.support.Config.MeekProxyForwardedForHeaders) > 0 {
  475. for _, header := range server.support.Config.MeekProxyForwardedForHeaders {
  476. value := request.Header.Get(header)
  477. if len(value) > 0 {
  478. // Some headers, such as X-Forwarded-For, are a comma-separated
  479. // list of IPs (each proxy in a chain). The first IP should be
  480. // the client IP.
  481. proxyClientIP := strings.Split(value, ",")[0]
  482. if net.ParseIP(proxyClientIP) != nil &&
  483. server.support.GeoIPService.Lookup(proxyClientIP).Country != GEOIP_UNKNOWN_VALUE {
  484. clientIP = proxyClientIP
  485. break
  486. }
  487. }
  488. }
  489. }
  490. if server.rateLimit(clientIP) {
  491. return "", nil, "", "", errors.TraceNew("rate limit exceeded")
  492. }
  493. // The session is new (or expired). Treat the cookie value as a new meek
  494. // cookie, extract the payload, and create a new session.
  495. payloadJSON, err := server.getMeekCookiePayload(clientIP, meekCookie.Value)
  496. if err != nil {
  497. return "", nil, "", "", errors.Trace(err)
  498. }
  499. // Note: this meek server ignores legacy values PsiphonClientSessionId
  500. // and PsiphonServerAddress.
  501. var clientSessionData protocol.MeekCookieData
  502. err = json.Unmarshal(payloadJSON, &clientSessionData)
  503. if err != nil {
  504. return "", nil, "", "", errors.Trace(err)
  505. }
  506. // Handle endpoints before enforcing CheckEstablishTunnels.
  507. // Currently, endpoints are tactics requests, and we allow these to be
  508. // handled by servers which would otherwise reject new tunnels.
  509. if clientSessionData.EndPoint != "" {
  510. return "", nil, clientSessionData.EndPoint, clientIP, nil
  511. }
  512. // Don't create new sessions when not establishing. A subsequent SSH handshake
  513. // will not succeed, so creating a meek session just wastes resources.
  514. if server.support.TunnelServer != nil &&
  515. !server.support.TunnelServer.CheckEstablishTunnels() {
  516. return "", nil, "", "", errors.TraceNew("not establishing tunnels")
  517. }
  518. // Create a new session
  519. bufferLength := MEEK_DEFAULT_RESPONSE_BUFFER_LENGTH
  520. if server.support.Config.MeekCachedResponseBufferSize != 0 {
  521. bufferLength = server.support.Config.MeekCachedResponseBufferSize
  522. }
  523. cachedResponse := NewCachedResponse(bufferLength, server.bufferPool)
  524. session = &meekSession{
  525. meekProtocolVersion: clientSessionData.MeekProtocolVersion,
  526. sessionIDSent: false,
  527. cachedResponse: cachedResponse,
  528. }
  529. session.touch()
  530. underlyingConn := request.Context().Value(meekNetConnContextKey).(net.Conn)
  531. // Create a new meek conn that will relay the payload
  532. // between meek request/responses and the tunnel server client
  533. // handler. The client IP is also used to initialize the
  534. // meek conn with a useful value to return when the tunnel
  535. // server calls conn.RemoteAddr() to get the client's IP address.
  536. // Assumes clientIP is a valid IP address; the port value is a stub
  537. // and is expected to be ignored.
  538. clientConn := newMeekConn(
  539. server,
  540. session,
  541. underlyingConn.LocalAddr(),
  542. underlyingConn.RemoteAddr(),
  543. &net.TCPAddr{
  544. IP: net.ParseIP(clientIP),
  545. Port: 0,
  546. },
  547. clientSessionData.MeekProtocolVersion)
  548. session.clientConn = clientConn
  549. // Note: MEEK_PROTOCOL_VERSION_1 doesn't support changing the
  550. // meek cookie to a session ID; v1 clients always send the
  551. // original meek cookie value with each request. The issue with
  552. // v1 is that clients which wake after a device sleep will attempt
  553. // to resume a meek session and the server can't differentiate
  554. // between resuming a session and creating a new session. This
  555. // causes the v1 client connection to hang/timeout.
  556. sessionID := meekCookie.Value
  557. if clientSessionData.MeekProtocolVersion >= MEEK_PROTOCOL_VERSION_2 {
  558. sessionID, err = makeMeekSessionID()
  559. if err != nil {
  560. return "", nil, "", "", errors.Trace(err)
  561. }
  562. }
  563. server.sessionsLock.Lock()
  564. server.sessions[sessionID] = session
  565. server.sessionsLock.Unlock()
  566. // Note: from the tunnel server's perspective, this client connection
  567. // will close when session.delete calls Close() on the meekConn.
  568. server.clientHandler(clientSessionData.ClientTunnelProtocol, session.clientConn)
  569. return sessionID, session, "", "", nil
  570. }
  571. func (server *MeekServer) rateLimit(clientIP string) bool {
  572. historySize, thresholdSeconds, regions, ISPs, cities, GCTriggerCount, _ :=
  573. server.support.TrafficRulesSet.GetMeekRateLimiterConfig()
  574. if historySize == 0 {
  575. return false
  576. }
  577. if len(regions) > 0 || len(ISPs) > 0 || len(cities) > 0 {
  578. // TODO: avoid redundant GeoIP lookups?
  579. geoIPData := server.support.GeoIPService.Lookup(clientIP)
  580. if len(regions) > 0 {
  581. if !common.Contains(regions, geoIPData.Country) {
  582. return false
  583. }
  584. }
  585. if len(ISPs) > 0 {
  586. if !common.Contains(ISPs, geoIPData.ISP) {
  587. return false
  588. }
  589. }
  590. if len(cities) > 0 {
  591. if !common.Contains(cities, geoIPData.City) {
  592. return false
  593. }
  594. }
  595. }
  596. limit := true
  597. triggerGC := false
  598. now := time.Now()
  599. threshold := now.Add(-time.Duration(thresholdSeconds) * time.Second)
  600. server.rateLimitLock.Lock()
  601. history, ok := server.rateLimitHistory[clientIP]
  602. if !ok || len(history) != historySize {
  603. history = make([]time.Time, historySize)
  604. server.rateLimitHistory[clientIP] = history
  605. }
  606. for i := 0; i < len(history); i++ {
  607. if history[i].IsZero() || history[i].Before(threshold) {
  608. limit = false
  609. }
  610. if i == len(history)-1 {
  611. history[i] = now
  612. } else {
  613. history[i] = history[i+1]
  614. }
  615. }
  616. if limit {
  617. server.rateLimitCount += 1
  618. if server.rateLimitCount >= GCTriggerCount {
  619. triggerGC = true
  620. server.rateLimitCount = 0
  621. }
  622. }
  623. server.rateLimitLock.Unlock()
  624. if triggerGC {
  625. select {
  626. case server.rateLimitSignalGC <- struct{}{}:
  627. default:
  628. }
  629. }
  630. return limit
  631. }
  632. func (server *MeekServer) rateLimitWorker() {
  633. _, _, _, _, _, _, reapFrequencySeconds :=
  634. server.support.TrafficRulesSet.GetMeekRateLimiterConfig()
  635. timer := time.NewTimer(time.Duration(reapFrequencySeconds) * time.Second)
  636. defer timer.Stop()
  637. for {
  638. select {
  639. case <-timer.C:
  640. _, thresholdSeconds, _, _, _, _, reapFrequencySeconds :=
  641. server.support.TrafficRulesSet.GetMeekRateLimiterConfig()
  642. server.rateLimitLock.Lock()
  643. threshold := time.Now().Add(-time.Duration(thresholdSeconds) * time.Second)
  644. for key, history := range server.rateLimitHistory {
  645. reap := true
  646. for i := 0; i < len(history); i++ {
  647. if !history[i].IsZero() && !history[i].Before(threshold) {
  648. reap = false
  649. }
  650. }
  651. if reap {
  652. delete(server.rateLimitHistory, key)
  653. }
  654. }
  655. // Enable rate limit history map to be garbage collected when possible.
  656. if len(server.rateLimitHistory) == 0 {
  657. server.rateLimitHistory = make(map[string][]time.Time)
  658. }
  659. server.rateLimitLock.Unlock()
  660. timer.Reset(time.Duration(reapFrequencySeconds) * time.Second)
  661. case <-server.rateLimitSignalGC:
  662. runtime.GC()
  663. case <-server.stopBroadcast:
  664. return
  665. }
  666. }
  667. }
  668. func (server *MeekServer) deleteSession(sessionID string) {
  669. // Don't obtain the server.sessionsLock write lock until modifying
  670. // server.sessions, as the session.delete can block for up to
  671. // MEEK_HTTP_CLIENT_IO_TIMEOUT. Allow new sessions to be added
  672. // concurrently.
  673. //
  674. // Since a lock isn't held for the duration, concurrent calls to
  675. // deleteSession with the same sessionID could happen; this is
  676. // not expected since only the reaper goroutine calls deleteExpiredSessions
  677. // (and in any case concurrent execution of the ok block is not an issue).
  678. server.sessionsLock.RLock()
  679. session, ok := server.sessions[sessionID]
  680. server.sessionsLock.RUnlock()
  681. if ok {
  682. session.delete(false)
  683. server.sessionsLock.Lock()
  684. delete(server.sessions, sessionID)
  685. server.sessionsLock.Unlock()
  686. }
  687. }
  688. func (server *MeekServer) deleteExpiredSessions() {
  689. // A deleteSession call may block for up to MEEK_HTTP_CLIENT_IO_TIMEOUT,
  690. // so grab a snapshot list of expired sessions and do not hold a lock for
  691. // the duration of deleteExpiredSessions. This allows new sessions to be
  692. // added concurrently.
  693. //
  694. // New sessions added after the snapshot is taken will be checked for
  695. // expiry on subsequent periodic calls to deleteExpiredSessions.
  696. //
  697. // To avoid long delays in releasing resources, individual deletes are
  698. // performed concurrently.
  699. server.sessionsLock.Lock()
  700. expiredSessionIDs := make([]string, 0)
  701. for sessionID, session := range server.sessions {
  702. if session.expired() {
  703. expiredSessionIDs = append(expiredSessionIDs, sessionID)
  704. }
  705. }
  706. server.sessionsLock.Unlock()
  707. start := time.Now()
  708. deleteWaitGroup := new(sync.WaitGroup)
  709. for _, sessionID := range expiredSessionIDs {
  710. deleteWaitGroup.Add(1)
  711. go func(sessionID string) {
  712. defer deleteWaitGroup.Done()
  713. server.deleteSession(sessionID)
  714. }(sessionID)
  715. }
  716. deleteWaitGroup.Wait()
  717. log.WithTraceFields(
  718. LogFields{"elapsed time": time.Since(start)}).Debug("deleted expired sessions")
  719. }
  720. // httpConnStateCallback tracks open persistent HTTP/HTTPS connections to the
  721. // meek server.
  722. func (server *MeekServer) httpConnStateCallback(conn net.Conn, connState http.ConnState) {
  723. switch connState {
  724. case http.StateNew:
  725. server.openConns.Add(conn)
  726. case http.StateHijacked, http.StateClosed:
  727. server.openConns.Remove(conn)
  728. }
  729. }
  730. // getMeekCookiePayload extracts the payload from a meek cookie. The cookie
  731. // payload is base64 encoded, obfuscated, and NaCl encrypted.
  732. func (server *MeekServer) getMeekCookiePayload(
  733. clientIP string, cookieValue string) ([]byte, error) {
  734. decodedValue, err := base64.StdEncoding.DecodeString(cookieValue)
  735. if err != nil {
  736. return nil, errors.Trace(err)
  737. }
  738. // The data consists of an obfuscated seed message prepended
  739. // to the obfuscated, encrypted payload. The server obfuscator
  740. // will read the seed message, leaving the remaining encrypted
  741. // data in the reader.
  742. reader := bytes.NewReader(decodedValue[:])
  743. obfuscator, err := obfuscator.NewServerObfuscator(
  744. &obfuscator.ObfuscatorConfig{
  745. Keyword: server.support.Config.MeekObfuscatedKey,
  746. SeedHistory: server.obfuscatorSeedHistory,
  747. IrregularLogger: func(clientIP string, err error, logFields common.LogFields) {
  748. logIrregularTunnel(
  749. server.support,
  750. server.listenerTunnelProtocol,
  751. server.listenerPort,
  752. clientIP,
  753. errors.Trace(err),
  754. LogFields(logFields))
  755. },
  756. },
  757. clientIP,
  758. reader)
  759. if err != nil {
  760. return nil, errors.Trace(err)
  761. }
  762. offset, err := reader.Seek(0, 1)
  763. if err != nil {
  764. return nil, errors.Trace(err)
  765. }
  766. encryptedPayload := decodedValue[offset:]
  767. obfuscator.ObfuscateClientToServer(encryptedPayload)
  768. var nonce [24]byte
  769. var privateKey, ephemeralPublicKey [32]byte
  770. decodedPrivateKey, err := base64.StdEncoding.DecodeString(
  771. server.support.Config.MeekCookieEncryptionPrivateKey)
  772. if err != nil {
  773. return nil, errors.Trace(err)
  774. }
  775. copy(privateKey[:], decodedPrivateKey)
  776. if len(encryptedPayload) < 32 {
  777. return nil, errors.TraceNew("unexpected encrypted payload size")
  778. }
  779. copy(ephemeralPublicKey[0:32], encryptedPayload[0:32])
  780. payload, ok := box.Open(nil, encryptedPayload[32:], &nonce, &ephemeralPublicKey, &privateKey)
  781. if !ok {
  782. return nil, errors.TraceNew("open box failed")
  783. }
  784. return payload, nil
  785. }
  786. // makeMeekTLSConfig creates a TLS config for a meek HTTPS listener.
  787. // Currently, this config is optimized for fronted meek where the nature
  788. // of the connection is non-circumvention; it's optimized for performance
  789. // assuming the peer is an uncensored CDN.
  790. func (server *MeekServer) makeMeekTLSConfig(
  791. isFronted bool, useObfuscatedSessionTickets bool) (*tris.Config, error) {
  792. certificate, privateKey, err := common.GenerateWebServerCertificate(values.GetHostName())
  793. if err != nil {
  794. return nil, errors.Trace(err)
  795. }
  796. tlsCertificate, err := tris.X509KeyPair(
  797. []byte(certificate), []byte(privateKey))
  798. if err != nil {
  799. return nil, errors.Trace(err)
  800. }
  801. // Vary the minimum version to frustrate scanning/fingerprinting of unfronted servers.
  802. // Limitation: like the certificate, this value changes on restart.
  803. minVersionCandidates := []uint16{tris.VersionTLS10, tris.VersionTLS11, tris.VersionTLS12}
  804. minVersion := minVersionCandidates[prng.Intn(len(minVersionCandidates))]
  805. config := &tris.Config{
  806. Certificates: []tris.Certificate{tlsCertificate},
  807. NextProtos: []string{"http/1.1"},
  808. MinVersion: minVersion,
  809. UseExtendedMasterSecret: true,
  810. }
  811. if isFronted {
  812. // This is a reordering of the supported CipherSuites in golang 1.6[*]. Non-ephemeral key
  813. // CipherSuites greatly reduce server load, and we try to select these since the meek
  814. // protocol is providing obfuscation, not privacy/integrity (this is provided by the
  815. // tunneled SSH), so we don't benefit from the perfect forward secrecy property provided
  816. // by ephemeral key CipherSuites.
  817. // https://github.com/golang/go/blob/1cb3044c9fcd88e1557eca1bf35845a4108bc1db/src/crypto/tls/cipher_suites.go#L75
  818. //
  819. // This optimization is applied only when there's a CDN in front of the meek server; in
  820. // unfronted cases we prefer a more natural TLS handshake.
  821. //
  822. // [*] the list has since been updated, removing CipherSuites using RC4 and 3DES.
  823. config.CipherSuites = []uint16{
  824. tris.TLS_RSA_WITH_AES_128_GCM_SHA256,
  825. tris.TLS_RSA_WITH_AES_256_GCM_SHA384,
  826. tris.TLS_RSA_WITH_AES_128_CBC_SHA,
  827. tris.TLS_RSA_WITH_AES_256_CBC_SHA,
  828. tris.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
  829. tris.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
  830. tris.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
  831. tris.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
  832. tris.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
  833. tris.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
  834. tris.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
  835. tris.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
  836. }
  837. config.PreferServerCipherSuites = true
  838. }
  839. if useObfuscatedSessionTickets {
  840. // See obfuscated session ticket overview
  841. // in NewObfuscatedClientSessionCache.
  842. config.UseObfuscatedSessionTickets = true
  843. var obfuscatedSessionTicketKey [32]byte
  844. key, err := hex.DecodeString(server.support.Config.MeekObfuscatedKey)
  845. if err == nil && len(key) != 32 {
  846. err = std_errors.New("invalid obfuscated session key length")
  847. }
  848. if err != nil {
  849. return nil, errors.Trace(err)
  850. }
  851. copy(obfuscatedSessionTicketKey[:], key)
  852. var standardSessionTicketKey [32]byte
  853. _, err = rand.Read(standardSessionTicketKey[:])
  854. if err != nil {
  855. return nil, errors.Trace(err)
  856. }
  857. // Note: SessionTicketKey needs to be set, or else, it appears,
  858. // tris.Config.serverInit() will clobber the value set by
  859. // SetSessionTicketKeys.
  860. config.SessionTicketKey = obfuscatedSessionTicketKey
  861. config.SetSessionTicketKeys([][32]byte{
  862. standardSessionTicketKey,
  863. obfuscatedSessionTicketKey})
  864. }
  865. // When configured, initialize passthrough mode, an anti-probing defense.
  866. // Clients must prove knowledge of the obfuscated key via a message sent in
  867. // the TLS ClientHello random field.
  868. //
  869. // When clients fail to provide a valid message, the client connection is
  870. // relayed to the designated passthrough address, typically another web site.
  871. // The entire flow is relayed, including the original ClientHello, so the
  872. // client will perform a TLS handshake with the passthrough target.
  873. //
  874. // Irregular events are logged for invalid client activity.
  875. if server.passthroughAddress != "" {
  876. config.PassthroughAddress = server.passthroughAddress
  877. passthroughKey, err := obfuscator.DeriveTLSPassthroughKey(
  878. server.support.Config.MeekObfuscatedKey)
  879. if err != nil {
  880. return nil, errors.Trace(err)
  881. }
  882. config.PassthroughKey = passthroughKey
  883. config.PassthroughLogInvalidMessage = func(
  884. clientIP string) {
  885. logIrregularTunnel(
  886. server.support,
  887. server.listenerTunnelProtocol,
  888. server.listenerPort,
  889. clientIP,
  890. errors.TraceNew("invalid passthrough message"),
  891. nil)
  892. }
  893. config.PassthroughHistoryAddNew = func(
  894. clientIP string,
  895. clientRandom []byte) bool {
  896. // strictMode is true as, unlike with meek cookies, legitimate meek clients
  897. // never retry TLS connections using a previous random value.
  898. ok, logFields := server.obfuscatorSeedHistory.AddNew(
  899. true,
  900. clientIP,
  901. "client-random",
  902. clientRandom)
  903. if logFields != nil {
  904. logIrregularTunnel(
  905. server.support,
  906. server.listenerTunnelProtocol,
  907. server.listenerPort,
  908. clientIP,
  909. errors.TraceNew("duplicate passthrough message"),
  910. LogFields(*logFields))
  911. }
  912. return ok
  913. }
  914. }
  915. return config, nil
  916. }
  917. type meekSession struct {
  918. // Note: 64-bit ints used with atomic operations are placed
  919. // at the start of struct to ensure 64-bit alignment.
  920. // (https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
  921. lastActivity int64
  922. requestCount int64
  923. metricClientRetries int64
  924. metricPeakResponseSize int64
  925. metricPeakCachedResponseSize int64
  926. metricPeakCachedResponseHitSize int64
  927. metricCachedResponseMissPosition int64
  928. lock sync.Mutex
  929. deleted bool
  930. clientConn *meekConn
  931. meekProtocolVersion int
  932. sessionIDSent bool
  933. cachedResponse *CachedResponse
  934. }
  935. func (session *meekSession) touch() {
  936. atomic.StoreInt64(&session.lastActivity, int64(monotime.Now()))
  937. }
  938. func (session *meekSession) expired() bool {
  939. lastActivity := monotime.Time(atomic.LoadInt64(&session.lastActivity))
  940. return monotime.Since(lastActivity) > MEEK_MAX_SESSION_STALENESS
  941. }
  942. // delete releases all resources allocated by a session.
  943. func (session *meekSession) delete(haveLock bool) {
  944. // TODO: close the persistent HTTP client connection, if one exists?
  945. // This final call session.cachedResponse.Reset releases shared resources.
  946. //
  947. // This call requires exclusive access. session.lock is be obtained before
  948. // calling session.cachedResponse.Reset. Once the lock is obtained, no
  949. // request for this session is being processed concurrently, and pending
  950. // requests will block at session.lock.
  951. //
  952. // This logic assumes that no further session.cachedResponse access occurs,
  953. // or else resources may deplete (buffers won't be returned to the pool).
  954. // These requirements are achieved by obtaining the lock, setting
  955. // session.deleted, and any subsequent request handlers checking
  956. // session.deleted immediately after obtaining the lock.
  957. //
  958. // session.lock.Lock may block for up to MEEK_HTTP_CLIENT_IO_TIMEOUT,
  959. // the timeout for any active request handler processing a session
  960. // request.
  961. //
  962. // When the lock must be acquired, clientConn.Close is called first, to
  963. // interrupt any existing request handler blocking on pumpReads or pumpWrites.
  964. session.clientConn.Close()
  965. if !haveLock {
  966. session.lock.Lock()
  967. }
  968. // Release all extended buffers back to the pool.
  969. // session.cachedResponse.Reset is not safe for concurrent calls.
  970. session.cachedResponse.Reset()
  971. session.deleted = true
  972. if !haveLock {
  973. session.lock.Unlock()
  974. }
  975. }
  976. // GetMetrics implements the common.MetricsSource interface.
  977. func (session *meekSession) GetMetrics() common.LogFields {
  978. logFields := make(common.LogFields)
  979. logFields["meek_client_retries"] = atomic.LoadInt64(&session.metricClientRetries)
  980. logFields["meek_peak_response_size"] = atomic.LoadInt64(&session.metricPeakResponseSize)
  981. logFields["meek_peak_cached_response_size"] = atomic.LoadInt64(&session.metricPeakCachedResponseSize)
  982. logFields["meek_peak_cached_response_hit_size"] = atomic.LoadInt64(&session.metricPeakCachedResponseHitSize)
  983. logFields["meek_cached_response_miss_position"] = atomic.LoadInt64(&session.metricCachedResponseMissPosition)
  984. return logFields
  985. }
  986. // makeMeekSessionID creates a new session ID. The variable size is intended to
  987. // frustrate traffic analysis of both plaintext and TLS meek traffic.
  988. func makeMeekSessionID() (string, error) {
  989. size := MEEK_MIN_SESSION_ID_LENGTH +
  990. prng.Intn(MEEK_MAX_SESSION_ID_LENGTH-MEEK_MIN_SESSION_ID_LENGTH)
  991. sessionID, err := common.MakeSecureRandomBytes(size)
  992. if err != nil {
  993. return "", errors.Trace(err)
  994. }
  995. // Omit padding to maximize variable size space. To the client, the session
  996. // ID is an opaque string cookie value.
  997. return base64.RawStdEncoding.EncodeToString(sessionID), nil
  998. }
  999. // meekConn implements the net.Conn interface and is to be used as a client
  1000. // connection by the tunnel server (being passed to sshServer.handleClient).
  1001. // meekConn bridges net/http request/response payload readers and writers
  1002. // and goroutines calling Read()s and Write()s.
  1003. //
  1004. // meekConn implements the UnderlyingTCPAddrSource, returning the TCP
  1005. // addresses for the _first_ underlying TCP connection in the meek tunnel.
  1006. type meekConn struct {
  1007. meekServer *MeekServer
  1008. meekSession *meekSession
  1009. remoteAddr net.Addr
  1010. underlyingLocalAddr net.Addr
  1011. underlyingRemoteAddr net.Addr
  1012. protocolVersion int
  1013. closeBroadcast chan struct{}
  1014. closed int32
  1015. lastReadChecksum *uint64
  1016. readLock sync.Mutex
  1017. emptyReadBuffer chan *bytes.Buffer
  1018. partialReadBuffer chan *bytes.Buffer
  1019. fullReadBuffer chan *bytes.Buffer
  1020. writeLock sync.Mutex
  1021. nextWriteBuffer chan []byte
  1022. writeResult chan error
  1023. }
  1024. func newMeekConn(
  1025. meekServer *MeekServer,
  1026. meekSession *meekSession,
  1027. underlyingLocalAddr net.Addr,
  1028. underlyingRemoteAddr net.Addr,
  1029. remoteAddr net.Addr,
  1030. protocolVersion int) *meekConn {
  1031. conn := &meekConn{
  1032. meekServer: meekServer,
  1033. meekSession: meekSession,
  1034. underlyingLocalAddr: underlyingLocalAddr,
  1035. underlyingRemoteAddr: underlyingRemoteAddr,
  1036. remoteAddr: remoteAddr,
  1037. protocolVersion: protocolVersion,
  1038. closeBroadcast: make(chan struct{}),
  1039. closed: 0,
  1040. emptyReadBuffer: make(chan *bytes.Buffer, 1),
  1041. partialReadBuffer: make(chan *bytes.Buffer, 1),
  1042. fullReadBuffer: make(chan *bytes.Buffer, 1),
  1043. nextWriteBuffer: make(chan []byte, 1),
  1044. writeResult: make(chan error, 1),
  1045. }
  1046. // Read() calls and pumpReads() are synchronized by exchanging control
  1047. // of a single readBuffer. This is the same scheme used in and described
  1048. // in psiphon.MeekConn.
  1049. conn.emptyReadBuffer <- new(bytes.Buffer)
  1050. return conn
  1051. }
  1052. func (conn *meekConn) GetUnderlyingTCPAddrs() (*net.TCPAddr, *net.TCPAddr, bool) {
  1053. localAddr, ok := conn.underlyingLocalAddr.(*net.TCPAddr)
  1054. if !ok {
  1055. return nil, nil, false
  1056. }
  1057. remoteAddr, ok := conn.underlyingRemoteAddr.(*net.TCPAddr)
  1058. if !ok {
  1059. return nil, nil, false
  1060. }
  1061. return localAddr, remoteAddr, true
  1062. }
  1063. // pumpReads causes goroutines blocking on meekConn.Read() to read
  1064. // from the specified reader. This function blocks until the reader
  1065. // is fully consumed or the meekConn is closed. A read buffer allows
  1066. // up to MEEK_MAX_REQUEST_PAYLOAD_LENGTH bytes to be read and buffered
  1067. // without a Read() immediately consuming the bytes, but there's still
  1068. // a possibility of a stall if no Read() calls are made after this
  1069. // read buffer is full.
  1070. // Note: assumes only one concurrent call to pumpReads
  1071. func (conn *meekConn) pumpReads(reader io.Reader) error {
  1072. // Use either an empty or partial buffer. By using a partial
  1073. // buffer, pumpReads will not block if the Read() caller has
  1074. // not fully drained the read buffer.
  1075. var readBuffer *bytes.Buffer
  1076. select {
  1077. case readBuffer = <-conn.emptyReadBuffer:
  1078. case readBuffer = <-conn.partialReadBuffer:
  1079. case <-conn.closeBroadcast:
  1080. return io.EOF
  1081. }
  1082. newDataOffset := readBuffer.Len()
  1083. // Since we need to read the full request payload in order to
  1084. // take its checksum before relaying it, the read buffer can
  1085. // grow to up to 2 x MEEK_MAX_REQUEST_PAYLOAD_LENGTH + 1.
  1086. // +1 allows for an explicit check for request payloads that
  1087. // exceed the maximum permitted length.
  1088. limitReader := io.LimitReader(reader, MEEK_MAX_REQUEST_PAYLOAD_LENGTH+1)
  1089. n, err := readBuffer.ReadFrom(limitReader)
  1090. if err == nil && n == MEEK_MAX_REQUEST_PAYLOAD_LENGTH+1 {
  1091. err = std_errors.New("invalid request payload length")
  1092. }
  1093. // If the request read fails, don't relay the new data. This allows
  1094. // the client to retry and resend its request payload without
  1095. // interrupting/duplicating the payload flow.
  1096. if err != nil {
  1097. readBuffer.Truncate(newDataOffset)
  1098. conn.replaceReadBuffer(readBuffer)
  1099. return errors.Trace(err)
  1100. }
  1101. // Check if request payload checksum matches immediately
  1102. // previous payload. On match, assume this is a client retry
  1103. // sending payload that was already relayed and skip this
  1104. // payload. Payload is OSSH ciphertext and almost surely
  1105. // will not repeat. In the highly unlikely case that it does,
  1106. // the underlying SSH connection will fail and the client
  1107. // must reconnect.
  1108. checksum := crc64.Checksum(
  1109. readBuffer.Bytes()[newDataOffset:], conn.meekServer.checksumTable)
  1110. if conn.lastReadChecksum == nil {
  1111. conn.lastReadChecksum = new(uint64)
  1112. } else if *conn.lastReadChecksum == checksum {
  1113. readBuffer.Truncate(newDataOffset)
  1114. }
  1115. *conn.lastReadChecksum = checksum
  1116. conn.replaceReadBuffer(readBuffer)
  1117. return nil
  1118. }
  1119. var errMeekConnectionHasClosed = std_errors.New("meek connection has closed")
  1120. // Read reads from the meekConn into buffer. Read blocks until
  1121. // some data is read or the meekConn closes. Under the hood, it
  1122. // waits for pumpReads to submit a reader to read from.
  1123. // Note: lock is to conform with net.Conn concurrency semantics
  1124. func (conn *meekConn) Read(buffer []byte) (int, error) {
  1125. conn.readLock.Lock()
  1126. defer conn.readLock.Unlock()
  1127. var readBuffer *bytes.Buffer
  1128. select {
  1129. case readBuffer = <-conn.partialReadBuffer:
  1130. case readBuffer = <-conn.fullReadBuffer:
  1131. case <-conn.closeBroadcast:
  1132. return 0, errors.Trace(errMeekConnectionHasClosed)
  1133. }
  1134. n, err := readBuffer.Read(buffer)
  1135. conn.replaceReadBuffer(readBuffer)
  1136. return n, err
  1137. }
  1138. func (conn *meekConn) replaceReadBuffer(readBuffer *bytes.Buffer) {
  1139. length := readBuffer.Len()
  1140. if length >= MEEK_MAX_REQUEST_PAYLOAD_LENGTH {
  1141. conn.fullReadBuffer <- readBuffer
  1142. } else if length == 0 {
  1143. conn.emptyReadBuffer <- readBuffer
  1144. } else {
  1145. conn.partialReadBuffer <- readBuffer
  1146. }
  1147. }
  1148. // pumpWrites causes goroutines blocking on meekConn.Write() to write
  1149. // to the specified writer. This function blocks until the meek response
  1150. // body limits (size for protocol v1, turn around time for protocol v2+)
  1151. // are met, or the meekConn is closed.
  1152. // Note: channel scheme assumes only one concurrent call to pumpWrites
  1153. func (conn *meekConn) pumpWrites(writer io.Writer) (int, error) {
  1154. startTime := time.Now()
  1155. timeout := time.NewTimer(MEEK_TURN_AROUND_TIMEOUT)
  1156. defer timeout.Stop()
  1157. n := 0
  1158. for {
  1159. select {
  1160. case buffer := <-conn.nextWriteBuffer:
  1161. written, err := writer.Write(buffer)
  1162. n += written
  1163. // Assumes that writeResult won't block.
  1164. // Note: always send the err to writeResult,
  1165. // as the Write() caller is blocking on this.
  1166. conn.writeResult <- err
  1167. if err != nil {
  1168. return n, err
  1169. }
  1170. if conn.protocolVersion < MEEK_PROTOCOL_VERSION_1 {
  1171. // Pre-protocol version 1 clients expect at most
  1172. // MEEK_MAX_REQUEST_PAYLOAD_LENGTH response bodies
  1173. return n, nil
  1174. }
  1175. totalElapsedTime := time.Since(startTime) / time.Millisecond
  1176. if totalElapsedTime >= MEEK_EXTENDED_TURN_AROUND_TIMEOUT {
  1177. return n, nil
  1178. }
  1179. timeout.Reset(MEEK_TURN_AROUND_TIMEOUT)
  1180. case <-timeout.C:
  1181. return n, nil
  1182. case <-conn.closeBroadcast:
  1183. return n, errors.Trace(errMeekConnectionHasClosed)
  1184. }
  1185. }
  1186. }
  1187. // Write writes the buffer to the meekConn. It blocks until the
  1188. // entire buffer is written to or the meekConn closes. Under the
  1189. // hood, it waits for sufficient pumpWrites calls to consume the
  1190. // write buffer.
  1191. // Note: lock is to conform with net.Conn concurrency semantics
  1192. func (conn *meekConn) Write(buffer []byte) (int, error) {
  1193. conn.writeLock.Lock()
  1194. defer conn.writeLock.Unlock()
  1195. // TODO: may be more efficient to send whole buffer
  1196. // and have pumpWrites stash partial buffer when can't
  1197. // send it all.
  1198. n := 0
  1199. for n < len(buffer) {
  1200. end := n + MEEK_MAX_REQUEST_PAYLOAD_LENGTH
  1201. if end > len(buffer) {
  1202. end = len(buffer)
  1203. }
  1204. // Only write MEEK_MAX_REQUEST_PAYLOAD_LENGTH at a time,
  1205. // to ensure compatibility with v1 protocol.
  1206. chunk := buffer[n:end]
  1207. select {
  1208. case conn.nextWriteBuffer <- chunk:
  1209. case <-conn.closeBroadcast:
  1210. return n, errors.Trace(errMeekConnectionHasClosed)
  1211. }
  1212. // Wait for the buffer to be processed.
  1213. select {
  1214. case <-conn.writeResult:
  1215. // The err from conn.writeResult comes from the
  1216. // io.MultiWriter used in pumpWrites, which writes
  1217. // to both the cached response and the HTTP response.
  1218. //
  1219. // Don't stop on error here, since only writing
  1220. // to the HTTP response will fail, and the client
  1221. // may retry and use the cached response.
  1222. //
  1223. // It's possible that the cached response buffer
  1224. // is too small for the client to successfully
  1225. // retry, but that cannot be determined. In this
  1226. // case, the meek connection will eventually fail.
  1227. //
  1228. // err is already logged in ServeHTTP.
  1229. case <-conn.closeBroadcast:
  1230. return n, errors.Trace(errMeekConnectionHasClosed)
  1231. }
  1232. n += len(chunk)
  1233. }
  1234. return n, nil
  1235. }
  1236. // Close closes the meekConn. This will interrupt any blocked
  1237. // Read, Write, pumpReads, and pumpWrites.
  1238. func (conn *meekConn) Close() error {
  1239. if atomic.CompareAndSwapInt32(&conn.closed, 0, 1) {
  1240. close(conn.closeBroadcast)
  1241. }
  1242. return nil
  1243. }
  1244. // Stub implementation of net.Conn.LocalAddr
  1245. func (conn *meekConn) LocalAddr() net.Addr {
  1246. return nil
  1247. }
  1248. // RemoteAddr returns the remoteAddr specified in newMeekConn. This
  1249. // acts as a proxy for the actual remote address, which is either a
  1250. // direct HTTP/HTTPS connection remote address, or in the case of
  1251. // downstream proxy of CDN fronts, some other value determined via
  1252. // HTTP headers.
  1253. func (conn *meekConn) RemoteAddr() net.Addr {
  1254. return conn.remoteAddr
  1255. }
  1256. // SetDeadline is not a true implementation of net.Conn.SetDeadline. It
  1257. // merely checks that the requested timeout exceeds the MEEK_MAX_SESSION_STALENESS
  1258. // period. When it does, and the session is idle, the meekConn Read/Write will
  1259. // be interrupted and return an error (not a timeout error) before the deadline.
  1260. // In other words, this conn will approximate the desired functionality of
  1261. // timing out on idle on or before the requested deadline.
  1262. func (conn *meekConn) SetDeadline(t time.Time) error {
  1263. // Overhead: nanoseconds (https://blog.cloudflare.com/its-go-time-on-linux/)
  1264. if time.Now().Add(MEEK_MAX_SESSION_STALENESS).Before(t) {
  1265. return nil
  1266. }
  1267. return errors.TraceNew("not supported")
  1268. }
  1269. // Stub implementation of net.Conn.SetReadDeadline
  1270. func (conn *meekConn) SetReadDeadline(t time.Time) error {
  1271. return errors.TraceNew("not supported")
  1272. }
  1273. // Stub implementation of net.Conn.SetWriteDeadline
  1274. func (conn *meekConn) SetWriteDeadline(t time.Time) error {
  1275. return errors.TraceNew("not supported")
  1276. }
  1277. // GetMetrics implements the common.MetricsSource interface. The metrics are
  1278. // maintained in the meek session type; but logTunnel, which calls
  1279. // MetricsSource.GetMetrics, has a pointer only to this conn, so it calls
  1280. // through to the session.
  1281. func (conn *meekConn) GetMetrics() common.LogFields {
  1282. logFields := conn.meekSession.GetMetrics()
  1283. if conn.meekServer.passthroughAddress != "" {
  1284. logFields["passthrough_address"] = conn.meekServer.passthroughAddress
  1285. }
  1286. return logFields
  1287. }