inproxy.go 91 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672
  1. /*
  2. * Copyright (c) 2023, Psiphon Inc.
  3. * All rights reserved.
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation, either version 3 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. *
  18. */
  19. package psiphon
  20. import (
  21. "bytes"
  22. "context"
  23. "encoding/binary"
  24. std_errors "errors"
  25. "fmt"
  26. "io"
  27. "net"
  28. "net/http"
  29. "net/netip"
  30. "strconv"
  31. "sync"
  32. "sync/atomic"
  33. "syscall"
  34. "time"
  35. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
  36. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
  37. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/fragmentor"
  38. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/inproxy"
  39. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters"
  40. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
  41. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
  42. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/resolver"
  43. utls "github.com/Psiphon-Labs/utls"
  44. "github.com/cespare/xxhash"
  45. "golang.org/x/net/bpf"
  46. )
  47. // InproxyBrokerClientManager manages an InproxyBrokerClientInstance, an
  48. // in-proxy broker client, and its associated broker dial parameters, that
  49. // may be shared by multiple client dials or proxy instances. There is no
  50. // explicit close operation for the managed InproxyBrokerClientInstance.
  51. //
  52. // Once used, the current InproxyBrokerClientInstance and its broker client is
  53. // left actively connected to the broker, to minimize transport round trips
  54. // for additional requests.
  55. //
  56. // The InproxyBrokerClientManager and its components implement a replay system
  57. // for broker client dials. As one broker client is shared access multiple
  58. // client in-proxy dials, the broker dial parameters are replayed
  59. // independently from tunnel dial parameters.
  60. //
  61. // The NewInproxyBrokerClientInstance layer provides a fixed association
  62. // between a broker client and its broker dial parameters, ensuring that
  63. // in-proxy success/failure callbacks reference the correct replay parameters
  64. // when setting or clearing replay.
  65. //
  66. // A new InproxyBrokerClientInstance, including the broker dial parameters and
  67. // broker client, is instantiated when the active network ID changes, using
  68. // tactics for the new network.
  69. type InproxyBrokerClientManager struct {
  70. config *Config
  71. isProxy bool
  72. mutex sync.Mutex
  73. brokerSelectCount int
  74. networkID string
  75. brokerClientInstance *InproxyBrokerClientInstance
  76. }
  77. // NewInproxyBrokerClientManager creates a new InproxyBrokerClientManager.
  78. // NewInproxyBrokerClientManager does not perform any network operations; the
  79. // managed InproxyBrokerClientInstance is initialized when used for a round
  80. // trip.
  81. func NewInproxyBrokerClientManager(
  82. config *Config, isProxy bool) *InproxyBrokerClientManager {
  83. b := &InproxyBrokerClientManager{
  84. config: config,
  85. isProxy: isProxy,
  86. }
  87. // b.brokerClientInstance is initialized on demand, when getBrokerClient
  88. // is called.
  89. return b
  90. }
  91. // TacticsApplied implements the TacticsAppliedReceiver interface, and is
  92. // called when tactics have changed, which triggers a broker client reset in
  93. // order to apply potentially changed parameters.
  94. func (b *InproxyBrokerClientManager) TacticsApplied() error {
  95. b.mutex.Lock()
  96. defer b.mutex.Unlock()
  97. // Don't reset when not yet initialized; b.brokerClientInstance is
  98. // initialized only on demand.
  99. if b.brokerClientInstance == nil {
  100. return nil
  101. }
  102. // TODO: as a future future enhancement, don't reset when the tactics
  103. // brokerSpecs.Hash() is unchanged?
  104. return errors.Trace(b.reset(resetBrokerClientReasonTacticsApplied))
  105. }
  106. // GetBrokerClient returns the current, shared broker client and its
  107. // corresponding dial parametrers (for metrics logging). If there is no
  108. // current broker client, if the network ID differs from the network ID
  109. // associated with the previous broker client, a new broker client is
  110. // initialized.
  111. func (b *InproxyBrokerClientManager) GetBrokerClient(
  112. networkID string) (*inproxy.BrokerClient, *InproxyBrokerDialParameters, error) {
  113. b.mutex.Lock()
  114. defer b.mutex.Unlock()
  115. if b.brokerClientInstance == nil || b.networkID != networkID {
  116. err := b.reset(resetBrokerClientReasonInit)
  117. if err != nil {
  118. return nil, nil, errors.Trace(err)
  119. }
  120. }
  121. // The b.brokerClientInstance.brokerClient is wired up to refer back to
  122. // b.brokerClientInstance.brokerDialParams/roundTripper, etc.
  123. return b.brokerClientInstance.brokerClient,
  124. b.brokerClientInstance.brokerDialParams,
  125. nil
  126. }
  127. func (b *InproxyBrokerClientManager) resetBrokerClientOnRoundTripperFailed(
  128. brokerClientInstance *InproxyBrokerClientInstance) error {
  129. b.mutex.Lock()
  130. defer b.mutex.Unlock()
  131. if b.brokerClientInstance != brokerClientInstance {
  132. // Ignore the reset if the signal comes from the non-current
  133. // brokerClientInstance, which may occur when multiple in-flight
  134. // round trips fail in close proximity.
  135. return nil
  136. }
  137. return errors.Trace(b.reset(resetBrokerClientReasonRoundTripperFailed))
  138. }
  139. func (b *InproxyBrokerClientManager) resetBrokerClientOnNoMatch(
  140. brokerClientInstance *InproxyBrokerClientInstance) error {
  141. // Ignore the no match callback for proxies. For personal pairing, the
  142. // broker rotation scheme has clients moving brokers to find relatively
  143. // static proxies. For common pairing, we want to achieve balanced supply
  144. // across brokers.
  145. //
  146. // Currently, inproxy.BrokerDialCoordinator.BrokerClientNoMatch is only
  147. // wired up for clients, but this check ensures it'll still be ignored in
  148. // case that changes.
  149. if b.isProxy {
  150. return nil
  151. }
  152. if b.brokerClientInstance != brokerClientInstance {
  153. // See comment for same logic in resetBrokerClientOnRoundTripperFailed.
  154. return nil
  155. }
  156. p := b.config.GetParameters().Get()
  157. defer p.Close()
  158. probability := parameters.InproxyClientNoMatchFailoverProbability
  159. if b.config.IsInproxyPersonalPairingMode() {
  160. probability = parameters.InproxyClientNoMatchFailoverPersonalProbability
  161. }
  162. if !p.WeightedCoinFlip(probability) {
  163. return nil
  164. }
  165. return errors.Trace(b.reset(resetBrokerClientReasonRoundNoMatch))
  166. }
  167. type resetBrokerClientReason int
  168. const (
  169. resetBrokerClientReasonInit resetBrokerClientReason = iota + 1
  170. resetBrokerClientReasonTacticsApplied
  171. resetBrokerClientReasonRoundTripperFailed
  172. resetBrokerClientReasonRoundNoMatch
  173. )
  174. func (b *InproxyBrokerClientManager) reset(reason resetBrokerClientReason) error {
  175. // Assumes b.mutex lock is held.
  176. if b.brokerClientInstance != nil {
  177. // Close the existing broker client. This will close all underlying
  178. // network connections, interrupting any in-flight requests. This
  179. // close is invoked in the resetBrokerClientOnRoundTripperFailed
  180. // case, where it's expected that the round tripped has permanently
  181. // failed.
  182. b.brokerClientInstance.Close()
  183. }
  184. // b.brokerSelectCount tracks the number of broker resets and is used to
  185. // iterate over the brokers in a deterministic rotation when running in
  186. // personal pairing mode.
  187. switch reason {
  188. case resetBrokerClientReasonInit,
  189. resetBrokerClientReasonTacticsApplied:
  190. b.brokerSelectCount = 0
  191. case resetBrokerClientReasonRoundTripperFailed,
  192. resetBrokerClientReasonRoundNoMatch:
  193. b.brokerSelectCount += 1
  194. }
  195. // Any existing broker client is removed, even if
  196. // NewInproxyBrokerClientInstance fails. This ensures, for example, that
  197. // an existing broker client is removed when its spec is no longer
  198. // available in tactics.
  199. b.networkID = ""
  200. b.brokerClientInstance = nil
  201. networkID := b.config.GetNetworkID()
  202. brokerClientInstance, err := NewInproxyBrokerClientInstance(
  203. b.config,
  204. b,
  205. networkID,
  206. b.isProxy,
  207. b.brokerSelectCount,
  208. reason == resetBrokerClientReasonRoundNoMatch)
  209. if err != nil {
  210. return errors.Trace(err)
  211. }
  212. b.networkID = networkID
  213. b.brokerClientInstance = brokerClientInstance
  214. return nil
  215. }
  216. // InproxyBrokerClientInstance pairs an inproxy.BrokerClient instance with an
  217. // implementation of the inproxy.BrokerDialCoordinator interface and the
  218. // associated, underlying broker dial parameters. InproxyBrokerClientInstance
  219. // implements broker client dial replay.
  220. type InproxyBrokerClientInstance struct {
  221. config *Config
  222. brokerClientManager *InproxyBrokerClientManager
  223. networkID string
  224. brokerClientPrivateKey inproxy.SessionPrivateKey
  225. brokerClient *inproxy.BrokerClient
  226. brokerPublicKey inproxy.SessionPublicKey
  227. brokerRootObfuscationSecret inproxy.ObfuscationSecret
  228. brokerDialParams *InproxyBrokerDialParameters
  229. replayEnabled bool
  230. isReplay bool
  231. roundTripper *InproxyBrokerRoundTripper
  232. personalCompartmentIDs []inproxy.ID
  233. commonCompartmentIDs []inproxy.ID
  234. sessionHandshakeTimeout time.Duration
  235. announceRequestTimeout time.Duration
  236. announceDelay time.Duration
  237. announceDelayJitter float64
  238. answerRequestTimeout time.Duration
  239. offerRequestTimeout time.Duration
  240. offerRequestPersonalTimeout time.Duration
  241. offerRetryDelay time.Duration
  242. offerRetryJitter float64
  243. relayedPacketRequestTimeout time.Duration
  244. replayRetainFailedProbability float64
  245. replayUpdateFrequency time.Duration
  246. mutex sync.Mutex
  247. lastStoreReplay time.Time
  248. }
  249. // NewInproxyBrokerClientInstance creates a new InproxyBrokerClientInstance.
  250. // NewInproxyBrokerClientManager does not perform any network operations; the
  251. // new InproxyBrokerClientInstance is initialized when used for a round
  252. // trip.
  253. func NewInproxyBrokerClientInstance(
  254. config *Config,
  255. brokerClientManager *InproxyBrokerClientManager,
  256. networkID string,
  257. isProxy bool,
  258. brokerSelectCount int,
  259. resetReasonNoMatch bool) (*InproxyBrokerClientInstance, error) {
  260. p := config.GetParameters().Get()
  261. defer p.Close()
  262. // Select common or personal compartment IDs. Clients must provide at
  263. // least on compartment ID.
  264. commonCompartmentIDs, personalCompartmentIDs, err := prepareCompartmentIDs(config, p, isProxy)
  265. if err != nil {
  266. return nil, errors.Trace(err)
  267. }
  268. if !isProxy && len(commonCompartmentIDs) == 0 && len(personalCompartmentIDs) == 0 {
  269. return nil, errors.TraceNew("no compartment IDs")
  270. }
  271. if len(personalCompartmentIDs) > 1 {
  272. return nil, errors.TraceNew("unexpected multiple personal compartment IDs")
  273. }
  274. // Select the broker to use, optionally favoring brokers with replay data.
  275. // In the InproxyBrokerSpecs calls, the first non-empty tactics parameter
  276. // list is used.
  277. //
  278. // Optional broker specs may be used to specify broker(s) dedicated to
  279. // personal pairing, a configuration which can be used to reserve more
  280. // capacity for personal pairing, given the simple rendezvous scheme below.
  281. var brokerSpecs parameters.InproxyBrokerSpecsValue
  282. if isProxy {
  283. if config.IsInproxyPersonalPairingMode() {
  284. brokerSpecs = p.InproxyBrokerSpecs(
  285. parameters.InproxyProxyPersonalPairingBrokerSpecs,
  286. parameters.InproxyPersonalPairingBrokerSpecs,
  287. parameters.InproxyProxyBrokerSpecs,
  288. parameters.InproxyBrokerSpecs)
  289. } else {
  290. brokerSpecs = p.InproxyBrokerSpecs(
  291. parameters.InproxyProxyBrokerSpecs,
  292. parameters.InproxyBrokerSpecs)
  293. }
  294. } else {
  295. if config.IsInproxyPersonalPairingMode() {
  296. brokerSpecs = p.InproxyBrokerSpecs(
  297. parameters.InproxyClientPersonalPairingBrokerSpecs,
  298. parameters.InproxyPersonalPairingBrokerSpecs,
  299. parameters.InproxyClientBrokerSpecs,
  300. parameters.InproxyBrokerSpecs)
  301. } else {
  302. brokerSpecs = p.InproxyBrokerSpecs(
  303. parameters.InproxyClientBrokerSpecs,
  304. parameters.InproxyBrokerSpecs)
  305. }
  306. }
  307. if len(brokerSpecs) == 0 {
  308. return nil, errors.TraceNew("no broker specs")
  309. }
  310. // Select a broker.
  311. // In common pairing mode, the available brokers are shuffled before
  312. // selection, for random load balancing. Brokers with available dial
  313. // parameter replay data are preferred. When rotating brokers due to a no
  314. // match, the available replay data is ignored to increase the chance of
  315. // selecting a different broker.
  316. //
  317. // In personal pairing mode, arrange for the proxy and client to
  318. // rendezvous at the same broker by shuffling based on the shared
  319. // personal compartment ID. Both the client and proxy will select the
  320. // same initial broker, and fail over to other brokers in the same order.
  321. // By design, clients will move between brokers aggressively, rotating on
  322. // no-match responses and applying a shorter client offer timeout; while
  323. // proxies will remain in place in order to be found. Since rendezvous
  324. // depends on the ordering, each broker is selected in shuffle order;
  325. // dial parameter replay data is used when available but not considered
  326. // in selection ordering. The brokerSelectCount input is used to
  327. // progressively index into the list of shuffled brokers.
  328. //
  329. // Potential future enhancements:
  330. //
  331. // - Use brokerSelectCount in the common pairing case as well, to ensure
  332. // that a no-match reset always selects a different broker; but, unlike
  333. // the personal pairing logic, still prefer brokers with replay rather
  334. // than following a strict shuffle order.
  335. //
  336. // - The common pairing no match broker rotation is intended to partially
  337. // mitigate poor common proxy load balancing that can leave a broker
  338. // with little proxy supply. A more robust mitigation would be to make
  339. // proxies distribute announcements across multiple or even all brokers.
  340. personalPairing := len(personalCompartmentIDs) > 0
  341. // In the following cases, don't shuffle or otherwise mutate the original
  342. // broker spec slice, as it is a tactics parameter.
  343. if personalPairing {
  344. if len(personalCompartmentIDs[0]) < prng.SEED_LENGTH {
  345. // Both inproxy.ID and prng.SEED_LENGTH are 32 bytes.
  346. return nil, errors.TraceNew("unexpected ID length")
  347. }
  348. seed := prng.Seed(personalCompartmentIDs[0][0:prng.SEED_LENGTH])
  349. PRNG := prng.NewPRNGWithSeed(&seed)
  350. permutedIndexes := PRNG.Perm(len(brokerSpecs))
  351. selectedIndex := permutedIndexes[brokerSelectCount%len(permutedIndexes)]
  352. brokerSpecs = brokerSpecs[selectedIndex : selectedIndex+1]
  353. } else {
  354. permutedIndexes := prng.Perm(len(brokerSpecs))
  355. shuffledBrokerSpecs := make(parameters.InproxyBrokerSpecsValue, len(brokerSpecs))
  356. for i, index := range permutedIndexes {
  357. shuffledBrokerSpecs[i] = brokerSpecs[index]
  358. }
  359. brokerSpecs = shuffledBrokerSpecs
  360. }
  361. selectFirstCandidate := resetReasonNoMatch || personalPairing
  362. // Replay broker dial parameters.
  363. // In selectFirstCandidate cases, SelectCandidateWithNetworkReplayParameters
  364. // will always select the first candidate, returning corresponding replay
  365. // data when available. Otherwise, SelectCandidateWithNetworkReplayParameters
  366. // iterates over the shuffled candidates and returns the first with replay data.
  367. var brokerSpec *parameters.InproxyBrokerSpec
  368. var brokerDialParams *InproxyBrokerDialParameters
  369. // Replay is disabled when the TTL, InproxyReplayBrokerDialParametersTTL,
  370. // is 0.
  371. now := time.Now()
  372. ttl := p.Duration(parameters.InproxyReplayBrokerDialParametersTTL)
  373. replayEnabled := ttl > 0 &&
  374. !config.DisableReplay &&
  375. prng.FlipWeightedCoin(p.Float(parameters.InproxyReplayBrokerDialParametersProbability))
  376. if replayEnabled {
  377. brokerSpec, brokerDialParams, err =
  378. SelectCandidateWithNetworkReplayParameters[parameters.InproxyBrokerSpec, InproxyBrokerDialParameters](
  379. networkID,
  380. selectFirstCandidate,
  381. brokerSpecs,
  382. func(spec *parameters.InproxyBrokerSpec) string { return spec.BrokerPublicKey },
  383. func(spec *parameters.InproxyBrokerSpec, dialParams *InproxyBrokerDialParameters) bool {
  384. return dialParams.LastUsedTimestamp.After(now.Add(-ttl)) &&
  385. bytes.Equal(dialParams.LastUsedBrokerSpecHash, hashBrokerSpec(spec))
  386. })
  387. if err != nil {
  388. NoticeWarning("SelectCandidateWithNetworkReplayParameters failed: %v", errors.Trace(err))
  389. // Continue without replay
  390. }
  391. }
  392. // Select the first broker in the shuffle when replay is not enabled or in
  393. // case SelectCandidateWithNetworkReplayParameters fails.
  394. if brokerSpec == nil {
  395. brokerSpec = brokerSpecs[0]
  396. }
  397. // Generate new broker dial parameters if not replaying. Later, isReplay
  398. // is used to report the replay metric.
  399. isReplay := brokerDialParams != nil
  400. if !isReplay {
  401. brokerDialParams, err = MakeInproxyBrokerDialParameters(config, p, networkID, brokerSpec)
  402. if err != nil {
  403. return nil, errors.Trace(err)
  404. }
  405. } else {
  406. brokerDialParams.brokerSpec = brokerSpec
  407. err := brokerDialParams.prepareDialConfigs(config, p, networkID, true, nil)
  408. if err != nil {
  409. return nil, errors.Trace(err)
  410. }
  411. }
  412. // Load broker key material.
  413. brokerPublicKey, err := inproxy.SessionPublicKeyFromString(brokerSpec.BrokerPublicKey)
  414. if err != nil {
  415. return nil, errors.Trace(err)
  416. }
  417. brokerRootObfuscationSecret, err := inproxy.ObfuscationSecretFromString(brokerSpec.BrokerRootObfuscationSecret)
  418. if err != nil {
  419. return nil, errors.Trace(err)
  420. }
  421. roundTripper := NewInproxyBrokerRoundTripper(p, brokerDialParams)
  422. // Clients always generate an ephemeral session key pair. Proxies may opt
  423. // to use a long-lived key pair for proxied traffic attribution.
  424. var brokerClientPrivateKey inproxy.SessionPrivateKey
  425. if isProxy && config.InproxyProxySessionPrivateKey != "" {
  426. brokerClientPrivateKey, err = inproxy.SessionPrivateKeyFromString(config.InproxyProxySessionPrivateKey)
  427. if err != nil {
  428. return nil, errors.Trace(err)
  429. }
  430. } else {
  431. brokerClientPrivateKey, err = inproxy.GenerateSessionPrivateKey()
  432. if err != nil {
  433. return nil, errors.Trace(err)
  434. }
  435. }
  436. // InproxyBrokerClientInstance implements the
  437. // inproxy.BrokerDialCoordinator interface and passes itself to
  438. // inproxy.NewBrokerClient in order to provide the round tripper, key
  439. // material, compartment IDs, timeouts, and other configuration to the
  440. // in-proxy broker client.
  441. //
  442. // Timeouts are not replayed, but snapshots are stored in the
  443. // InproxyBrokerClientInstance for efficient lookup.
  444. b := &InproxyBrokerClientInstance{
  445. config: config,
  446. brokerClientManager: brokerClientManager,
  447. networkID: networkID,
  448. brokerClientPrivateKey: brokerClientPrivateKey,
  449. brokerPublicKey: brokerPublicKey,
  450. brokerRootObfuscationSecret: brokerRootObfuscationSecret,
  451. brokerDialParams: brokerDialParams,
  452. replayEnabled: replayEnabled,
  453. isReplay: isReplay,
  454. roundTripper: roundTripper,
  455. personalCompartmentIDs: personalCompartmentIDs,
  456. commonCompartmentIDs: commonCompartmentIDs,
  457. sessionHandshakeTimeout: p.Duration(parameters.InproxySessionHandshakeRoundTripTimeout),
  458. announceRequestTimeout: p.Duration(parameters.InproxyProxyAnnounceRequestTimeout),
  459. announceDelay: p.Duration(parameters.InproxyProxyAnnounceDelay),
  460. announceDelayJitter: p.Float(parameters.InproxyProxyAnnounceDelayJitter),
  461. answerRequestTimeout: p.Duration(parameters.InproxyProxyAnswerRequestTimeout),
  462. offerRequestTimeout: p.Duration(parameters.InproxyClientOfferRequestTimeout),
  463. offerRequestPersonalTimeout: p.Duration(parameters.InproxyClientOfferRequestPersonalTimeout),
  464. offerRetryDelay: p.Duration(parameters.InproxyClientOfferRetryDelay),
  465. offerRetryJitter: p.Float(parameters.InproxyClientOfferRetryJitter),
  466. relayedPacketRequestTimeout: p.Duration(parameters.InproxyClientRelayedPacketRequestTimeout),
  467. replayRetainFailedProbability: p.Float(parameters.InproxyReplayBrokerRetainFailedProbability),
  468. replayUpdateFrequency: p.Duration(parameters.InproxyReplayBrokerUpdateFrequency),
  469. }
  470. // Adjust long-polling request timeouts to respect any maximum request
  471. // timeout supported by the provider fronting the request.
  472. maxRequestTimeout, ok := p.KeyDurations(
  473. parameters.InproxyFrontingProviderClientMaxRequestTimeouts)[brokerDialParams.FrontingProviderID]
  474. if ok && maxRequestTimeout > 0 {
  475. if b.announceRequestTimeout > maxRequestTimeout {
  476. b.announceRequestTimeout = maxRequestTimeout
  477. }
  478. if b.offerRequestTimeout > maxRequestTimeout {
  479. b.offerRequestTimeout = maxRequestTimeout
  480. }
  481. if b.offerRequestPersonalTimeout > maxRequestTimeout {
  482. b.offerRequestPersonalTimeout = maxRequestTimeout
  483. }
  484. }
  485. // Initialize broker client. This will start with a fresh broker session.
  486. //
  487. // When resetBrokerClientOnRoundTripperFailed is invoked due to a failure
  488. // at the transport level -- TLS or domain fronting --
  489. // NewInproxyBrokerClientInstance is invoked, resetting both the broker
  490. // client round tripper and the broker session. As a future enhancement,
  491. // consider distinguishing between transport and session errors and
  492. // retaining a valid established session when only the transport needs to
  493. // be reset/retried.
  494. b.brokerClient, err = inproxy.NewBrokerClient(b)
  495. if err != nil {
  496. return nil, errors.Trace(err)
  497. }
  498. return b, nil
  499. }
  500. func prepareCompartmentIDs(
  501. config *Config,
  502. p parameters.ParametersAccessor,
  503. isProxy bool) ([]inproxy.ID, []inproxy.ID, error) {
  504. // Personal compartment IDs are loaded from the tunnel-core config; these
  505. // are set by the external app based on user input/configuration of IDs
  506. // generated by or obtained from personal proxies. Both clients and
  507. // proxies send personal compartment IDs to the in-proxy broker. For
  508. // clients, when personal compartment IDs are configured, no common
  509. // compartment IDs are prepared, ensuring matches with only proxies that
  510. // supply the corresponding personal compartment IDs.
  511. //
  512. // Common compartment IDs are obtained from tactics and merged with
  513. // previously learned IDs stored in the local datastore. When new IDs are
  514. // obtained from tactics, the merged list is written back to the
  515. // datastore. This allows for schemes where common compartment IDs are
  516. // distributed to sets of clients, then removed from distibution, and
  517. // still used to match proxies to those sets of clients. Only clients
  518. // send common compartment IDs to the in-proxy broker. Proxies are
  519. // automatically assigned to common compartments by the broker.
  520. //
  521. // Maximum compartment ID list lengths are enforced to ensure broker
  522. // request sizes don't grow unbounded.
  523. //
  524. // Limitation: currently, in max length trimming, new common compartment
  525. // IDs take precedence over older IDs.
  526. maxCompartmentIDListLength := p.Int(parameters.InproxyMaxCompartmentIDListLength)
  527. // Personal compartment ID limitations:
  528. //
  529. // The broker API messages, ProxyAnnounceRequest and ClientOfferRequest,
  530. // support lists of personal compartment IDs. However, both the proxy and
  531. // the client are currently limited to specifying at most one personal
  532. // compartment ID due to the following limitations:
  533. //
  534. // - On the broker side, the matcher queue implementation supports at most
  535. // one proxy personal compartment ID. See inproxy/Matcher.Announce. The
  536. // broker currently enforces that at most one personal compartment ID
  537. // may be specified per ProxyAnnounceRequest.
  538. //
  539. // - On the proxy/client side, the personal pairing rendezvous logic --
  540. // which aims for proxies and clients to select the same initial broker
  541. // and same order of failover to other brokers -- uses a shuffle that
  542. // assumes both the proxy and client use the same single, personal
  543. // compartment ID
  544. var configPersonalCompartmentIDs []string
  545. if isProxy && len(config.InproxyProxyPersonalCompartmentID) > 0 {
  546. configPersonalCompartmentIDs = []string{config.InproxyProxyPersonalCompartmentID}
  547. } else if !isProxy && len(config.InproxyClientPersonalCompartmentID) > 0 {
  548. configPersonalCompartmentIDs = []string{config.InproxyClientPersonalCompartmentID}
  549. }
  550. personalCompartmentIDs, err := inproxy.IDsFromStrings(configPersonalCompartmentIDs)
  551. if err != nil {
  552. return nil, nil, errors.Trace(err)
  553. }
  554. if len(personalCompartmentIDs) > maxCompartmentIDListLength {
  555. // Trim the list. It's not expected that user-configured personal
  556. // compartment ID lists will exceed the max length.
  557. //
  558. // TODO: shuffle before trimming? Prioritize previous matches?
  559. personalCompartmentIDs = personalCompartmentIDs[:maxCompartmentIDListLength]
  560. }
  561. var commonCompartmentIDs []inproxy.ID
  562. if !isProxy && len(personalCompartmentIDs) == 0 {
  563. tacticsCommonCompartmentIDs := p.InproxyCompartmentIDs(parameters.InproxyCommonCompartmentIDs)
  564. knownCommonCompartmentIDs, err := LoadInproxyCommonCompartmentIDs()
  565. if err != nil {
  566. NoticeWarning("LoadInproxyCommonCompartmentIDs failed: %v", errors.Trace(err))
  567. // Continue with only the tactics common compartment IDs.
  568. }
  569. newCompartmentIDs := make([]string, 0, len(tacticsCommonCompartmentIDs))
  570. for _, compartmentID := range tacticsCommonCompartmentIDs {
  571. // TODO: faster lookup?
  572. if !common.Contains(knownCommonCompartmentIDs, compartmentID) {
  573. newCompartmentIDs = append(newCompartmentIDs, compartmentID)
  574. }
  575. }
  576. if len(newCompartmentIDs) > 0 {
  577. newCompartmentIDs = append(newCompartmentIDs, knownCommonCompartmentIDs...)
  578. // Locally store more than InproxyMaxCompartmentIDListLength known
  579. // common compartment IDs, in case the request limit parameter is
  580. // increased in the future.
  581. // maxPersistedCommonCompartmentIDListLength still limits the
  582. // length of the list to cap local memory and disk impact.
  583. maxPersistedCommonCompartmentIDListLength := 500 // ~16K
  584. if maxCompartmentIDListLength > maxPersistedCommonCompartmentIDListLength {
  585. maxPersistedCommonCompartmentIDListLength = maxCompartmentIDListLength
  586. }
  587. if len(newCompartmentIDs) > maxPersistedCommonCompartmentIDListLength {
  588. newCompartmentIDs = newCompartmentIDs[:maxPersistedCommonCompartmentIDListLength]
  589. }
  590. err := StoreInproxyCommonCompartmentIDs(newCompartmentIDs)
  591. if err != nil {
  592. NoticeWarning("StoreInproxyCommonCompartmentIDs failed: %v", errors.Trace(err))
  593. // Continue without persisting new common compartment IDs.
  594. }
  595. knownCommonCompartmentIDs = newCompartmentIDs
  596. }
  597. commonCompartmentIDs, err = inproxy.IDsFromStrings(knownCommonCompartmentIDs)
  598. if err != nil {
  599. return nil, nil, errors.Trace(err)
  600. }
  601. if len(commonCompartmentIDs) > maxCompartmentIDListLength {
  602. // TODO: shuffle before trimming? Prioritize previous matches?
  603. commonCompartmentIDs = commonCompartmentIDs[:maxCompartmentIDListLength]
  604. }
  605. }
  606. return commonCompartmentIDs, personalCompartmentIDs, nil
  607. }
  608. // Close closes the broker client round tripped, including closing all
  609. // underlying network connections, which will interrupt any in-flight round
  610. // trips.
  611. func (b *InproxyBrokerClientInstance) Close() error {
  612. err := b.roundTripper.Close()
  613. return errors.Trace(err)
  614. }
  615. // Implements the inproxy.BrokerDialCoordinator interface.
  616. func (b *InproxyBrokerClientInstance) NetworkID() string {
  617. return b.networkID
  618. }
  619. // Implements the inproxy.BrokerDialCoordinator interface.
  620. func (b *InproxyBrokerClientInstance) NetworkType() inproxy.NetworkType {
  621. return getInproxyNetworkType(GetNetworkType(b.networkID))
  622. }
  623. // Implements the inproxy.BrokerDialCoordinator interface.
  624. func (b *InproxyBrokerClientInstance) CommonCompartmentIDs() []inproxy.ID {
  625. return b.commonCompartmentIDs
  626. }
  627. // Implements the inproxy.BrokerDialCoordinator interface.
  628. func (b *InproxyBrokerClientInstance) PersonalCompartmentIDs() []inproxy.ID {
  629. return b.personalCompartmentIDs
  630. }
  631. // Implements the inproxy.BrokerDialCoordinator interface.
  632. func (b *InproxyBrokerClientInstance) BrokerClientPrivateKey() inproxy.SessionPrivateKey {
  633. return b.brokerClientPrivateKey
  634. }
  635. // Implements the inproxy.BrokerDialCoordinator interface.
  636. func (b *InproxyBrokerClientInstance) BrokerPublicKey() inproxy.SessionPublicKey {
  637. return b.brokerPublicKey
  638. }
  639. // Implements the inproxy.BrokerDialCoordinator interface.
  640. func (b *InproxyBrokerClientInstance) BrokerRootObfuscationSecret() inproxy.ObfuscationSecret {
  641. return b.brokerRootObfuscationSecret
  642. }
  643. // Implements the inproxy.BrokerDialCoordinator interface.
  644. func (b *InproxyBrokerClientInstance) BrokerClientRoundTripper() (inproxy.RoundTripper, error) {
  645. // Returns the same round tripper for the lifetime of the
  646. // inproxy.BrokerDialCoordinator, ensuring all requests for one in-proxy
  647. // dial or proxy relay use the same broker, as is necessary due to the
  648. // broker state for the proxy announce/answer, client broker/server
  649. // relay, etc.
  650. return b.roundTripper, nil
  651. }
  652. // Implements the inproxy.BrokerDialCoordinator interface.
  653. func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperSucceeded(roundTripper inproxy.RoundTripper) {
  654. b.mutex.Lock()
  655. defer b.mutex.Unlock()
  656. if rt, ok := roundTripper.(*InproxyBrokerRoundTripper); !ok || rt != b.roundTripper {
  657. // Passing in the round tripper obtained from BrokerClientRoundTripper
  658. // is just used for sanity check in this implementation, since each
  659. // InproxyBrokerClientInstance has exactly one round tripper.
  660. NoticeError("BrokerClientRoundTripperSucceeded: roundTripper instance mismatch")
  661. return
  662. }
  663. // Set replay or extend the broker dial parameters replay TTL after a
  664. // success. With tunnel dial parameters, the replay TTL is extended after
  665. // every successful tunnel connection. Since there are potentially more
  666. // and more frequent broker round trips one tunnel dial, the TTL is only
  667. // extended after some target duration has elapsed, to avoid excessive
  668. // datastore writes.
  669. now := time.Now()
  670. if b.replayEnabled && now.Sub(b.lastStoreReplay) > b.replayUpdateFrequency {
  671. b.brokerDialParams.LastUsedTimestamp = time.Now()
  672. err := SetNetworkReplayParameters[InproxyBrokerDialParameters](
  673. b.networkID, b.brokerDialParams.brokerSpec.BrokerPublicKey, b.brokerDialParams)
  674. if err != nil {
  675. NoticeWarning("StoreBrokerDialParameters failed: %v", errors.Trace(err))
  676. // Continue without persisting replay changes.
  677. } else {
  678. b.lastStoreReplay = now
  679. }
  680. }
  681. // Verify/extend the resolver cache entry for any resolved domain after a
  682. // success.
  683. //
  684. // Limitation: currently this re-extends regardless of how long ago the DNS
  685. // resolve happened.
  686. resolver := b.config.GetResolver()
  687. if resolver != nil {
  688. resolver.VerifyCacheExtension(b.brokerDialParams.FrontingDialAddress)
  689. }
  690. }
  691. // Implements the inproxy.BrokerDialCoordinator interface.
  692. func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperFailed(roundTripper inproxy.RoundTripper) {
  693. b.mutex.Lock()
  694. defer b.mutex.Unlock()
  695. if rt, ok := roundTripper.(*InproxyBrokerRoundTripper); !ok || rt != b.roundTripper {
  696. // Passing in the round tripper obtained from BrokerClientRoundTripper
  697. // is just used for sanity check in this implementation, since each
  698. // InproxyBrokerClientInstance has exactly one round tripper.
  699. NoticeError("BrokerClientRoundTripperFailed: roundTripper instance mismatch")
  700. return
  701. }
  702. // Delete any persistent replay dial parameters. Unlike with the success
  703. // case, consecutive, repeated deletes shouldn't write to storage, so
  704. // they are not avoided.
  705. if b.replayEnabled &&
  706. !prng.FlipWeightedCoin(b.replayRetainFailedProbability) {
  707. // Limitation: there's a race condition with multiple
  708. // InproxyBrokerClientInstances writing to the replay datastore for
  709. // the same broker, such as in the case where there's a dual-mode
  710. // in-proxy client and proxy; this delete could potentially clobber a
  711. // concurrent fresh replay store after a success.
  712. //
  713. // TODO: add an additional storage key distinguisher for each instance?
  714. err := DeleteNetworkReplayParameters[InproxyBrokerDialParameters](
  715. b.networkID, b.brokerDialParams.brokerSpec.BrokerPublicKey)
  716. if err != nil {
  717. NoticeWarning("DeleteBrokerDialParameters failed: %v", errors.Trace(err))
  718. // Continue without resetting replay.
  719. }
  720. }
  721. // Invoke resetBrokerClientOnRoundTripperFailed to signal the
  722. // InproxyBrokerClientManager to create a new
  723. // InproxyBrokerClientInstance, with new dial parameters and a new round
  724. // tripper, after a failure.
  725. //
  726. // This InproxyBrokerClientInstance doesn't change its dial parameters or
  727. // round tripper to ensure that any concurrent usage retains affinity
  728. // with the same parameters and broker.
  729. //
  730. // Limitation: a transport-level failure may unnecessarily reset the
  731. // broker session state; see comment in NewInproxyBrokerClientInstance.
  732. err := b.brokerClientManager.resetBrokerClientOnRoundTripperFailed(b)
  733. if err != nil {
  734. NoticeWarning("reset broker client failed: %v", errors.Trace(err))
  735. // Continue with old broker client instance.
  736. }
  737. }
  738. // Implements the inproxy.BrokerDialCoordinator interface.
  739. func (b *InproxyBrokerClientInstance) BrokerClientNoMatch(roundTripper inproxy.RoundTripper) {
  740. b.mutex.Lock()
  741. defer b.mutex.Unlock()
  742. if rt, ok := roundTripper.(*InproxyBrokerRoundTripper); !ok || rt != b.roundTripper {
  743. // See roundTripper check comment in BrokerClientRoundTripperFailed.
  744. NoticeError("BrokerClientNoMatch: roundTripper instance mismatch")
  745. return
  746. }
  747. // Any persistent replay dial parameters are retained and not deleted,
  748. // since the broker client successfully transacted with the broker.
  749. err := b.brokerClientManager.resetBrokerClientOnNoMatch(b)
  750. if err != nil {
  751. NoticeWarning("reset broker client failed: %v", errors.Trace(err))
  752. // Continue with old broker client instance.
  753. }
  754. }
  755. // Implements the inproxy.BrokerDialCoordinator interface.
  756. func (b *InproxyBrokerClientInstance) AnnounceRequestTimeout() time.Duration {
  757. return b.announceRequestTimeout
  758. }
  759. // Implements the inproxy.BrokerDialCoordinator interface.
  760. func (b *InproxyBrokerClientInstance) SessionHandshakeRoundTripTimeout() time.Duration {
  761. return b.sessionHandshakeTimeout
  762. }
  763. // Implements the inproxy.BrokerDialCoordinator interface.
  764. func (b *InproxyBrokerClientInstance) AnnounceDelay() time.Duration {
  765. return b.announceDelay
  766. }
  767. // Implements the inproxy.BrokerDialCoordinator interface.
  768. func (b *InproxyBrokerClientInstance) AnnounceDelayJitter() float64 {
  769. return b.announceDelayJitter
  770. }
  771. // Implements the inproxy.BrokerDialCoordinator interface.
  772. func (b *InproxyBrokerClientInstance) AnswerRequestTimeout() time.Duration {
  773. return b.answerRequestTimeout
  774. }
  775. // Implements the inproxy.BrokerDialCoordinator interface.
  776. func (b *InproxyBrokerClientInstance) OfferRequestTimeout() time.Duration {
  777. return b.offerRequestTimeout
  778. }
  779. // Implements the inproxy.BrokerDialCoordinator interface.
  780. func (b *InproxyBrokerClientInstance) OfferRequestPersonalTimeout() time.Duration {
  781. return b.offerRequestPersonalTimeout
  782. }
  783. // Implements the inproxy.BrokerDialCoordinator interface.
  784. func (b *InproxyBrokerClientInstance) OfferRetryDelay() time.Duration {
  785. return b.offerRetryDelay
  786. }
  787. // Implements the inproxy.BrokerDialCoordinator interface.
  788. func (b *InproxyBrokerClientInstance) OfferRetryJitter() float64 {
  789. return b.offerRetryJitter
  790. }
  791. // Implements the inproxy.BrokerDialCoordinator interface.
  792. func (b *InproxyBrokerClientInstance) RelayedPacketRequestTimeout() time.Duration {
  793. return b.relayedPacketRequestTimeout
  794. }
  795. // InproxyBrokerDialParameters represents a selected broker transport and dial
  796. // paramaters.
  797. //
  798. // InproxyBrokerDialParameters is used to configure dialers; as a persistent
  799. // record to store successful dial parameters for replay; and to report dial
  800. // stats in notices and Psiphon API calls.
  801. //
  802. // InproxyBrokerDialParameters is similar to tunnel DialParameters, but is
  803. // specific to the in-proxy broker dial phase.
  804. type InproxyBrokerDialParameters struct {
  805. brokerSpec *parameters.InproxyBrokerSpec `json:"-"`
  806. isReplay bool `json:"-"`
  807. LastUsedTimestamp time.Time
  808. LastUsedBrokerSpecHash []byte
  809. NetworkLatencyMultiplier float64
  810. BrokerTransport string
  811. DialAddress string
  812. FrontingProviderID string
  813. FrontingDialAddress string
  814. SNIServerName string
  815. TransformedHostName bool
  816. VerifyServerName string
  817. VerifyPins []string
  818. HostHeader string
  819. ResolvedIPAddress atomic.Value `json:"-"`
  820. TLSProfile string
  821. TLSVersion string
  822. RandomizedTLSProfileSeed *prng.Seed
  823. NoDefaultTLSSessionID bool
  824. TLSFragmentClientHello bool
  825. SelectedUserAgent bool
  826. UserAgent string
  827. BPFProgramName string
  828. BPFProgramInstructions []bpf.RawInstruction
  829. FragmentorSeed *prng.Seed
  830. ResolveParameters *resolver.ResolveParameters
  831. dialConfig *DialConfig `json:"-"`
  832. meekConfig *MeekConfig `json:"-"`
  833. }
  834. // MakeInproxyBrokerDialParameters creates a new InproxyBrokerDialParameters.
  835. func MakeInproxyBrokerDialParameters(
  836. config *Config,
  837. p parameters.ParametersAccessor,
  838. networkID string,
  839. brokerSpec *parameters.InproxyBrokerSpec) (*InproxyBrokerDialParameters, error) {
  840. // This function duplicates some code from MakeDialParameters and
  841. // makeFrontedHTTPClient. To simplify the logic, the Replay<Component>
  842. // tactic flags for individual dial components are ignored.
  843. //
  844. // TODO: merge common functionality?
  845. if config.UseUpstreamProxy() {
  846. return nil, errors.TraceNew("upstream proxy unsupported")
  847. }
  848. currentTimestamp := time.Now()
  849. var brokerDialParams *InproxyBrokerDialParameters
  850. // Select new broker dial parameters
  851. brokerDialParams = &InproxyBrokerDialParameters{
  852. brokerSpec: brokerSpec,
  853. LastUsedTimestamp: currentTimestamp,
  854. LastUsedBrokerSpecHash: hashBrokerSpec(brokerSpec),
  855. }
  856. // Network latency multiplier
  857. brokerDialParams.NetworkLatencyMultiplier = prng.ExpFloat64Range(
  858. p.Float(parameters.NetworkLatencyMultiplierMin),
  859. p.Float(parameters.NetworkLatencyMultiplierMax),
  860. p.Float(parameters.NetworkLatencyMultiplierLambda))
  861. // Select fronting configuration
  862. var err error
  863. brokerDialParams.FrontingProviderID,
  864. brokerDialParams.BrokerTransport,
  865. brokerDialParams.FrontingDialAddress,
  866. brokerDialParams.SNIServerName,
  867. brokerDialParams.VerifyServerName,
  868. brokerDialParams.VerifyPins,
  869. brokerDialParams.HostHeader,
  870. err = brokerDialParams.brokerSpec.BrokerFrontingSpecs.SelectParameters()
  871. if err != nil {
  872. return nil, errors.Trace(err)
  873. }
  874. // At this time, the broker client, the transport is limited to fronted
  875. // HTTPS.
  876. //
  877. // As a future enhancement, allow HTTP for the in-proxy broker case, skip
  878. // selecting TLS tactics and select HTTP tactics such as
  879. // HTTPTransformerParameters.
  880. if brokerDialParams.BrokerTransport == protocol.FRONTING_TRANSPORT_HTTP {
  881. return nil, errors.TraceNew("unsupported fronting transport")
  882. }
  883. // Determine and use the equivilent tunnel protocol for tactics
  884. // selections. For example, for the broker transport FRONTED-HTTPS, use
  885. // the tactics for FRONTED-MEEK-OSSH.
  886. equivilentTunnelProtocol, err := protocol.EquivilentTunnelProtocol(brokerDialParams.BrokerTransport)
  887. if err != nil {
  888. return nil, errors.Trace(err)
  889. }
  890. // FrontSpec.Addresses may include a port; default to 443 if none.
  891. if _, _, err := net.SplitHostPort(brokerDialParams.FrontingDialAddress); err == nil {
  892. brokerDialParams.DialAddress = brokerDialParams.FrontingDialAddress
  893. } else {
  894. brokerDialParams.DialAddress = net.JoinHostPort(brokerDialParams.FrontingDialAddress, "443")
  895. }
  896. // SNI configuration
  897. //
  898. // For a FrontingSpec, an SNI value of "" indicates to disable/omit SNI, so
  899. // never transform in that case.
  900. if brokerDialParams.SNIServerName != "" {
  901. if p.WeightedCoinFlip(parameters.TransformHostNameProbability) {
  902. brokerDialParams.SNIServerName = selectHostName(equivilentTunnelProtocol, p)
  903. brokerDialParams.TransformedHostName = true
  904. }
  905. }
  906. // TLS configuration
  907. //
  908. // The requireTLS13 flag is set to true in order to use only modern TLS
  909. // fingerprints which should support HTTP/2 in the ALPN.
  910. //
  911. // TODO: TLS padding, NoDefaultTLSSessionID
  912. brokerDialParams.TLSProfile,
  913. brokerDialParams.TLSVersion,
  914. brokerDialParams.RandomizedTLSProfileSeed,
  915. err = SelectTLSProfile(false, true, true, brokerDialParams.FrontingProviderID, p)
  916. brokerDialParams.NoDefaultTLSSessionID = p.WeightedCoinFlip(
  917. parameters.NoDefaultTLSSessionIDProbability)
  918. if brokerDialParams.SNIServerName != "" && net.ParseIP(brokerDialParams.SNIServerName) == nil {
  919. tlsFragmentorLimitProtocols := p.TunnelProtocols(parameters.TLSFragmentClientHelloLimitProtocols)
  920. if len(tlsFragmentorLimitProtocols) == 0 || common.Contains(tlsFragmentorLimitProtocols, equivilentTunnelProtocol) {
  921. brokerDialParams.TLSFragmentClientHello = p.WeightedCoinFlip(parameters.TLSFragmentClientHelloProbability)
  922. }
  923. }
  924. // User Agent configuration
  925. dialCustomHeaders := makeDialCustomHeaders(config, p)
  926. brokerDialParams.SelectedUserAgent, brokerDialParams.UserAgent = selectUserAgentIfUnset(p, dialCustomHeaders)
  927. // BPF configuration
  928. if ClientBPFEnabled() &&
  929. protocol.TunnelProtocolMayUseClientBPF(equivilentTunnelProtocol) {
  930. if p.WeightedCoinFlip(parameters.BPFClientTCPProbability) {
  931. brokerDialParams.BPFProgramName = ""
  932. brokerDialParams.BPFProgramInstructions = nil
  933. ok, name, rawInstructions := p.BPFProgram(parameters.BPFClientTCPProgram)
  934. if ok {
  935. brokerDialParams.BPFProgramName = name
  936. brokerDialParams.BPFProgramInstructions = rawInstructions
  937. }
  938. }
  939. }
  940. // Fragmentor configuration
  941. brokerDialParams.FragmentorSeed, err = prng.NewSeed()
  942. if err != nil {
  943. return nil, errors.Trace(err)
  944. }
  945. // Resolver configuration
  946. //
  947. // The custom resolcer is wired up only when there is a domain to be
  948. // resolved; GetMetrics will log resolver metrics when the resolver is set.
  949. if net.ParseIP(brokerDialParams.FrontingDialAddress) == nil {
  950. resolver := config.GetResolver()
  951. if resolver == nil {
  952. return nil, errors.TraceNew("missing resolver")
  953. }
  954. brokerDialParams.ResolveParameters, err = resolver.MakeResolveParameters(
  955. p, brokerDialParams.FrontingProviderID, brokerDialParams.FrontingDialAddress)
  956. if err != nil {
  957. return nil, errors.Trace(err)
  958. }
  959. }
  960. // Initialize Dial/MeekConfigs to be passed to the corresponding dialers.
  961. err = brokerDialParams.prepareDialConfigs(config, p, networkID, false, dialCustomHeaders)
  962. if err != nil {
  963. return nil, errors.Trace(err)
  964. }
  965. return brokerDialParams, nil
  966. }
  967. // prepareDialConfigs is called for both new and replayed broker dial parameters.
  968. func (brokerDialParams *InproxyBrokerDialParameters) prepareDialConfigs(
  969. config *Config,
  970. p parameters.ParametersAccessor,
  971. networkID string,
  972. isReplay bool,
  973. dialCustomHeaders http.Header) error {
  974. brokerDialParams.isReplay = isReplay
  975. equivilentTunnelProtocol, err := protocol.EquivilentTunnelProtocol(brokerDialParams.BrokerTransport)
  976. if err != nil {
  977. return errors.Trace(err)
  978. }
  979. // Custom headers and User Agent
  980. if dialCustomHeaders == nil {
  981. dialCustomHeaders = makeDialCustomHeaders(config, p)
  982. }
  983. if brokerDialParams.SelectedUserAgent {
  984. // Limitation: if config.CustomHeaders adds a User-Agent between
  985. // replays, it may be ignored due to replaying a selected User-Agent.
  986. dialCustomHeaders.Set("User-Agent", brokerDialParams.UserAgent)
  987. }
  988. // Fragmentor
  989. fragmentorConfig := fragmentor.NewUpstreamConfig(
  990. p, equivilentTunnelProtocol, brokerDialParams.FragmentorSeed)
  991. // Resolver
  992. //
  993. // DialConfig.ResolveIP is required and called even when the destination
  994. // is an IP address.
  995. resolver := config.GetResolver()
  996. if resolver == nil {
  997. return errors.TraceNew("missing resolver")
  998. }
  999. resolveIP := func(ctx context.Context, hostname string) ([]net.IP, error) {
  1000. IPs, err := resolver.ResolveIP(
  1001. ctx, networkID, brokerDialParams.ResolveParameters, hostname)
  1002. return IPs, errors.Trace(err)
  1003. }
  1004. // DialConfig
  1005. brokerDialParams.ResolvedIPAddress.Store("")
  1006. brokerDialParams.dialConfig = &DialConfig{
  1007. DiagnosticID: brokerDialParams.brokerSpec.BrokerPublicKey,
  1008. CustomHeaders: dialCustomHeaders,
  1009. BPFProgramInstructions: brokerDialParams.BPFProgramInstructions,
  1010. DeviceBinder: config.deviceBinder,
  1011. IPv6Synthesizer: config.IPv6Synthesizer,
  1012. ResolveIP: resolveIP,
  1013. TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
  1014. FragmentorConfig: fragmentorConfig,
  1015. ResolvedIPCallback: func(IPAddress string) {
  1016. brokerDialParams.ResolvedIPAddress.Store(IPAddress)
  1017. },
  1018. }
  1019. // MeekDialConfig
  1020. //
  1021. // The broker round trips use MeekModeWrappedPlaintextRoundTrip without
  1022. // meek cookies, so meek obfuscation is not configured. The in-proxy
  1023. // broker session payloads have their own obfuscation layer.
  1024. addPsiphonFrontingHeader := false
  1025. if brokerDialParams.FrontingProviderID != "" {
  1026. addPsiphonFrontingHeader = common.Contains(
  1027. p.LabeledTunnelProtocols(
  1028. parameters.AddFrontingProviderPsiphonFrontingHeader,
  1029. brokerDialParams.FrontingProviderID),
  1030. equivilentTunnelProtocol)
  1031. }
  1032. brokerDialParams.meekConfig = &MeekConfig{
  1033. Mode: MeekModeWrappedPlaintextRoundTrip,
  1034. DiagnosticID: brokerDialParams.FrontingProviderID,
  1035. Parameters: config.GetParameters(),
  1036. DialAddress: brokerDialParams.DialAddress,
  1037. TLSProfile: brokerDialParams.TLSProfile,
  1038. NoDefaultTLSSessionID: brokerDialParams.NoDefaultTLSSessionID,
  1039. RandomizedTLSProfileSeed: brokerDialParams.RandomizedTLSProfileSeed,
  1040. SNIServerName: brokerDialParams.SNIServerName,
  1041. AddPsiphonFrontingHeader: addPsiphonFrontingHeader,
  1042. VerifyServerName: brokerDialParams.VerifyServerName,
  1043. VerifyPins: brokerDialParams.VerifyPins,
  1044. HostHeader: brokerDialParams.HostHeader,
  1045. TransformedHostName: brokerDialParams.TransformedHostName,
  1046. NetworkLatencyMultiplier: brokerDialParams.NetworkLatencyMultiplier,
  1047. AdditionalHeaders: config.MeekAdditionalHeaders,
  1048. TLSClientSessionCache: common.WrapUtlsClientSessionCache(utls.NewLRUClientSessionCache(0), brokerDialParams.DialAddress),
  1049. }
  1050. switch brokerDialParams.BrokerTransport {
  1051. case protocol.FRONTING_TRANSPORT_HTTPS:
  1052. brokerDialParams.meekConfig.UseHTTPS = true
  1053. case protocol.FRONTING_TRANSPORT_QUIC:
  1054. brokerDialParams.meekConfig.UseQUIC = true
  1055. }
  1056. return nil
  1057. }
  1058. // GetBrokerMetrics returns dial parameter log fields to be reported to a
  1059. // broker.
  1060. func (brokerDialParams *InproxyBrokerDialParameters) GetBrokerMetrics() common.LogFields {
  1061. logFields := common.LogFields{}
  1062. // TODO: add additional broker fronting dial parameters to be logged by
  1063. // the broker -- as successful parameters might not otherwise by logged
  1064. // via server_tunnel if the subsequent WebRTC dials fail.
  1065. logFields["fronting_provider_id"] = brokerDialParams.FrontingProviderID
  1066. return logFields
  1067. }
  1068. // GetMetrics implements the common.MetricsSource interface and returns log
  1069. // fields detailing the broker dial parameters.
  1070. func (brokerDialParams *InproxyBrokerDialParameters) GetMetrics() common.LogFields {
  1071. logFields := common.LogFields{}
  1072. logFields["inproxy_broker_transport"] = brokerDialParams.BrokerTransport
  1073. isReplay := "0"
  1074. if brokerDialParams.isReplay {
  1075. isReplay = "1"
  1076. }
  1077. logFields["inproxy_broker_is_replay"] = isReplay
  1078. // Note: as At the broker client transport is currently limited to domain
  1079. // fronted HTTPS, the following related parameters are included
  1080. // unconditionally.
  1081. logFields["inproxy_broker_fronting_provider_id"] = brokerDialParams.FrontingProviderID
  1082. logFields["inproxy_broker_dial_address"] = brokerDialParams.FrontingDialAddress
  1083. resolvedIPAddress := brokerDialParams.ResolvedIPAddress.Load().(string)
  1084. if resolvedIPAddress != "" {
  1085. logFields["inproxy_broker_resolved_ip_address"] = resolvedIPAddress
  1086. }
  1087. if brokerDialParams.SNIServerName != "" {
  1088. logFields["inproxy_broker_sni_server_name"] = brokerDialParams.SNIServerName
  1089. }
  1090. logFields["inproxy_broker_host_header"] = brokerDialParams.HostHeader
  1091. transformedHostName := "0"
  1092. if brokerDialParams.TransformedHostName {
  1093. transformedHostName = "1"
  1094. }
  1095. logFields["inproxy_broker_transformed_host_name"] = transformedHostName
  1096. if brokerDialParams.UserAgent != "" {
  1097. logFields["inproxy_broker_user_agent"] = brokerDialParams.UserAgent
  1098. }
  1099. if brokerDialParams.BrokerTransport == protocol.FRONTING_TRANSPORT_HTTPS {
  1100. if brokerDialParams.TLSProfile != "" {
  1101. logFields["inproxy_broker_tls_profile"] = brokerDialParams.TLSProfile
  1102. }
  1103. logFields["inproxy_broker_tls_version"] = brokerDialParams.TLSVersion
  1104. tlsFragmented := "0"
  1105. if brokerDialParams.TLSFragmentClientHello {
  1106. tlsFragmented = "1"
  1107. }
  1108. logFields["inproxy_broker_tls_fragmented"] = tlsFragmented
  1109. }
  1110. if brokerDialParams.BPFProgramName != "" {
  1111. logFields["inproxy_broker_client_bpf"] = brokerDialParams.BPFProgramName
  1112. }
  1113. if brokerDialParams.ResolveParameters != nil {
  1114. // See comment for dialParams.ResolveParameters handling in
  1115. // getBaseAPIParameters.
  1116. if brokerDialParams.ResolveParameters.PreresolvedIPAddress != "" {
  1117. dialDomain, _, _ := net.SplitHostPort(brokerDialParams.DialAddress)
  1118. if brokerDialParams.ResolveParameters.PreresolvedDomain == dialDomain {
  1119. logFields["inproxy_broker_dns_preresolved"] = brokerDialParams.ResolveParameters.PreresolvedIPAddress
  1120. }
  1121. }
  1122. if brokerDialParams.ResolveParameters.PreferAlternateDNSServer {
  1123. logFields["inproxy_broker_dns_preferred"] = brokerDialParams.ResolveParameters.AlternateDNSServer
  1124. }
  1125. if brokerDialParams.ResolveParameters.ProtocolTransformName != "" {
  1126. logFields["inproxy_broker_dns_transform"] = brokerDialParams.ResolveParameters.ProtocolTransformName
  1127. }
  1128. logFields["inproxy_broker_dns_attempt"] = strconv.Itoa(
  1129. brokerDialParams.ResolveParameters.GetFirstAttemptWithAnswer())
  1130. }
  1131. // TODO: get fragmentor metrics, if any, from MeekConn.
  1132. return logFields
  1133. }
  1134. // hashBrokerSpec hashes the broker spec. The hash is used to detect when
  1135. // broker spec tactics have changed.
  1136. func hashBrokerSpec(spec *parameters.InproxyBrokerSpec) []byte {
  1137. var hash [8]byte
  1138. binary.BigEndian.PutUint64(
  1139. hash[:],
  1140. uint64(xxhash.Sum64String(fmt.Sprintf("%+v", spec))))
  1141. return hash[:]
  1142. }
  1143. // InproxyBrokerRoundTripper is a broker request round trip transport
  1144. // implemented using MeekConn in MeekModePlaintextRoundTrip mode, utilizing
  1145. // MeekConn's domain fronting capabilities and using persistent and
  1146. // multiplexed connections, via HTTP/2, to support multiple concurrent
  1147. // in-flight round trips.
  1148. //
  1149. // InproxyBrokerRoundTripper implements the inproxy.RoundTripper interface.
  1150. type InproxyBrokerRoundTripper struct {
  1151. brokerDialParams *InproxyBrokerDialParameters
  1152. runCtx context.Context
  1153. stopRunning context.CancelFunc
  1154. dial int32
  1155. dialCompleted chan struct{}
  1156. dialErr error
  1157. conn *MeekConn
  1158. failureThreshold time.Duration
  1159. }
  1160. // NewInproxyBrokerRoundTripper creates a new InproxyBrokerRoundTripper. The
  1161. // initial DialMeek is defered until the first call to RoundTrip, so
  1162. // NewInproxyBrokerRoundTripper does not perform any network operations.
  1163. //
  1164. // The input brokerDialParams dial parameter and config fields must not
  1165. // modifed after NewInproxyBrokerRoundTripper is called.
  1166. func NewInproxyBrokerRoundTripper(
  1167. p parameters.ParametersAccessor,
  1168. brokerDialParams *InproxyBrokerDialParameters) *InproxyBrokerRoundTripper {
  1169. runCtx, stopRunning := context.WithCancel(context.Background())
  1170. return &InproxyBrokerRoundTripper{
  1171. brokerDialParams: brokerDialParams,
  1172. runCtx: runCtx,
  1173. stopRunning: stopRunning,
  1174. dialCompleted: make(chan struct{}),
  1175. failureThreshold: p.Duration(
  1176. parameters.InproxyBrokerRoundTripStatusCodeFailureThreshold),
  1177. }
  1178. }
  1179. // Close interrupts any in-flight request and closes the underlying
  1180. // MeekConn.
  1181. func (rt *InproxyBrokerRoundTripper) Close() error {
  1182. // Interrupt any DialMeek or RoundTrip.
  1183. rt.stopRunning()
  1184. if atomic.CompareAndSwapInt32(&rt.dial, 0, 1) {
  1185. // RoundTrip has not yet been called or has not yet kicked off
  1186. // DialMeek, so there is no MeekConn to close. Prevent any future
  1187. // DialMeek by signaling dialCompleted and fail any future round trip
  1188. // attempt by setting dialErr.
  1189. rt.dialErr = errors.TraceNew("closed")
  1190. close(rt.dialCompleted)
  1191. } else {
  1192. // Await any ongoing DialMeek or RoundTrip (stopRunning should
  1193. // interrupt either one quickly).
  1194. <-rt.dialCompleted
  1195. if rt.conn != nil {
  1196. _ = rt.conn.Close()
  1197. }
  1198. }
  1199. // As with MeekConn.Close, any Close errors from underlying conns are not
  1200. // propagated.
  1201. return nil
  1202. }
  1203. // RoundTrip transports a request to the broker endpoint and returns a
  1204. // response.
  1205. func (rt *InproxyBrokerRoundTripper) RoundTrip(
  1206. ctx context.Context,
  1207. roundTripDelay time.Duration,
  1208. roundTripTimeout time.Duration,
  1209. requestPayload []byte) (_ []byte, retErr error) {
  1210. defer func() {
  1211. // Log any error which results in invoking BrokerClientRoundTripperFailed.
  1212. var failedError *inproxy.RoundTripperFailedError
  1213. if std_errors.As(retErr, &failedError) {
  1214. NoticeWarning("RoundTripperFailedError: %v", retErr)
  1215. }
  1216. }()
  1217. // Cancel DialMeek or MeekConn.RoundTrip when:
  1218. // - Close is called
  1219. // - the input context is done
  1220. ctx, cancelFunc := common.MergeContextCancel(ctx, rt.runCtx)
  1221. defer cancelFunc()
  1222. // Apply any round trip delay. Currently, this is used to apply an
  1223. // announce request delay post-waitToShareSession, pre-network round
  1224. // trip, and cancelable by the above merged context.
  1225. if roundTripDelay > 0 {
  1226. common.SleepWithContext(ctx, roundTripDelay)
  1227. }
  1228. // Apply the round trip timeout after any delay is complete.
  1229. //
  1230. // This timeout includes any TLS handshake network round trips, as
  1231. // performed by the initial DialMeek and may be performed subsequently by
  1232. // net/http via MeekConn.RoundTrip. These extra round trips should be
  1233. // accounted for in the in the difference between client-side request
  1234. // timeouts, such as InproxyProxyAnswerRequestTimeout, and broker-side
  1235. // handler timeouts, such as InproxyBrokerProxyAnnounceTimeout, with the
  1236. // former allowing more time for network round trips.
  1237. requestCtx := ctx
  1238. if roundTripTimeout > 0 {
  1239. var requestCancelFunc context.CancelFunc
  1240. requestCtx, requestCancelFunc = context.WithTimeout(ctx, roundTripTimeout)
  1241. defer requestCancelFunc()
  1242. }
  1243. // The first RoundTrip caller will perform the DialMeek step, which
  1244. // establishes the TLS trasport connection to the fronted endpoint.
  1245. // Following callers will await that DialMeek or share an established
  1246. // connection.
  1247. //
  1248. // To accomodate using custom utls fingerprints, with varying ALPNs, with
  1249. // net/http, DialMeek completes a full TLS handshake before instantiating
  1250. // the appropriate http.Transport or http2.Transport. Until that first
  1251. // DialMeek completes, and unlike standard net/http round trips,
  1252. // InproxyBrokerRoundTripper won't spawn distinct TLS persistent
  1253. // connections for concurrent round trips. After DialMeek, concurrent
  1254. // round trips over HTTP/2 connections may simply share the one TLS
  1255. // connection, while concurrent round trips over HTTP connections may
  1256. // spawn additional TLS persistent connections.
  1257. //
  1258. // There is no retry here if DialMeek fails, as higher levels will invoke
  1259. // BrokerClientRoundTripperFailed on failure, clear any replay, select
  1260. // new dial parameters, and retry.
  1261. if atomic.CompareAndSwapInt32(&rt.dial, 0, 1) {
  1262. // DialMeek hasn't been called yet.
  1263. conn, err := DialMeek(
  1264. requestCtx,
  1265. rt.brokerDialParams.meekConfig,
  1266. rt.brokerDialParams.dialConfig)
  1267. if err != nil && ctx.Err() != context.Canceled {
  1268. // DialMeek performs an initial TLS handshake. DialMeek errors,
  1269. // excluding a cancelled context as happens on shutdown, are
  1270. // classified as as RoundTripperFailedErrors, which will invoke
  1271. // BrokerClientRoundTripperFailed, resetting the round tripper
  1272. // and clearing replay parameters.
  1273. err = inproxy.NewRoundTripperFailedError(err)
  1274. }
  1275. rt.conn = conn
  1276. rt.dialErr = err
  1277. close(rt.dialCompleted)
  1278. if err != nil {
  1279. return nil, errors.Trace(rt.dialErr)
  1280. }
  1281. } else {
  1282. // Await any ongoing DialMeek run by a concurrent RoundTrip caller.
  1283. select {
  1284. case <-rt.dialCompleted:
  1285. case <-ctx.Done():
  1286. return nil, errors.Trace(ctx.Err())
  1287. }
  1288. if rt.dialErr != nil {
  1289. // There is no NewRoundTripperFailedError wrapping here, as the
  1290. // DialMeek caller will wrap its error and
  1291. // BrokerClientRoundTripperFailed will be invoked already.
  1292. return nil, errors.Trace(rt.dialErr)
  1293. }
  1294. }
  1295. // At this point, rt.conn is an established MeekConn.
  1296. // Note that the network address portion of the URL will be ignored by
  1297. // MeekConn in favor of the MeekDialConfig, while the path will be used.
  1298. url := fmt.Sprintf(
  1299. "https://%s/%s",
  1300. rt.brokerDialParams.DialAddress,
  1301. inproxy.BrokerEndPointName)
  1302. request, err := http.NewRequestWithContext(
  1303. requestCtx, "POST", url, bytes.NewBuffer(requestPayload))
  1304. if err != nil {
  1305. return nil, errors.Trace(err)
  1306. }
  1307. startTime := time.Now()
  1308. response, err := rt.conn.RoundTrip(request)
  1309. roundTripDuration := time.Since(startTime)
  1310. if err == nil {
  1311. defer response.Body.Close()
  1312. if response.StatusCode != http.StatusOK {
  1313. err = fmt.Errorf(
  1314. "unexpected response status code %d after %v",
  1315. response.StatusCode,
  1316. roundTripDuration)
  1317. // Depending on the round trip duration, this case is treated as a
  1318. // temporary round tripper failure, since we received a response
  1319. // from the CDN, secured with TLS and VerifyPins, or from broker
  1320. // itself. One common scenario is the CDN returning a temporary
  1321. // timeout error, as can happen when CDN timeouts and broker
  1322. // timeouts are misaligned, especially for long-polling requests.
  1323. //
  1324. // In this scenario, we can reuse the existing round tripper and
  1325. // it may be counterproductive to return a RoundTripperFailedError
  1326. // which will trigger a clearing of any broker dial replay
  1327. // parameters as well as reseting the round tripper.
  1328. //
  1329. // When the round trip duration is sufficiently short, much
  1330. // shorter than expected round trip timeouts, this is still
  1331. // classified as a RoundTripperFailedError error, as it is more
  1332. // likely due to a more serious issue between the CDN and broker.
  1333. if rt.failureThreshold > 0 &&
  1334. roundTripDuration <= rt.failureThreshold {
  1335. err = inproxy.NewRoundTripperFailedError(err)
  1336. }
  1337. }
  1338. } else if ctx.Err() != context.Canceled {
  1339. // Other round trip errors, including TLS failures and client-side
  1340. // timeouts, but excluding a cancelled context as happens on
  1341. // shutdown, are classified as RoundTripperFailedErrors.
  1342. err = inproxy.NewRoundTripperFailedError(err)
  1343. }
  1344. if err != nil {
  1345. return nil, errors.Trace(err)
  1346. }
  1347. responsePayload, err := io.ReadAll(response.Body)
  1348. if err != nil {
  1349. err = inproxy.NewRoundTripperFailedError(err)
  1350. return nil, errors.Trace(err)
  1351. }
  1352. return responsePayload, nil
  1353. }
  1354. // InproxyWebRTCDialInstance is the network state and dial parameters for a
  1355. // single WebRTC client or proxy connection.
  1356. //
  1357. // InproxyWebRTCDialInstance implements the inproxy.WebRTCDialCoordinator
  1358. // interface, which provides the WebRTC dial configuration and support to the
  1359. // in-proxy package.
  1360. type InproxyWebRTCDialInstance struct {
  1361. config *Config
  1362. networkID string
  1363. natStateManager *InproxyNATStateManager
  1364. stunDialParameters *InproxySTUNDialParameters
  1365. webRTCDialParameters *InproxyWebRTCDialParameters
  1366. discoverNAT bool
  1367. disableSTUN bool
  1368. disablePortMapping bool
  1369. disableInboundForMobileNetworks bool
  1370. disableIPv6ICECandidates bool
  1371. discoverNATTimeout time.Duration
  1372. webRTCAnswerTimeout time.Duration
  1373. webRTCAwaitPortMappingTimeout time.Duration
  1374. awaitDataChannelTimeout time.Duration
  1375. proxyDestinationDialTimeout time.Duration
  1376. proxyRelayInactivityTimeout time.Duration
  1377. }
  1378. // NewInproxyWebRTCDialInstance creates a new InproxyWebRTCDialInstance.
  1379. //
  1380. // The caller provides STUN and WebRTC dial parameters that are either newly
  1381. // generated or replayed. Proxies may optionally pass in nil for either
  1382. // stunDialParameters or webRTCDialParameters, and new parameters will be
  1383. // generated.
  1384. func NewInproxyWebRTCDialInstance(
  1385. config *Config,
  1386. networkID string,
  1387. isProxy bool,
  1388. natStateManager *InproxyNATStateManager,
  1389. stunDialParameters *InproxySTUNDialParameters,
  1390. webRTCDialParameters *InproxyWebRTCDialParameters) (*InproxyWebRTCDialInstance, error) {
  1391. p := config.GetParameters().Get()
  1392. defer p.Close()
  1393. if isProxy && stunDialParameters == nil {
  1394. // Auto-generate STUN dial parameters. There's no replay in this case.
  1395. var err error
  1396. stunDialParameters, err = MakeInproxySTUNDialParameters(config, p, isProxy)
  1397. if err != nil {
  1398. return nil, errors.Trace(err)
  1399. }
  1400. }
  1401. if isProxy && webRTCDialParameters == nil {
  1402. // Auto-generate STUN dial parameters. There's no replay in this case.
  1403. var err error
  1404. webRTCDialParameters, err = MakeInproxyWebRTCDialParameters(p)
  1405. if err != nil {
  1406. return nil, errors.Trace(err)
  1407. }
  1408. }
  1409. disableSTUN := p.Bool(parameters.InproxyDisableSTUN)
  1410. disablePortMapping := p.Bool(parameters.InproxyDisablePortMapping)
  1411. disableInboundForMobileNetworks := p.Bool(parameters.InproxyDisableInboundForMobileNetworks)
  1412. disableIPv6ICECandidates := p.Bool(parameters.InproxyDisableIPv6ICECandidates)
  1413. var discoverNATTimeout, awaitDataChannelTimeout time.Duration
  1414. if isProxy {
  1415. disableSTUN = disableSTUN || p.Bool(parameters.InproxyProxyDisableSTUN)
  1416. disablePortMapping = disablePortMapping || p.Bool(parameters.InproxyProxyDisablePortMapping)
  1417. disableInboundForMobileNetworks = disableInboundForMobileNetworks ||
  1418. p.Bool(parameters.InproxyProxyDisableInboundForMobileNetworks)
  1419. disableIPv6ICECandidates = disableIPv6ICECandidates ||
  1420. p.Bool(parameters.InproxyProxyDisableIPv6ICECandidates)
  1421. discoverNATTimeout = p.Duration(parameters.InproxyProxyDiscoverNATTimeout)
  1422. awaitDataChannelTimeout = p.Duration(parameters.InproxyProxyWebRTCAwaitDataChannelTimeout)
  1423. } else {
  1424. disableSTUN = disableSTUN || p.Bool(parameters.InproxyClientDisableSTUN)
  1425. disablePortMapping = disablePortMapping || p.Bool(parameters.InproxyClientDisablePortMapping)
  1426. disableInboundForMobileNetworks = disableInboundForMobileNetworks ||
  1427. p.Bool(parameters.InproxyClientDisableInboundForMobileNetworks)
  1428. disableIPv6ICECandidates = disableIPv6ICECandidates ||
  1429. p.Bool(parameters.InproxyClientDisableIPv6ICECandidates)
  1430. discoverNATTimeout = p.Duration(parameters.InproxyClientDiscoverNATTimeout)
  1431. awaitDataChannelTimeout = p.Duration(parameters.InproxyClientWebRTCAwaitDataChannelTimeout)
  1432. }
  1433. // Parameters such as disabling certain operations and operation timeouts
  1434. // are not replayed, but snapshots are stored in the
  1435. // InproxyWebRTCDialInstance for efficient lookup.
  1436. return &InproxyWebRTCDialInstance{
  1437. config: config,
  1438. networkID: networkID,
  1439. natStateManager: natStateManager,
  1440. stunDialParameters: stunDialParameters,
  1441. webRTCDialParameters: webRTCDialParameters,
  1442. // discoverNAT is ignored by proxies, which always attempt discovery.
  1443. // webRTCAnswerTimeout, proxyDestinationDialTimeout, and
  1444. // proxyRelayInactivityTimeout are used only by proxies.
  1445. discoverNAT: p.WeightedCoinFlip(parameters.InproxyClientDiscoverNATProbability),
  1446. disableSTUN: disableSTUN,
  1447. disablePortMapping: disablePortMapping,
  1448. disableInboundForMobileNetworks: disableInboundForMobileNetworks,
  1449. disableIPv6ICECandidates: disableIPv6ICECandidates,
  1450. discoverNATTimeout: discoverNATTimeout,
  1451. webRTCAnswerTimeout: p.Duration(parameters.InproxyWebRTCAnswerTimeout),
  1452. webRTCAwaitPortMappingTimeout: p.Duration(parameters.InproxyWebRTCAwaitPortMappingTimeout),
  1453. awaitDataChannelTimeout: awaitDataChannelTimeout,
  1454. proxyDestinationDialTimeout: p.Duration(parameters.InproxyProxyDestinationDialTimeout),
  1455. proxyRelayInactivityTimeout: p.Duration(parameters.InproxyProxyRelayInactivityTimeout),
  1456. }, nil
  1457. }
  1458. // Implements the inproxy.WebRTCDialCoordinator interface.
  1459. func (w *InproxyWebRTCDialInstance) NetworkID() string {
  1460. return w.networkID
  1461. }
  1462. // Implements the inproxy.WebRTCDialCoordinator interface.
  1463. func (w *InproxyWebRTCDialInstance) NetworkType() inproxy.NetworkType {
  1464. return getInproxyNetworkType(GetNetworkType(w.networkID))
  1465. }
  1466. // Implements the inproxy.WebRTCDialCoordinator interface.
  1467. func (w *InproxyWebRTCDialInstance) ClientRootObfuscationSecret() inproxy.ObfuscationSecret {
  1468. return w.webRTCDialParameters.RootObfuscationSecret
  1469. }
  1470. // Implements the inproxy.WebRTCDialCoordinator interface.
  1471. func (w *InproxyWebRTCDialInstance) DoDTLSRandomization() bool {
  1472. return w.webRTCDialParameters.DoDTLSRandomization
  1473. }
  1474. // Implements the inproxy.WebRTCDialCoordinator interface.
  1475. func (w *InproxyWebRTCDialInstance) DataChannelTrafficShapingParameters() *inproxy.DataChannelTrafficShapingParameters {
  1476. return w.webRTCDialParameters.DataChannelTrafficShapingParameters
  1477. }
  1478. // Implements the inproxy.WebRTCDialCoordinator interface.
  1479. func (w *InproxyWebRTCDialInstance) STUNServerAddress(RFC5780 bool) string {
  1480. if RFC5780 {
  1481. return w.stunDialParameters.STUNServerAddressRFC5780
  1482. } else {
  1483. return w.stunDialParameters.STUNServerAddress
  1484. }
  1485. }
  1486. // Implements the inproxy.WebRTCDialCoordinator interface.
  1487. func (w *InproxyWebRTCDialInstance) STUNServerAddressResolved(RFC5780 bool) string {
  1488. if RFC5780 {
  1489. return w.stunDialParameters.STUNServerAddressRFC5780
  1490. } else {
  1491. return w.stunDialParameters.STUNServerAddress
  1492. }
  1493. }
  1494. // Implements the inproxy.WebRTCDialCoordinator interface.
  1495. func (w *InproxyWebRTCDialInstance) STUNServerAddressSucceeded(RFC5780 bool, address string) {
  1496. // Currently, for client tunnel dials, STUN dial parameter replay is
  1497. // managed by DialParameters and DialParameters.InproxySTUNDialParameters
  1498. // are replayed only when the entire dial succeeds.
  1499. //
  1500. // Note that, for a client tunnel dial, even if the STUN step fails and
  1501. // there are no STUN ICE candidates, the subsequent WebRTC connection may
  1502. // still proceed and be successful. In this case, the failed STUN dial
  1503. // parameters may be replayed.
  1504. //
  1505. // For proxies, there is no STUN dial parameter replay.
  1506. //
  1507. // As a future enhancement, consider independent and shared replay of
  1508. // working STUN servers, similar to how broker client dial parameters are
  1509. // replayed independent of overall dials and proxy relays, and shared
  1510. // between local client and proxy instances.
  1511. // Verify/extend the resolver cache entry for any resolved domain after a
  1512. // success.
  1513. resolver := w.config.GetResolver()
  1514. if resolver != nil {
  1515. resolver.VerifyCacheExtension(address)
  1516. }
  1517. }
  1518. // Implements the inproxy.WebRTCDialCoordinator interface.
  1519. func (w *InproxyWebRTCDialInstance) STUNServerAddressFailed(RFC5780 bool, address string) {
  1520. // Currently there is no independent replay for STUN dial parameters. See
  1521. // comment in STUNServerAddressSucceeded.
  1522. }
  1523. // Implements the inproxy.WebRTCDialCoordinator interface.
  1524. func (w *InproxyWebRTCDialInstance) DiscoverNAT() bool {
  1525. return w.discoverNAT
  1526. }
  1527. // Implements the inproxy.WebRTCDialCoordinator interface.
  1528. func (w *InproxyWebRTCDialInstance) DisableSTUN() bool {
  1529. return w.disableSTUN
  1530. }
  1531. // Implements the inproxy.WebRTCDialCoordinator interface.
  1532. func (w *InproxyWebRTCDialInstance) DisablePortMapping() bool {
  1533. return w.disablePortMapping
  1534. }
  1535. // Implements the inproxy.WebRTCDialCoordinator interface.
  1536. func (w *InproxyWebRTCDialInstance) DisableInboundForMobileNetworks() bool {
  1537. return w.disableInboundForMobileNetworks
  1538. }
  1539. // Implements the inproxy.WebRTCDialCoordinator interface.
  1540. func (w *InproxyWebRTCDialInstance) DisableIPv6ICECandidates() bool {
  1541. return w.disableIPv6ICECandidates
  1542. }
  1543. // Implements the inproxy.WebRTCDialCoordinator interface.
  1544. func (w *InproxyWebRTCDialInstance) NATType() inproxy.NATType {
  1545. return w.natStateManager.getNATType(w.networkID)
  1546. }
  1547. // Implements the inproxy.WebRTCDialCoordinator interface.
  1548. func (w *InproxyWebRTCDialInstance) SetNATType(natType inproxy.NATType) {
  1549. w.natStateManager.setNATType(w.networkID, natType)
  1550. }
  1551. // Implements the inproxy.WebRTCDialCoordinator interface.
  1552. func (w *InproxyWebRTCDialInstance) PortMappingTypes() inproxy.PortMappingTypes {
  1553. return w.natStateManager.getPortMappingTypes(w.networkID)
  1554. }
  1555. // Implements the inproxy.WebRTCDialCoordinator interface.
  1556. func (w *InproxyWebRTCDialInstance) SetPortMappingTypes(
  1557. portMappingTypes inproxy.PortMappingTypes) {
  1558. w.natStateManager.setPortMappingTypes(w.networkID, portMappingTypes)
  1559. }
  1560. // Implements the inproxy.WebRTCDialCoordinator interface.
  1561. func (w *InproxyWebRTCDialInstance) PortMappingProbe() *inproxy.PortMappingProbe {
  1562. return w.natStateManager.getPortMappingProbe(w.networkID)
  1563. }
  1564. // Implements the inproxy.WebRTCDialCoordinator interface.
  1565. func (w *InproxyWebRTCDialInstance) SetPortMappingProbe(
  1566. portMappingProbe *inproxy.PortMappingProbe) {
  1567. w.natStateManager.setPortMappingProbe(w.networkID, portMappingProbe)
  1568. }
  1569. // Implements the inproxy.WebRTCDialCoordinator interface.
  1570. func (w *InproxyWebRTCDialInstance) ResolveAddress(ctx context.Context, network, address string) (string, error) {
  1571. // Use the Psiphon resolver to resolve addresses.
  1572. r := w.config.GetResolver()
  1573. if r == nil {
  1574. return "", errors.TraceNew("missing resolver")
  1575. }
  1576. // Identify when the address to be resolved is one of the configured STUN
  1577. // servers, and, in those cases, use/replay any STUN dial parameters
  1578. // ResolveParameters; and record the resolved IP address for metrics.
  1579. //
  1580. // In the in-proxy proxy case, ResolveAddress is invoked for the upstream,
  1581. // 2nd hop dial as well as for STUN server addresses.
  1582. //
  1583. // Limitation: there's no ResolveParameters, including no preresolved DNS
  1584. // tactics, for 2nd hop dials.
  1585. isSTUNServerAddress := address == w.stunDialParameters.STUNServerAddress
  1586. isSTUNServerAddressRFC5780 := address == w.stunDialParameters.STUNServerAddressRFC5780
  1587. var resolveParams *resolver.ResolveParameters
  1588. if isSTUNServerAddress || isSTUNServerAddressRFC5780 {
  1589. resolveParams = w.stunDialParameters.ResolveParameters
  1590. }
  1591. resolved, err := r.ResolveAddress(
  1592. ctx, w.networkID, resolveParams, network, address)
  1593. if err != nil {
  1594. return "", errors.Trace(err)
  1595. }
  1596. // Invoke the resolved IP callbacks only when the input is not the
  1597. // resolved IP address (this differs from the meek
  1598. // DialConfig.ResolvedIPCallback case).
  1599. if resolved != address {
  1600. if isSTUNServerAddress {
  1601. w.stunDialParameters.STUNServerResolvedIPAddress.Store(resolved)
  1602. } else if isSTUNServerAddressRFC5780 {
  1603. w.stunDialParameters.STUNServerRFC5780ResolvedIPAddress.Store(resolved)
  1604. }
  1605. }
  1606. return resolved, nil
  1607. }
  1608. // Implements the inproxy.WebRTCDialCoordinator interface.
  1609. func (w *InproxyWebRTCDialInstance) UDPListen(ctx context.Context) (net.PacketConn, error) {
  1610. // Create a new inproxyUDPConn for use as the in-proxy STUN and/ord WebRTC
  1611. // UDP socket.
  1612. conn, err := newInproxyUDPConn(ctx, w.config)
  1613. if err != nil {
  1614. return nil, errors.Trace(err)
  1615. }
  1616. return conn, nil
  1617. }
  1618. // Implements the inproxy.WebRTCDialCoordinator interface.
  1619. func (w *InproxyWebRTCDialInstance) UDPConn(
  1620. ctx context.Context, network, remoteAddress string) (net.PacketConn, error) {
  1621. // Create a new UDPConn bound to the specified remote address. This UDP
  1622. // conn is used, by the inproxy package, to determine the local address
  1623. // of the active interface the OS will select for the specified remote
  1624. // destination.
  1625. //
  1626. // Only IP address destinations are supported. ResolveIP is wired up only
  1627. // because NewUDPConn requires a non-nil resolver.
  1628. dialConfig := &DialConfig{
  1629. DeviceBinder: w.config.deviceBinder,
  1630. IPv6Synthesizer: w.config.IPv6Synthesizer,
  1631. ResolveIP: func(_ context.Context, hostname string) ([]net.IP, error) {
  1632. IP := net.ParseIP(hostname)
  1633. if IP == nil {
  1634. return nil, errors.TraceNew("not supported")
  1635. }
  1636. return []net.IP{IP}, nil
  1637. },
  1638. }
  1639. conn, _, err := NewUDPConn(ctx, network, true, "", remoteAddress, dialConfig)
  1640. if err != nil {
  1641. return nil, errors.Trace(err)
  1642. }
  1643. return conn, nil
  1644. }
  1645. // Implements the inproxy.WebRTCDialCoordinator interface.
  1646. func (w *InproxyWebRTCDialInstance) BindToDevice(fileDescriptor int) error {
  1647. if w.config.deviceBinder == nil {
  1648. return nil
  1649. }
  1650. // Use config.deviceBinder, with wired up logging, not
  1651. // config.DeviceBinder; other tunnel-core dials do this indirectly via
  1652. // psiphon.DialConfig.
  1653. _, err := w.config.deviceBinder.BindToDevice(fileDescriptor)
  1654. return errors.Trace(err)
  1655. }
  1656. func (w *InproxyWebRTCDialInstance) ProxyUpstreamDial(
  1657. ctx context.Context, network, address string) (net.Conn, error) {
  1658. // This implementation of ProxyUpstreamDial applies additional socket
  1659. // options and BindToDevice as required, but is otherwise a stock dialer.
  1660. //
  1661. // TODO: Use custom UDP and TCP dialers, and wire up TCP/UDP-level
  1662. // tactics, including BPF and the custom resolver, which may be enabled
  1663. // for the proxy's ISP or geolocation. Orchestrating preresolved DNS
  1664. // requires additional information from either from the broker, the
  1665. // FrontingProviderID, to be applied to any
  1666. // DNSResolverPreresolvedIPAddressCIDRs proxy tactics. In addition,
  1667. // replay the selected upstream dial tactics parameters.
  1668. dialer := net.Dialer{
  1669. Control: func(_, _ string, c syscall.RawConn) error {
  1670. var controlErr error
  1671. err := c.Control(func(fd uintptr) {
  1672. socketFD := int(fd)
  1673. setAdditionalSocketOptions(socketFD)
  1674. if w.config.deviceBinder != nil {
  1675. _, err := w.config.deviceBinder.BindToDevice(socketFD)
  1676. if err != nil {
  1677. controlErr = errors.Tracef("BindToDevice failed: %s", err)
  1678. return
  1679. }
  1680. }
  1681. })
  1682. if controlErr != nil {
  1683. return errors.Trace(controlErr)
  1684. }
  1685. return errors.Trace(err)
  1686. },
  1687. }
  1688. conn, err := dialer.DialContext(ctx, network, address)
  1689. if err != nil {
  1690. return nil, errors.Trace(err)
  1691. }
  1692. return conn, nil
  1693. }
  1694. // Implements the inproxy.WebRTCDialCoordinator interface.
  1695. func (w *InproxyWebRTCDialInstance) DiscoverNATTimeout() time.Duration {
  1696. return w.discoverNATTimeout
  1697. }
  1698. // Implements the inproxy.WebRTCDialCoordinator interface.
  1699. func (w *InproxyWebRTCDialInstance) WebRTCAnswerTimeout() time.Duration {
  1700. return w.webRTCAnswerTimeout
  1701. }
  1702. // Implements the inproxy.WebRTCDialCoordinator interface.
  1703. func (w *InproxyWebRTCDialInstance) WebRTCAwaitPortMappingTimeout() time.Duration {
  1704. return w.webRTCAwaitPortMappingTimeout
  1705. }
  1706. // Implements the inproxy.WebRTCDialCoordinator interface.
  1707. func (w *InproxyWebRTCDialInstance) WebRTCAwaitDataChannelTimeout() time.Duration {
  1708. return w.awaitDataChannelTimeout
  1709. }
  1710. // Implements the inproxy.WebRTCDialCoordinator interface.
  1711. func (w *InproxyWebRTCDialInstance) ProxyDestinationDialTimeout() time.Duration {
  1712. return w.proxyDestinationDialTimeout
  1713. }
  1714. // Implements the inproxy.WebRTCDialCoordinator interface.
  1715. func (w *InproxyWebRTCDialInstance) ProxyRelayInactivityTimeout() time.Duration {
  1716. return w.proxyRelayInactivityTimeout
  1717. }
  1718. // InproxySTUNDialParameters is a set of STUN dial parameters.
  1719. // InproxySTUNDialParameters is compatible with DialParameters JSON
  1720. // marshaling. For client in-proxy tunnel dials, DialParameters will manage
  1721. // STUN dial parameter selection and replay.
  1722. //
  1723. // When an instance of InproxySTUNDialParameters is unmarshaled from JSON,
  1724. // Prepare must be called to initialize the instance for use.
  1725. type InproxySTUNDialParameters struct {
  1726. ResolveParameters *resolver.ResolveParameters
  1727. STUNServerAddress string
  1728. STUNServerAddressRFC5780 string
  1729. STUNServerResolvedIPAddress atomic.Value `json:"-"`
  1730. STUNServerRFC5780ResolvedIPAddress atomic.Value `json:"-"`
  1731. }
  1732. // MakeInproxySTUNDialParameters generates new STUN dial parameters from the
  1733. // given tactics parameters.
  1734. func MakeInproxySTUNDialParameters(
  1735. config *Config,
  1736. p parameters.ParametersAccessor,
  1737. isProxy bool) (*InproxySTUNDialParameters, error) {
  1738. var stunServerAddresses, stunServerAddressesRFC5780 []string
  1739. if isProxy {
  1740. stunServerAddresses = p.Strings(
  1741. parameters.InproxyProxySTUNServerAddresses, parameters.InproxySTUNServerAddresses)
  1742. stunServerAddressesRFC5780 = p.Strings(
  1743. parameters.InproxyProxySTUNServerAddressesRFC5780, parameters.InproxySTUNServerAddressesRFC5780)
  1744. } else {
  1745. stunServerAddresses = p.Strings(
  1746. parameters.InproxyClientSTUNServerAddresses, parameters.InproxySTUNServerAddresses)
  1747. stunServerAddressesRFC5780 = p.Strings(
  1748. parameters.InproxyClientSTUNServerAddressesRFC5780, parameters.InproxySTUNServerAddressesRFC5780)
  1749. }
  1750. // Empty STUN server address lists are not an error condition. When used
  1751. // for WebRTC, the STUN ICE candidate gathering will be skipped but the
  1752. // WebRTC connection may still be established via other candidate types.
  1753. var stunServerAddress, stunServerAddressRFC5780 string
  1754. if len(stunServerAddresses) > 0 {
  1755. stunServerAddress = stunServerAddresses[prng.Range(0, len(stunServerAddresses)-1)]
  1756. }
  1757. if len(stunServerAddressesRFC5780) > 0 {
  1758. stunServerAddressRFC5780 =
  1759. stunServerAddressesRFC5780[prng.Range(0, len(stunServerAddressesRFC5780)-1)]
  1760. }
  1761. // Create DNS resolver dial parameters to use when resolving STUN server
  1762. // domain addresses. Instantiate only when there is a domain to be
  1763. // resolved; when recording DNS fields, GetMetrics will assume that a nil
  1764. // InproxySTUNDialParameters.ResolveParameters implies no resolve was
  1765. // attempted.
  1766. var resolveParameters *resolver.ResolveParameters
  1767. if (stunServerAddress != "" && net.ParseIP(stunServerAddress) == nil) ||
  1768. (stunServerAddressRFC5780 != "" && net.ParseIP(stunServerAddressRFC5780) == nil) {
  1769. // No DNSResolverPreresolvedIPAddressCIDRs will be selected since no
  1770. // fronting provider ID is specified.
  1771. //
  1772. // It would be possible to overload the meaning of the fronting
  1773. // provider ID field by using a string derived from STUN server
  1774. // address as the key.
  1775. //
  1776. // However, preresolved STUN configuration can already be achieved
  1777. // with IP addresses in the STUNServerAddresses tactics parameters.
  1778. // This approach results in slightly different metrics log fields vs.
  1779. // preresolved.
  1780. var err error
  1781. resolveParameters, err = config.GetResolver().MakeResolveParameters(p, "", "")
  1782. if err != nil {
  1783. return nil, errors.Trace(err)
  1784. }
  1785. }
  1786. dialParams := &InproxySTUNDialParameters{
  1787. ResolveParameters: resolveParameters,
  1788. STUNServerAddress: stunServerAddress,
  1789. STUNServerAddressRFC5780: stunServerAddressRFC5780,
  1790. }
  1791. dialParams.Prepare()
  1792. return dialParams, nil
  1793. }
  1794. // Prepare initializes an InproxySTUNDialParameters for use. Prepare should be
  1795. // called for any InproxySTUNDialParameters instance unmarshaled from JSON.
  1796. func (dialParams *InproxySTUNDialParameters) Prepare() {
  1797. dialParams.STUNServerResolvedIPAddress.Store("")
  1798. dialParams.STUNServerRFC5780ResolvedIPAddress.Store("")
  1799. }
  1800. // IsValidClientReplay checks that the selected STUN servers remain configured
  1801. // STUN server candidates for in-proxy clients.
  1802. func (dialParams *InproxySTUNDialParameters) IsValidClientReplay(
  1803. p parameters.ParametersAccessor) bool {
  1804. return (dialParams.STUNServerAddress == "" ||
  1805. common.Contains(
  1806. p.Strings(parameters.InproxyClientSTUNServerAddresses),
  1807. dialParams.STUNServerAddress)) &&
  1808. (dialParams.STUNServerAddressRFC5780 == "" ||
  1809. common.Contains(
  1810. p.Strings(parameters.InproxyClientSTUNServerAddressesRFC5780),
  1811. dialParams.STUNServerAddressRFC5780))
  1812. }
  1813. // GetMetrics implements the common.MetricsSource interface and returns log
  1814. // fields detailing the STUN dial parameters.
  1815. func (dialParams *InproxySTUNDialParameters) GetMetrics() common.LogFields {
  1816. // There is no is_replay-type field added here; replay is handled at a
  1817. // higher level, and, for client in-proxy tunnel dials, is part of the
  1818. // main tunnel dial parameters.
  1819. logFields := make(common.LogFields)
  1820. logFields["inproxy_webrtc_stun_server"] = dialParams.STUNServerAddress
  1821. resolvedIPAddress := dialParams.STUNServerResolvedIPAddress.Load().(string)
  1822. if resolvedIPAddress != "" {
  1823. logFields["inproxy_webrtc_stun_server_resolved_ip_address"] = resolvedIPAddress
  1824. }
  1825. // TODO: log RFC5780 selection only if used?
  1826. logFields["inproxy_webrtc_stun_server_RFC5780"] = dialParams.STUNServerAddressRFC5780
  1827. resolvedIPAddress = dialParams.STUNServerRFC5780ResolvedIPAddress.Load().(string)
  1828. if resolvedIPAddress != "" {
  1829. logFields["inproxy_webrtc_stun_server_RFC5780_resolved_ip_address"] = resolvedIPAddress
  1830. }
  1831. if dialParams.ResolveParameters != nil {
  1832. // See comment in getBaseAPIParameters regarding
  1833. // dialParams.ResolveParameters handling. As noted in
  1834. // MakeInproxySTUNDialParameters, no preresolved parameters are set,
  1835. // so none are checked for logging.
  1836. //
  1837. // Limitation: the potential use of single ResolveParameters to
  1838. // resolve multiple, different STUN server domains can skew the
  1839. // meaning of GetFirstAttemptWithAnswer.
  1840. if dialParams.ResolveParameters.PreferAlternateDNSServer {
  1841. logFields["inproxy_webrtc_dns_preferred"] = dialParams.ResolveParameters.AlternateDNSServer
  1842. }
  1843. if dialParams.ResolveParameters.ProtocolTransformName != "" {
  1844. logFields["inproxy_webrtc_dns_transform"] = dialParams.ResolveParameters.ProtocolTransformName
  1845. }
  1846. logFields["inproxy_webrtc_dns_attempt"] = strconv.Itoa(
  1847. dialParams.ResolveParameters.GetFirstAttemptWithAnswer())
  1848. }
  1849. return logFields
  1850. }
  1851. // InproxyWebRTCDialParameters is a set of WebRTC obfuscation dial parameters.
  1852. // InproxyWebRTCDialParameters is compatible with DialParameters JSON
  1853. // marshaling. For client in-proxy tunnel dials, DialParameters will manage
  1854. // WebRTC dial parameter selection and replay.
  1855. type InproxyWebRTCDialParameters struct {
  1856. RootObfuscationSecret inproxy.ObfuscationSecret
  1857. DataChannelTrafficShapingParameters *inproxy.DataChannelTrafficShapingParameters
  1858. DoDTLSRandomization bool
  1859. }
  1860. // MakeInproxyWebRTCDialParameters generates new InproxyWebRTCDialParameters.
  1861. func MakeInproxyWebRTCDialParameters(
  1862. p parameters.ParametersAccessor) (*InproxyWebRTCDialParameters, error) {
  1863. rootObfuscationSecret, err := inproxy.GenerateRootObfuscationSecret()
  1864. if err != nil {
  1865. return nil, errors.Trace(err)
  1866. }
  1867. var trafficSharingParams inproxy.DataChannelTrafficShapingParameters
  1868. if p.WeightedCoinFlip(parameters.InproxyDataChannelTrafficShapingProbability) {
  1869. trafficSharingParams = inproxy.DataChannelTrafficShapingParameters(
  1870. p.InproxyDataChannelTrafficShapingParameters(
  1871. parameters.InproxyDataChannelTrafficShapingParameters))
  1872. }
  1873. doDTLSRandomization := p.WeightedCoinFlip(parameters.InproxyDTLSRandomizationProbability)
  1874. return &InproxyWebRTCDialParameters{
  1875. RootObfuscationSecret: rootObfuscationSecret,
  1876. DataChannelTrafficShapingParameters: &trafficSharingParams,
  1877. DoDTLSRandomization: doDTLSRandomization,
  1878. }, nil
  1879. }
  1880. // GetMetrics implements the common.MetricsSource interface.
  1881. func (dialParams *InproxyWebRTCDialParameters) GetMetrics() common.LogFields {
  1882. // There is no is_replay-type field added here; replay is handled at a
  1883. // higher level, and, for client in-proxy tunnel dials, is part of the
  1884. // main tunnel dial parameters.
  1885. // Currently, all WebRTC metrics are delivered via
  1886. // inproxy.ClientConn/WebRTCConn GetMetrics.
  1887. return common.LogFields{}
  1888. }
  1889. // InproxyNATStateManager manages the NAT-related network topology state for
  1890. // the current network, caching the discovered network NAT type and supported
  1891. // port mapping types, if any.
  1892. type InproxyNATStateManager struct {
  1893. config *Config
  1894. mutex sync.Mutex
  1895. networkID string
  1896. natType inproxy.NATType
  1897. portMappingTypes inproxy.PortMappingTypes
  1898. portMappingProbe *inproxy.PortMappingProbe
  1899. }
  1900. // NewInproxyNATStateManager creates a new InproxyNATStateManager.
  1901. func NewInproxyNATStateManager(config *Config) *InproxyNATStateManager {
  1902. s := &InproxyNATStateManager{
  1903. config: config,
  1904. natType: inproxy.NATTypeUnknown,
  1905. portMappingTypes: inproxy.PortMappingTypes{},
  1906. }
  1907. s.reset()
  1908. return s
  1909. }
  1910. // TacticsApplied implements the TacticsAppliedReceiver interface, and is
  1911. // called when tactics have changed, which triggers a cached NAT state reset
  1912. // in order to apply potentially changed parameters.
  1913. func (s *InproxyNATStateManager) TacticsApplied() error {
  1914. s.reset()
  1915. return nil
  1916. }
  1917. func (s *InproxyNATStateManager) reset() {
  1918. s.mutex.Lock()
  1919. defer s.mutex.Unlock()
  1920. networkID := s.config.GetNetworkID()
  1921. s.networkID = networkID
  1922. s.natType = inproxy.NATTypeUnknown
  1923. s.portMappingTypes = inproxy.PortMappingTypes{}
  1924. }
  1925. func (s *InproxyNATStateManager) getNATType(
  1926. networkID string) inproxy.NATType {
  1927. s.mutex.Lock()
  1928. defer s.mutex.Unlock()
  1929. if s.networkID != networkID {
  1930. return inproxy.NATTypeUnknown
  1931. }
  1932. return s.natType
  1933. }
  1934. func (s *InproxyNATStateManager) setNATType(
  1935. networkID string, natType inproxy.NATType) {
  1936. s.mutex.Lock()
  1937. defer s.mutex.Unlock()
  1938. if s.networkID != networkID {
  1939. return
  1940. }
  1941. s.natType = natType
  1942. }
  1943. func (s *InproxyNATStateManager) getPortMappingTypes(
  1944. networkID string) inproxy.PortMappingTypes {
  1945. s.mutex.Lock()
  1946. defer s.mutex.Unlock()
  1947. if s.networkID != networkID {
  1948. return inproxy.PortMappingTypes{}
  1949. }
  1950. return s.portMappingTypes
  1951. }
  1952. func (s *InproxyNATStateManager) setPortMappingTypes(
  1953. networkID string,
  1954. portMappingTypes inproxy.PortMappingTypes) {
  1955. s.mutex.Lock()
  1956. defer s.mutex.Unlock()
  1957. if s.networkID != networkID {
  1958. return
  1959. }
  1960. s.portMappingTypes = portMappingTypes
  1961. }
  1962. func (s *InproxyNATStateManager) getPortMappingProbe(
  1963. networkID string) *inproxy.PortMappingProbe {
  1964. s.mutex.Lock()
  1965. defer s.mutex.Unlock()
  1966. if s.networkID != networkID {
  1967. return nil
  1968. }
  1969. return s.portMappingProbe
  1970. }
  1971. func (s *InproxyNATStateManager) setPortMappingProbe(
  1972. networkID string,
  1973. portMappingProbe *inproxy.PortMappingProbe) {
  1974. s.mutex.Lock()
  1975. defer s.mutex.Unlock()
  1976. if s.networkID != networkID {
  1977. return
  1978. }
  1979. s.portMappingProbe = portMappingProbe
  1980. }
  1981. // inproxyUDPConn is based on NewUDPConn and includes the write timeout
  1982. // workaround from common.WriteTimeoutUDPConn.
  1983. //
  1984. // inproxyUDPConn expands the NewUDPConn IPv6Synthesizer to support many
  1985. // destination addresses, as the inproxyUDPConn will be used to send/receive
  1986. // packets between many remote destination addresses.
  1987. //
  1988. // inproxyUDPConn implements the net.PacketConn interface.
  1989. type inproxyUDPConn struct {
  1990. udpConn *net.UDPConn
  1991. ipv6Synthesizer IPv6Synthesizer
  1992. synthesizerMutex sync.Mutex
  1993. ipv4ToIPv6 map[netip.Addr]net.IP
  1994. ipv6ToIPv4 map[netip.Addr]net.IP
  1995. }
  1996. func newInproxyUDPConn(ctx context.Context, config *Config) (net.PacketConn, error) {
  1997. listen := &net.ListenConfig{
  1998. Control: func(_, _ string, c syscall.RawConn) error {
  1999. var controlErr error
  2000. err := c.Control(func(fd uintptr) {
  2001. socketFD := int(fd)
  2002. setAdditionalSocketOptions(socketFD)
  2003. // Use config.deviceBinder, with wired up logging, not
  2004. // config.DeviceBinder; other tunnel-core dials do this
  2005. // indirectly via psiphon.DialConfig.
  2006. if config.deviceBinder != nil {
  2007. _, err := config.deviceBinder.BindToDevice(socketFD)
  2008. if err != nil {
  2009. controlErr = errors.Tracef("BindToDevice failed: %s", err)
  2010. return
  2011. }
  2012. }
  2013. })
  2014. if controlErr != nil {
  2015. return errors.Trace(controlErr)
  2016. }
  2017. return errors.Trace(err)
  2018. },
  2019. }
  2020. // Create an "unconnected" UDP socket for use with WriteTo and listening
  2021. // on all interfaces. See the limitation comment in NewUDPConn regarding
  2022. // its equivilent mode.
  2023. packetConn, err := listen.ListenPacket(ctx, "udp", "")
  2024. if err != nil {
  2025. return nil, errors.Trace(err)
  2026. }
  2027. var ok bool
  2028. udpConn, ok := packetConn.(*net.UDPConn)
  2029. if !ok {
  2030. return nil, errors.Tracef("unexpected conn type: %T", packetConn)
  2031. }
  2032. conn := &inproxyUDPConn{
  2033. udpConn: udpConn,
  2034. ipv6Synthesizer: config.IPv6Synthesizer,
  2035. }
  2036. if conn.ipv6Synthesizer != nil {
  2037. conn.ipv4ToIPv6 = make(map[netip.Addr]net.IP)
  2038. conn.ipv6ToIPv4 = make(map[netip.Addr]net.IP)
  2039. }
  2040. return conn, nil
  2041. }
  2042. func inproxyUDPAddrFromAddrPort(addrPort netip.AddrPort) *net.UDPAddr {
  2043. return &net.UDPAddr{
  2044. IP: addrPort.Addr().AsSlice(),
  2045. Port: int(addrPort.Port()),
  2046. }
  2047. }
  2048. func (conn *inproxyUDPConn) ReadFrom(p []byte) (int, net.Addr, error) {
  2049. // net.UDPConn.ReadFrom currently allocates a &UDPAddr{} per call, and so
  2050. // the &net.UDPAddr{} allocations done in the following synthesizer code
  2051. // path are no more than the standard code path.
  2052. //
  2053. // TODO: avoid all address allocations in both ReadFrom and WriteTo by:
  2054. //
  2055. // - changing ipvXToIPvY to map[netip.AddrPort]*net.UDPAddr
  2056. // - using a similar lookup for the non-synthesizer code path
  2057. //
  2058. // Such a scheme would work only if the caller is guaranteed to not mutate
  2059. // the returned net.Addr.
  2060. if conn.ipv6Synthesizer == nil {
  2061. // Do not wrap any I/O err returned by UDPConn
  2062. return conn.udpConn.ReadFrom(p)
  2063. }
  2064. n, addrPort, err := conn.udpConn.ReadFromUDPAddrPort(p)
  2065. // Reverse any synthesized address before returning err.
  2066. // Reverse the IPv6 synthesizer, returning the original IPv4 address
  2067. // as expected by the caller, including pion/webrtc. This logic
  2068. // assumes that no synthesized IPv6 address will conflict with any
  2069. // real IPv6 address.
  2070. var IP net.IP
  2071. ipAddr := addrPort.Addr()
  2072. if ipAddr.Is6() {
  2073. conn.synthesizerMutex.Lock()
  2074. IP, _ = conn.ipv6ToIPv4[ipAddr]
  2075. conn.synthesizerMutex.Unlock()
  2076. }
  2077. if IP == nil {
  2078. IP = ipAddr.AsSlice()
  2079. }
  2080. // Do not wrap any I/O err returned by UDPConn
  2081. return n, &net.UDPAddr{IP: IP, Port: int(addrPort.Port())}, err
  2082. }
  2083. func (conn *inproxyUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
  2084. // See common.WriteTimeoutUDPConn.
  2085. err := conn.udpConn.SetWriteDeadline(
  2086. time.Now().Add(common.UDP_PACKET_WRITE_TIMEOUT))
  2087. if err != nil {
  2088. return 0, errors.Trace(err)
  2089. }
  2090. if conn.ipv6Synthesizer == nil {
  2091. // Do not wrap any I/O err returned by UDPConn
  2092. return conn.udpConn.WriteTo(b, addr)
  2093. }
  2094. // When configured, attempt to synthesize IPv6 addresses from an IPv4
  2095. // addresses for compatibility on DNS64/NAT64 networks.
  2096. //
  2097. // Store any synthesized addresses in a lookup table and reuse for
  2098. // subsequent writes to the same destination as well as reversing the
  2099. // conversion on reads.
  2100. //
  2101. // If synthesize fails, fall back to trying the original address.
  2102. // The netip.Addr type is used as the map key and the input address is
  2103. // assumed to be of the type *net.UDPAddr. This allows for more efficient
  2104. // lookup operations vs. a string key and parsing the input address via
  2105. // addr.String()/net.SplitHostPort().
  2106. udpAddr, ok := addr.(*net.UDPAddr)
  2107. if !ok {
  2108. return 0, errors.Tracef("unexpected addr type: %T", addr)
  2109. }
  2110. // Stack allocate to avoid an extra heap allocation per write.
  2111. var synthesizedAddr net.UDPAddr
  2112. if udpAddr.IP.To4() != nil {
  2113. ip4Addr, ok := netip.AddrFromSlice(udpAddr.IP)
  2114. if !ok {
  2115. return 0, errors.Tracef("invalid addr")
  2116. }
  2117. conn.synthesizerMutex.Lock()
  2118. synthesizedIP, ok := conn.ipv4ToIPv6[ip4Addr]
  2119. conn.synthesizerMutex.Unlock()
  2120. if ok {
  2121. synthesizedAddr = net.UDPAddr{IP: synthesizedIP, Port: udpAddr.Port}
  2122. } else {
  2123. synthesized := conn.ipv6Synthesizer.IPv6Synthesize(udpAddr.IP.String())
  2124. if synthesized != "" {
  2125. synthesizedIP := net.ParseIP(synthesized)
  2126. if synthesizedIP != nil {
  2127. conn.synthesizerMutex.Lock()
  2128. conn.ipv4ToIPv6[ip4Addr] = synthesizedIP
  2129. ipv6Addr, _ := netip.AddrFromSlice(synthesizedIP)
  2130. conn.ipv6ToIPv4[ipv6Addr] = udpAddr.IP
  2131. conn.synthesizerMutex.Unlock()
  2132. synthesizedAddr = net.UDPAddr{IP: synthesizedIP, Port: udpAddr.Port}
  2133. }
  2134. }
  2135. }
  2136. }
  2137. if synthesizedAddr.IP == nil {
  2138. // Do not wrap any I/O err returned by UDPConn
  2139. return conn.udpConn.WriteTo(b, addr)
  2140. }
  2141. return conn.udpConn.WriteTo(b, &synthesizedAddr)
  2142. }
  2143. func (conn *inproxyUDPConn) Close() error {
  2144. // Do not wrap any I/O err returned by UDPConn
  2145. return conn.udpConn.Close()
  2146. }
  2147. func (conn *inproxyUDPConn) LocalAddr() net.Addr {
  2148. // Do not wrap any I/O err returned by UDPConn
  2149. return conn.udpConn.LocalAddr()
  2150. }
  2151. func (conn *inproxyUDPConn) SetDeadline(t time.Time) error {
  2152. // Do not wrap any I/O err returned by UDPConn
  2153. return conn.udpConn.SetDeadline(t)
  2154. }
  2155. func (conn *inproxyUDPConn) SetReadDeadline(t time.Time) error {
  2156. // Do not wrap any I/O err returned by UDPConn
  2157. return conn.udpConn.SetReadDeadline(t)
  2158. }
  2159. func (conn *inproxyUDPConn) SetWriteDeadline(t time.Time) error {
  2160. // Do not wrap any I/O err returned by UDPConn
  2161. return conn.udpConn.SetWriteDeadline(t)
  2162. }
  2163. // getInproxyNetworkType converts a legacy string network type to an inproxy
  2164. // package type.
  2165. func getInproxyNetworkType(networkType string) inproxy.NetworkType {
  2166. // There is no VPN type conversion; clients and proxies will skip/fail
  2167. // in-proxy operations on non-Psiphon VPN networks.
  2168. switch networkType {
  2169. case "WIFI":
  2170. return inproxy.NetworkTypeWiFi
  2171. case "MOBILE":
  2172. return inproxy.NetworkTypeMobile
  2173. }
  2174. return inproxy.NetworkTypeUnknown
  2175. }