inproxy.go 92 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667
  1. /*
  2. * Copyright (c) 2023, Psiphon Inc.
  3. * All rights reserved.
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation, either version 3 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. *
  18. */
  19. package psiphon
  20. import (
  21. "bytes"
  22. "context"
  23. "encoding/binary"
  24. std_errors "errors"
  25. "fmt"
  26. "io"
  27. "net"
  28. "net/http"
  29. "net/netip"
  30. "strconv"
  31. "sync"
  32. "sync/atomic"
  33. "syscall"
  34. "time"
  35. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
  36. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
  37. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/inproxy"
  38. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters"
  39. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
  40. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/resolver"
  41. utls "github.com/Psiphon-Labs/utls"
  42. "github.com/cespare/xxhash"
  43. )
  44. // InproxyBrokerClientManager manages an InproxyBrokerClientInstance, an
  45. // in-proxy broker client, and its associated broker dial parameters, that
  46. // may be shared by multiple client dials or proxy instances. There is no
  47. // explicit close operation for the managed InproxyBrokerClientInstance.
  48. //
  49. // Once used, the current InproxyBrokerClientInstance and its broker client is
  50. // left actively connected to the broker, to minimize transport round trips
  51. // for additional requests.
  52. //
  53. // The InproxyBrokerClientManager and its components implement a replay system
  54. // for broker client dials. As one broker client is shared access multiple
  55. // client in-proxy dials, the broker dial parameters are replayed
  56. // independently from tunnel dial parameters.
  57. //
  58. // The NewInproxyBrokerClientInstance layer provides a fixed association
  59. // between a broker client and its broker dial parameters, ensuring that
  60. // in-proxy success/failure callbacks reference the correct replay parameters
  61. // when setting or clearing replay.
  62. //
  63. // A new InproxyBrokerClientInstance, including the broker dial parameters and
  64. // broker client, is instantiated when the active network ID changes, using
  65. // tactics for the new network.
  66. type InproxyBrokerClientManager struct {
  67. config *Config
  68. isProxy bool
  69. tlsCache utls.ClientSessionCache
  70. mutex sync.Mutex
  71. brokerSelectCount int
  72. networkID string
  73. brokerClientInstance *InproxyBrokerClientInstance
  74. }
  75. // NewInproxyBrokerClientManager creates a new InproxyBrokerClientManager.
  76. // NewInproxyBrokerClientManager does not perform any network operations; the
  77. // managed InproxyBrokerClientInstance is initialized when used for a round
  78. // trip.
  79. func NewInproxyBrokerClientManager(
  80. config *Config, isProxy bool, tlsCache utls.ClientSessionCache) *InproxyBrokerClientManager {
  81. b := &InproxyBrokerClientManager{
  82. config: config,
  83. isProxy: isProxy,
  84. tlsCache: tlsCache,
  85. }
  86. // b.brokerClientInstance is initialized on demand, when getBrokerClient
  87. // is called.
  88. return b
  89. }
  90. // TacticsApplied implements the TacticsAppliedReceiver interface, and is
  91. // called when tactics have changed, which triggers a broker client reset in
  92. // order to apply potentially changed parameters.
  93. func (b *InproxyBrokerClientManager) TacticsApplied() error {
  94. b.mutex.Lock()
  95. defer b.mutex.Unlock()
  96. // Don't reset when not yet initialized; b.brokerClientInstance is
  97. // initialized only on demand.
  98. if b.brokerClientInstance == nil {
  99. return nil
  100. }
  101. // TODO: as a future future enhancement, don't reset when the tactics
  102. // brokerSpecs.Hash() is unchanged?
  103. return errors.Trace(b.reset(resetBrokerClientReasonTacticsApplied))
  104. }
  105. // NetworkChanged is called when the active network changes, to trigger a
  106. // broker client reset.
  107. func (b *InproxyBrokerClientManager) NetworkChanged() error {
  108. b.mutex.Lock()
  109. defer b.mutex.Unlock()
  110. // Don't reset when not yet initialized; b.brokerClientInstance is
  111. // initialized only on demand.
  112. if b.brokerClientInstance == nil {
  113. return nil
  114. }
  115. return errors.Trace(b.reset(resetBrokerClientReasonNetworkChanged))
  116. }
  117. // GetBrokerClient returns the current, shared broker client and its
  118. // corresponding dial parameters (for metrics logging). If there is no
  119. // current broker client, if the network ID differs from the network ID
  120. // associated with the previous broker client, a new broker client is
  121. // initialized.
  122. func (b *InproxyBrokerClientManager) GetBrokerClient(
  123. networkID string) (*inproxy.BrokerClient, *InproxyBrokerDialParameters, error) {
  124. b.mutex.Lock()
  125. if b.brokerClientInstance == nil || b.networkID != networkID {
  126. err := b.reset(resetBrokerClientReasonInit)
  127. if err != nil {
  128. b.mutex.Unlock()
  129. return nil, nil, errors.Trace(err)
  130. }
  131. }
  132. brokerClientInstance := b.brokerClientInstance
  133. // Release lock before calling brokerClientInstance.HasSuccess. Otherwise,
  134. // there's a potential deadlock that would result from this code path
  135. // locking InproxyBrokerClientManager.mutex then InproxyBrokerClientInstance.mutex,
  136. // while the BrokerClientRoundTripperFailed code path locks in the reverse order.
  137. b.mutex.Unlock()
  138. // Set isReuse, which will record a metric indicating if this broker
  139. // client has already been used for a successful round trip, a case which
  140. // should result in faster overall dials.
  141. //
  142. // Limitations with HasSuccess, and the resulting isReuse metric: in some
  143. // cases, it's possible that the underlying TLS connection is still
  144. // redialed by net/http; or it's possible that the Noise session is
  145. // invalid/expired and must be reestablished; or it can be the case that
  146. // a shared broker client is only partially established at this point in
  147. // time.
  148. //
  149. // Return a shallow copy of the broker dial params in order to record the
  150. // correct isReuse, which varies depending on previous use.
  151. brokerDialParams := *brokerClientInstance.brokerDialParams
  152. brokerDialParams.isReuse = brokerClientInstance.HasSuccess()
  153. // The b.brokerClientInstance.brokerClient is wired up to refer back to
  154. // b.brokerClientInstance.brokerDialParams/roundTripper, etc.
  155. return brokerClientInstance.brokerClient,
  156. &brokerDialParams,
  157. nil
  158. }
  159. func (b *InproxyBrokerClientManager) resetBrokerClientOnRoundTripperFailed(
  160. brokerClientInstance *InproxyBrokerClientInstance) error {
  161. b.mutex.Lock()
  162. defer b.mutex.Unlock()
  163. if b.brokerClientInstance != brokerClientInstance {
  164. // Ignore the reset if the signal comes from the non-current
  165. // brokerClientInstance, which may occur when multiple in-flight
  166. // round trips fail in close proximity.
  167. return nil
  168. }
  169. return errors.Trace(b.reset(resetBrokerClientReasonRoundTripperFailed))
  170. }
  171. func (b *InproxyBrokerClientManager) resetBrokerClientOnNoMatch(
  172. brokerClientInstance *InproxyBrokerClientInstance) error {
  173. // Ignore the no match callback for proxies. For personal pairing, the
  174. // broker rotation scheme has clients moving brokers to find relatively
  175. // static proxies. For common pairing, we want to achieve balanced supply
  176. // across brokers.
  177. //
  178. // Currently, inproxy.BrokerDialCoordinator.BrokerClientNoMatch is only
  179. // wired up for clients, but this check ensures it'll still be ignored in
  180. // case that changes.
  181. if b.isProxy {
  182. return nil
  183. }
  184. if b.brokerClientInstance != brokerClientInstance {
  185. // See comment for same logic in resetBrokerClientOnRoundTripperFailed.
  186. return nil
  187. }
  188. p := b.config.GetParameters().Get()
  189. defer p.Close()
  190. probability := parameters.InproxyClientNoMatchFailoverProbability
  191. if b.config.IsInproxyClientPersonalPairingMode() {
  192. probability = parameters.InproxyClientNoMatchFailoverPersonalProbability
  193. }
  194. if !p.WeightedCoinFlip(probability) {
  195. return nil
  196. }
  197. return errors.Trace(b.reset(resetBrokerClientReasonRoundNoMatch))
  198. }
  199. type resetBrokerClientReason int
  200. const (
  201. resetBrokerClientReasonInit resetBrokerClientReason = iota + 1
  202. resetBrokerClientReasonTacticsApplied
  203. resetBrokerClientReasonNetworkChanged
  204. resetBrokerClientReasonRoundTripperFailed
  205. resetBrokerClientReasonRoundNoMatch
  206. )
  207. func (b *InproxyBrokerClientManager) reset(reason resetBrokerClientReason) error {
  208. // Assumes b.mutex lock is held.
  209. if b.brokerClientInstance != nil {
  210. // Close the existing broker client. This will close all underlying
  211. // network connections, interrupting any in-flight requests. This
  212. // close is invoked in the resetBrokerClientOnRoundTripperFailed
  213. // case, where it's expected that the round tripped has permanently
  214. // failed.
  215. b.brokerClientInstance.Close()
  216. }
  217. // b.brokerSelectCount tracks the number of broker resets and is used to
  218. // iterate over the brokers in a deterministic rotation when running in
  219. // personal pairing mode.
  220. switch reason {
  221. case resetBrokerClientReasonInit,
  222. resetBrokerClientReasonTacticsApplied,
  223. resetBrokerClientReasonNetworkChanged:
  224. b.brokerSelectCount = 0
  225. case resetBrokerClientReasonRoundTripperFailed,
  226. resetBrokerClientReasonRoundNoMatch:
  227. b.brokerSelectCount += 1
  228. }
  229. // Any existing broker client is removed, even if
  230. // NewInproxyBrokerClientInstance fails. This ensures, for example, that
  231. // an existing broker client is removed when its spec is no longer
  232. // available in tactics.
  233. b.networkID = ""
  234. b.brokerClientInstance = nil
  235. networkID := b.config.GetNetworkID()
  236. brokerClientInstance, err := NewInproxyBrokerClientInstance(
  237. b.config,
  238. b,
  239. networkID,
  240. b.isProxy,
  241. b.brokerSelectCount,
  242. reason == resetBrokerClientReasonRoundNoMatch)
  243. if err != nil {
  244. return errors.Trace(err)
  245. }
  246. b.networkID = networkID
  247. b.brokerClientInstance = brokerClientInstance
  248. return nil
  249. }
  250. // InproxyBrokerClientInstance pairs an inproxy.BrokerClient instance with an
  251. // implementation of the inproxy.BrokerDialCoordinator interface and the
  252. // associated, underlying broker dial parameters. InproxyBrokerClientInstance
  253. // implements broker client dial replay.
  254. type InproxyBrokerClientInstance struct {
  255. config *Config
  256. brokerClientManager *InproxyBrokerClientManager
  257. networkID string
  258. brokerClientPrivateKey inproxy.SessionPrivateKey
  259. brokerClient *inproxy.BrokerClient
  260. brokerPublicKey inproxy.SessionPublicKey
  261. brokerRootObfuscationSecret inproxy.ObfuscationSecret
  262. brokerDialParams *InproxyBrokerDialParameters
  263. replayEnabled bool
  264. roundTripper *InproxyBrokerRoundTripper
  265. personalCompartmentIDs []inproxy.ID
  266. commonCompartmentIDs []inproxy.ID
  267. disableWaitToShareSession bool
  268. sessionHandshakeTimeout time.Duration
  269. announceRequestTimeout time.Duration
  270. announceDelay time.Duration
  271. announceMaxBackoffDelay time.Duration
  272. announceDelayJitter float64
  273. answerRequestTimeout time.Duration
  274. offerRequestTimeout time.Duration
  275. offerRequestPersonalTimeout time.Duration
  276. offerRetryDelay time.Duration
  277. offerRetryJitter float64
  278. relayedPacketRequestTimeout time.Duration
  279. dslRequestTimeout time.Duration
  280. replayRetainFailedProbability float64
  281. replayUpdateFrequency time.Duration
  282. retryOnFailedPeriod time.Duration
  283. mutex sync.Mutex
  284. lastStoreReplay time.Time
  285. lastSuccess time.Time
  286. }
  287. // NewInproxyBrokerClientInstance creates a new InproxyBrokerClientInstance.
  288. // NewInproxyBrokerClientManager does not perform any network operations; the
  289. // new InproxyBrokerClientInstance is initialized when used for a round
  290. // trip.
  291. func NewInproxyBrokerClientInstance(
  292. config *Config,
  293. brokerClientManager *InproxyBrokerClientManager,
  294. networkID string,
  295. isProxy bool,
  296. brokerSelectCount int,
  297. resetReasonNoMatch bool) (*InproxyBrokerClientInstance, error) {
  298. p := config.GetParameters().Get()
  299. defer p.Close()
  300. // Select common or personal compartment IDs. Clients must provide at
  301. // least on compartment ID.
  302. //
  303. // A here check for !isProxy && len(commonCompartmentIDs) == 0 && len
  304. // (personalCompartmentIDs) == 0 is now deferred until
  305. // inproxy.DialClient, to allow broker connections for DSL requests
  306. // without in-proxy compartment IDs.
  307. commonCompartmentIDs, personalCompartmentIDs, err :=
  308. prepareInproxyCompartmentIDs(config, p, isProxy)
  309. if err != nil {
  310. return nil, errors.Trace(err)
  311. }
  312. if len(personalCompartmentIDs) > 1 {
  313. return nil, errors.TraceNew("unexpected multiple personal compartment IDs")
  314. }
  315. // Select the broker to use, optionally favoring brokers with replay data.
  316. // In the InproxyBrokerSpecs calls, the first non-empty tactics parameter
  317. // list is used.
  318. //
  319. // Optional broker specs may be used to specify broker(s) dedicated to
  320. // personal pairing, a configuration which can be used to reserve more
  321. // capacity for personal pairing, given the simple rendezvous scheme below.
  322. brokerSpecs := getInproxyBrokerSpecs(config, p, isProxy)
  323. if len(brokerSpecs) == 0 {
  324. return nil, errors.TraceNew("no broker specs")
  325. }
  326. // Select a broker.
  327. // In common pairing mode, the available brokers are shuffled before
  328. // selection, for random load balancing. Brokers with available dial
  329. // parameter replay data are preferred. When rotating brokers due to a no
  330. // match, the available replay data is ignored to increase the chance of
  331. // selecting a different broker.
  332. //
  333. // In personal pairing mode, arrange for the proxy and client to
  334. // rendezvous at the same broker by shuffling based on the shared
  335. // personal compartment ID. Both the client and proxy will select the
  336. // same initial broker, and fail over to other brokers in the same order.
  337. // By design, clients will move between brokers aggressively, rotating on
  338. // no-match responses and applying a shorter client offer timeout; while
  339. // proxies will remain in place in order to be found. Since rendezvous
  340. // depends on the ordering, each broker is selected in shuffle order;
  341. // dial parameter replay data is used when available but not considered
  342. // in selection ordering. The brokerSelectCount input is used to
  343. // progressively index into the list of shuffled brokers.
  344. //
  345. // Potential future enhancements:
  346. //
  347. // - Use brokerSelectCount in the common pairing case as well, to ensure
  348. // that a no-match reset always selects a different broker; but, unlike
  349. // the personal pairing logic, still prefer brokers with replay rather
  350. // than following a strict shuffle order.
  351. //
  352. // - The common pairing no match broker rotation is intended to partially
  353. // mitigate poor common proxy load balancing that can leave a broker
  354. // with little proxy supply. A more robust mitigation would be to make
  355. // proxies distribute announcements across multiple or even all brokers.
  356. personalPairing := len(personalCompartmentIDs) > 0
  357. // In the following cases, don't shuffle or otherwise mutate the original
  358. // broker spec slice, as it is a tactics parameter.
  359. if personalPairing {
  360. if len(personalCompartmentIDs[0]) < prng.SEED_LENGTH {
  361. // Both inproxy.ID and prng.SEED_LENGTH are 32 bytes.
  362. return nil, errors.TraceNew("unexpected ID length")
  363. }
  364. seed := prng.Seed(personalCompartmentIDs[0][0:prng.SEED_LENGTH])
  365. PRNG := prng.NewPRNGWithSeed(&seed)
  366. permutedIndexes := PRNG.Perm(len(brokerSpecs))
  367. // Minimize rendezvous time by reducing the number of brokers this
  368. // personal compartment ID maps over to. With a reduced number of
  369. // possible brokers, the client and proxy have fewer brokers to check
  370. // after fail overs.
  371. //
  372. // Given that permutedIndexes is a randomized shuffle, each personal
  373. // compartment ID will map to a different set of reduced brokers,
  374. // preserving overall broker load balancing.
  375. //
  376. // InproxyPersonalPairingMaxBrokerSpecCount will be configured high
  377. // enough to also preserve reasonable availability when brokers fail.
  378. // When InproxyPersonalPairingMaxBrokerSpecCount is 0, there is no max.
  379. //
  380. // This scheme depends on the len(personalCompartmentIDs) <= 1
  381. // constraint checked above.
  382. maxBrokerSpecs := p.Int(parameters.InproxyPersonalPairingMaxBrokerSpecCount)
  383. if maxBrokerSpecs > 0 && len(permutedIndexes) > maxBrokerSpecs {
  384. permutedIndexes = permutedIndexes[:maxBrokerSpecs]
  385. }
  386. selectedIndex := permutedIndexes[brokerSelectCount%len(permutedIndexes)]
  387. brokerSpecs = brokerSpecs[selectedIndex : selectedIndex+1]
  388. } else {
  389. permutedIndexes := prng.Perm(len(brokerSpecs))
  390. shuffledBrokerSpecs := make(parameters.InproxyBrokerSpecsValue, len(brokerSpecs))
  391. for i, index := range permutedIndexes {
  392. shuffledBrokerSpecs[i] = brokerSpecs[index]
  393. }
  394. brokerSpecs = shuffledBrokerSpecs
  395. }
  396. selectFirstCandidate := resetReasonNoMatch || personalPairing
  397. // Replay broker dial parameters.
  398. // In selectFirstCandidate cases, SelectCandidateWithNetworkReplayParameters
  399. // will always select the first candidate, returning corresponding replay
  400. // data when available. Otherwise, SelectCandidateWithNetworkReplayParameters
  401. // iterates over the shuffled candidates and returns the first with replay data.
  402. var brokerSpec *parameters.InproxyBrokerSpec
  403. var brokerDialParams *InproxyBrokerDialParameters
  404. // Replay is disabled when the TTL, InproxyReplayBrokerDialParametersTTL,
  405. // is 0.
  406. now := time.Now()
  407. ttl := p.Duration(parameters.InproxyReplayBrokerDialParametersTTL)
  408. replayEnabled := ttl > 0 &&
  409. !config.DisableReplay &&
  410. prng.FlipWeightedCoin(p.Float(parameters.InproxyReplayBrokerDialParametersProbability))
  411. if replayEnabled {
  412. brokerSpec, brokerDialParams, err =
  413. SelectCandidateWithNetworkReplayParameters[parameters.InproxyBrokerSpec, InproxyBrokerDialParameters](
  414. networkID,
  415. selectFirstCandidate,
  416. brokerSpecs,
  417. func(spec *parameters.InproxyBrokerSpec) string { return spec.BrokerPublicKey },
  418. func(spec *parameters.InproxyBrokerSpec, dialParams *InproxyBrokerDialParameters) bool {
  419. // Replay the successful broker spec, if present, by
  420. // comparing its hash with that of the candidate.
  421. return dialParams.LastUsedTimestamp.After(now.Add(-ttl)) &&
  422. bytes.Equal(dialParams.LastUsedBrokerSpecHash, hashBrokerSpec(spec))
  423. })
  424. if err != nil {
  425. NoticeWarning("SelectCandidateWithNetworkReplayParameters failed: %v", errors.Trace(err))
  426. // Continue without replay
  427. }
  428. }
  429. // Select the first broker in the shuffle when replay is not enabled or in
  430. // case SelectCandidateWithNetworkReplayParameters fails.
  431. if brokerSpec == nil {
  432. brokerSpec = brokerSpecs[0]
  433. }
  434. // Generate new broker dial parameters if not replaying. Later, isReplay
  435. // is used to report the replay metric.
  436. isReplay := brokerDialParams != nil
  437. // Handle legacy replay records by discarding replay when required fields
  438. // are missing.
  439. if isReplay && brokerDialParams.FrontedHTTPDialParameters == nil {
  440. isReplay = false
  441. }
  442. if !isReplay {
  443. brokerDialParams, err = MakeInproxyBrokerDialParameters(config, p, networkID, brokerSpec, brokerClientManager.tlsCache)
  444. if err != nil {
  445. return nil, errors.Trace(err)
  446. }
  447. } else {
  448. brokerDialParams.brokerSpec = brokerSpec
  449. err := brokerDialParams.prepareDialConfigs(config, p, true, brokerClientManager.tlsCache)
  450. if err != nil {
  451. return nil, errors.Trace(err)
  452. }
  453. }
  454. // Load broker key material.
  455. brokerPublicKey, err := inproxy.SessionPublicKeyFromString(brokerSpec.BrokerPublicKey)
  456. if err != nil {
  457. return nil, errors.Trace(err)
  458. }
  459. brokerRootObfuscationSecret, err := inproxy.ObfuscationSecretFromString(brokerSpec.BrokerRootObfuscationSecret)
  460. if err != nil {
  461. return nil, errors.Trace(err)
  462. }
  463. roundTripper := NewInproxyBrokerRoundTripper(p, brokerDialParams)
  464. // Clients always generate an ephemeral session key pair. Proxies may opt
  465. // to use a long-lived key pair for proxied traffic attribution.
  466. var brokerClientPrivateKey inproxy.SessionPrivateKey
  467. if isProxy && config.InproxyProxySessionPrivateKey != "" {
  468. brokerClientPrivateKey, err = inproxy.SessionPrivateKeyFromString(config.InproxyProxySessionPrivateKey)
  469. if err != nil {
  470. return nil, errors.Trace(err)
  471. }
  472. } else {
  473. brokerClientPrivateKey, err = inproxy.GenerateSessionPrivateKey()
  474. if err != nil {
  475. return nil, errors.Trace(err)
  476. }
  477. }
  478. // InproxyBrokerClientInstance implements the
  479. // inproxy.BrokerDialCoordinator interface and passes itself to
  480. // inproxy.NewBrokerClient in order to provide the round tripper, key
  481. // material, compartment IDs, timeouts, and other configuration to the
  482. // in-proxy broker client.
  483. //
  484. // Timeouts are not replayed, but snapshots are stored in the
  485. // InproxyBrokerClientInstance for efficient lookup.
  486. b := &InproxyBrokerClientInstance{
  487. config: config,
  488. brokerClientManager: brokerClientManager,
  489. networkID: networkID,
  490. brokerClientPrivateKey: brokerClientPrivateKey,
  491. brokerPublicKey: brokerPublicKey,
  492. brokerRootObfuscationSecret: brokerRootObfuscationSecret,
  493. brokerDialParams: brokerDialParams,
  494. replayEnabled: replayEnabled,
  495. roundTripper: roundTripper,
  496. personalCompartmentIDs: personalCompartmentIDs,
  497. commonCompartmentIDs: commonCompartmentIDs,
  498. sessionHandshakeTimeout: p.Duration(parameters.InproxySessionHandshakeRoundTripTimeout),
  499. announceRequestTimeout: p.Duration(parameters.InproxyProxyAnnounceRequestTimeout),
  500. announceDelay: p.Duration(parameters.InproxyProxyAnnounceDelay),
  501. announceMaxBackoffDelay: p.Duration(parameters.InproxyProxyAnnounceMaxBackoffDelay),
  502. announceDelayJitter: p.Float(parameters.InproxyProxyAnnounceDelayJitter),
  503. answerRequestTimeout: p.Duration(parameters.InproxyProxyAnswerRequestTimeout),
  504. offerRequestTimeout: p.Duration(parameters.InproxyClientOfferRequestTimeout),
  505. offerRequestPersonalTimeout: p.Duration(parameters.InproxyClientOfferRequestPersonalTimeout),
  506. offerRetryDelay: p.Duration(parameters.InproxyClientOfferRetryDelay),
  507. offerRetryJitter: p.Float(parameters.InproxyClientOfferRetryJitter),
  508. relayedPacketRequestTimeout: p.Duration(parameters.InproxyClientRelayedPacketRequestTimeout),
  509. dslRequestTimeout: p.Duration(parameters.InproxyClientDSLRequestTimeout),
  510. replayRetainFailedProbability: p.Float(parameters.InproxyReplayBrokerRetainFailedProbability),
  511. replayUpdateFrequency: p.Duration(parameters.InproxyReplayBrokerUpdateFrequency),
  512. }
  513. if isProxy && !config.IsInproxyProxyPersonalPairingMode() {
  514. // This retry is applied only for proxies and only in common pairing
  515. // mode. See comment in BrokerClientRoundTripperFailed.
  516. b.retryOnFailedPeriod = p.Duration(parameters.InproxyProxyOnBrokerClientFailedRetryPeriod)
  517. }
  518. // Limitation: currently, disableWaitToShareSession is neither replayed
  519. // nor is the selected value reported in metrics. The default tactics
  520. // parameters are considered to be optimal: the in-proxy clients
  521. // disabling wait and proxies using wait. The tactics flag can be used to
  522. // enable wait for clients in case performance is poor or load on
  523. // brokers -- due to simultaneous sessions -- is unexpectedly high.
  524. //
  525. // Note that, for broker dial parameter replay, the isValidReplay function
  526. // currently invalidates replay only when broker specs change, and not
  527. // when tactics in general change; so changes to these
  528. // disableWaitToShareSession parameters would not properly invalidate
  529. // replays in any case.
  530. //
  531. // Potential future enhancements for in-proxy client broker clients
  532. // include using a pool of broker clients, with each one potentially
  533. // using a different broker and/or fronting spec. In this scenario,
  534. // waitToShareSession would be less impactful.
  535. if isProxy {
  536. b.disableWaitToShareSession = p.Bool(parameters.InproxyProxyDisableWaitToShareSession)
  537. } else {
  538. b.disableWaitToShareSession = p.Bool(parameters.InproxyClientDisableWaitToShareSession)
  539. }
  540. // Adjust long-polling request timeouts to respect any maximum request
  541. // timeout supported by the provider fronting the request.
  542. maxRequestTimeout, ok := p.KeyDurations(
  543. parameters.InproxyFrontingProviderClientMaxRequestTimeouts)[brokerDialParams.FrontedHTTPDialParameters.FrontingProviderID]
  544. if ok && maxRequestTimeout > 0 {
  545. if b.announceRequestTimeout > maxRequestTimeout {
  546. b.announceRequestTimeout = maxRequestTimeout
  547. }
  548. if b.offerRequestTimeout > maxRequestTimeout {
  549. b.offerRequestTimeout = maxRequestTimeout
  550. }
  551. if b.offerRequestPersonalTimeout > maxRequestTimeout {
  552. b.offerRequestPersonalTimeout = maxRequestTimeout
  553. }
  554. }
  555. // Initialize broker client. This will start with a fresh broker session.
  556. //
  557. // When resetBrokerClientOnRoundTripperFailed is invoked due to a failure
  558. // at the transport level -- TLS or domain fronting --
  559. // NewInproxyBrokerClientInstance is invoked, resetting both the broker
  560. // client round tripper and the broker session. As a future enhancement,
  561. // consider distinguishing between transport and session errors and
  562. // retaining a valid established session when only the transport needs to
  563. // be reset/retried.
  564. b.brokerClient, err = inproxy.NewBrokerClient(b)
  565. if err != nil {
  566. return nil, errors.Trace(err)
  567. }
  568. // The broker ID is the broker's session public key in Curve25519 form.
  569. brokerID, err := brokerPublicKey.ToCurve25519()
  570. if err != nil {
  571. return nil, errors.Trace(err)
  572. }
  573. NoticeInfo("inproxy: selected broker %s", inproxy.ID(brokerID))
  574. return b, nil
  575. }
  576. func haveInproxyProxyBrokerSpecs(config *Config) bool {
  577. p := config.GetParameters().Get()
  578. defer p.Close()
  579. return len(getInproxyBrokerSpecs(config, p, true)) > 0
  580. }
  581. func haveInproxyClientBrokerSpecs(config *Config) bool {
  582. p := config.GetParameters().Get()
  583. defer p.Close()
  584. return len(getInproxyBrokerSpecs(config, p, false)) > 0
  585. }
  586. func getInproxyBrokerSpecs(
  587. config *Config,
  588. p parameters.ParametersAccessor,
  589. isProxy bool) parameters.InproxyBrokerSpecsValue {
  590. if isProxy {
  591. if config.IsInproxyProxyPersonalPairingMode() {
  592. return p.InproxyBrokerSpecs(
  593. parameters.InproxyProxyPersonalPairingBrokerSpecs,
  594. parameters.InproxyPersonalPairingBrokerSpecs,
  595. parameters.InproxyProxyBrokerSpecs,
  596. parameters.InproxyBrokerSpecs)
  597. } else {
  598. return p.InproxyBrokerSpecs(
  599. parameters.InproxyProxyBrokerSpecs,
  600. parameters.InproxyBrokerSpecs)
  601. }
  602. } else {
  603. if config.IsInproxyClientPersonalPairingMode() {
  604. return p.InproxyBrokerSpecs(
  605. parameters.InproxyClientPersonalPairingBrokerSpecs,
  606. parameters.InproxyPersonalPairingBrokerSpecs,
  607. parameters.InproxyClientBrokerSpecs,
  608. parameters.InproxyBrokerSpecs)
  609. } else {
  610. return p.InproxyBrokerSpecs(
  611. parameters.InproxyClientBrokerSpecs,
  612. parameters.InproxyBrokerSpecs)
  613. }
  614. }
  615. }
  616. func haveInproxyCommonCompartmentIDs(config *Config) bool {
  617. p := config.GetParameters().Get()
  618. defer p.Close()
  619. if len(p.InproxyCompartmentIDs(parameters.InproxyCommonCompartmentIDs)) > 0 {
  620. return true
  621. }
  622. commonCompartmentIDs, _ := LoadInproxyCommonCompartmentIDs()
  623. return len(commonCompartmentIDs) > 0
  624. }
  625. func prepareInproxyCompartmentIDs(
  626. config *Config,
  627. p parameters.ParametersAccessor,
  628. isProxy bool) ([]inproxy.ID, []inproxy.ID, error) {
  629. // Personal compartment IDs are loaded from the tunnel-core config; these
  630. // are set by the external app based on user input/configuration of IDs
  631. // generated by or obtained from personal proxies. Both clients and
  632. // proxies send personal compartment IDs to the in-proxy broker. For
  633. // clients, when personal compartment IDs are configured, no common
  634. // compartment IDs are prepared, ensuring matches with only proxies that
  635. // supply the corresponding personal compartment IDs.
  636. //
  637. // Common compartment IDs are obtained from tactics and merged with
  638. // previously learned IDs stored in the local datastore. When new IDs are
  639. // obtained from tactics, the merged list is written back to the
  640. // datastore. This allows for schemes where common compartment IDs are
  641. // distributed to sets of clients, then removed from distibution, and
  642. // still used to match proxies to those sets of clients. Only clients
  643. // send common compartment IDs to the in-proxy broker. Proxies are
  644. // automatically assigned to common compartments by the broker.
  645. //
  646. // Maximum compartment ID list lengths are enforced to ensure broker
  647. // request sizes don't grow unbounded.
  648. //
  649. // Limitation: currently, in max length trimming, new common compartment
  650. // IDs take precedence over older IDs.
  651. maxCompartmentIDListLength := p.Int(parameters.InproxyMaxCompartmentIDListLength)
  652. // Personal compartment ID limitations:
  653. //
  654. // The broker API messages, ProxyAnnounceRequest and ClientOfferRequest,
  655. // support lists of personal compartment IDs. However, both the proxy and
  656. // the client are currently limited to specifying at most one personal
  657. // compartment ID due to the following limitations:
  658. //
  659. // - On the broker side, the matcher queue implementation supports at most
  660. // one proxy personal compartment ID. See inproxy/Matcher.Announce. The
  661. // broker currently enforces that at most one personal compartment ID
  662. // may be specified per ProxyAnnounceRequest.
  663. //
  664. // - On the proxy/client side, the personal pairing rendezvous logic --
  665. // which aims for proxies and clients to select the same initial broker
  666. // and same order of failover to other brokers -- uses a shuffle that
  667. // assumes both the proxy and client use the same single, personal
  668. // compartment ID
  669. var configPersonalCompartmentIDs []string
  670. if isProxy && len(config.InproxyProxyPersonalCompartmentID) > 0 {
  671. configPersonalCompartmentIDs = []string{config.InproxyProxyPersonalCompartmentID}
  672. } else if !isProxy && len(config.InproxyClientPersonalCompartmentID) > 0 {
  673. configPersonalCompartmentIDs = []string{config.InproxyClientPersonalCompartmentID}
  674. }
  675. personalCompartmentIDs, err := inproxy.IDsFromStrings(configPersonalCompartmentIDs)
  676. if err != nil {
  677. return nil, nil, errors.Trace(err)
  678. }
  679. if len(personalCompartmentIDs) > maxCompartmentIDListLength {
  680. // Trim the list. It's not expected that user-configured personal
  681. // compartment ID lists will exceed the max length.
  682. //
  683. // TODO: shuffle before trimming? Prioritize previous matches?
  684. personalCompartmentIDs = personalCompartmentIDs[:maxCompartmentIDListLength]
  685. }
  686. var commonCompartmentIDs []inproxy.ID
  687. if !isProxy && len(personalCompartmentIDs) == 0 {
  688. tacticsCommonCompartmentIDs := p.InproxyCompartmentIDs(parameters.InproxyCommonCompartmentIDs)
  689. knownCommonCompartmentIDs, err := LoadInproxyCommonCompartmentIDs()
  690. if err != nil {
  691. NoticeWarning("LoadInproxyCommonCompartmentIDs failed: %v", errors.Trace(err))
  692. // Continue with only the tactics common compartment IDs.
  693. }
  694. newCompartmentIDs := make([]string, 0, len(tacticsCommonCompartmentIDs))
  695. for _, compartmentID := range tacticsCommonCompartmentIDs {
  696. // TODO: faster lookup?
  697. if !common.Contains(knownCommonCompartmentIDs, compartmentID) {
  698. newCompartmentIDs = append(newCompartmentIDs, compartmentID)
  699. }
  700. }
  701. if len(newCompartmentIDs) > 0 {
  702. newCompartmentIDs = append(newCompartmentIDs, knownCommonCompartmentIDs...)
  703. // Locally store more than InproxyMaxCompartmentIDListLength known
  704. // common compartment IDs, in case the request limit parameter is
  705. // increased in the future.
  706. // maxPersistedCommonCompartmentIDListLength still limits the
  707. // length of the list to cap local memory and disk impact.
  708. maxPersistedCommonCompartmentIDListLength := 500 // ~16K
  709. if maxCompartmentIDListLength > maxPersistedCommonCompartmentIDListLength {
  710. maxPersistedCommonCompartmentIDListLength = maxCompartmentIDListLength
  711. }
  712. if len(newCompartmentIDs) > maxPersistedCommonCompartmentIDListLength {
  713. newCompartmentIDs = newCompartmentIDs[:maxPersistedCommonCompartmentIDListLength]
  714. }
  715. err := StoreInproxyCommonCompartmentIDs(newCompartmentIDs)
  716. if err != nil {
  717. NoticeWarning("StoreInproxyCommonCompartmentIDs failed: %v", errors.Trace(err))
  718. // Continue without persisting new common compartment IDs.
  719. }
  720. knownCommonCompartmentIDs = newCompartmentIDs
  721. }
  722. commonCompartmentIDs, err = inproxy.IDsFromStrings(knownCommonCompartmentIDs)
  723. if err != nil {
  724. return nil, nil, errors.Trace(err)
  725. }
  726. if len(commonCompartmentIDs) > maxCompartmentIDListLength {
  727. // TODO: shuffle before trimming? Prioritize previous matches?
  728. commonCompartmentIDs = commonCompartmentIDs[:maxCompartmentIDListLength]
  729. }
  730. }
  731. return commonCompartmentIDs, personalCompartmentIDs, nil
  732. }
  733. // HasSuccess indicates whether this broker client instance has completed at
  734. // least one successful round trip.
  735. func (b *InproxyBrokerClientInstance) HasSuccess() bool {
  736. b.mutex.Lock()
  737. defer b.mutex.Unlock()
  738. return !b.lastSuccess.IsZero()
  739. }
  740. // Close closes the broker client round tripper, including closing all
  741. // underlying network connections, which will interrupt any in-flight round
  742. // trips.
  743. func (b *InproxyBrokerClientInstance) Close() error {
  744. // Concurrency note: Close is called from InproxyBrokerClientManager with
  745. // its mutex locked. Close must not lock InproxyBrokerClientInstance's
  746. // mutex, or else there is a risk of deadlock similar to the HasSuccess
  747. // case documented in InproxyBrokerClientManager.GetBrokerClient.
  748. err := b.roundTripper.Close()
  749. return errors.Trace(err)
  750. }
  751. // Implements the inproxy.BrokerDialCoordinator interface.
  752. func (b *InproxyBrokerClientInstance) NetworkID() string {
  753. return b.networkID
  754. }
  755. // Implements the inproxy.BrokerDialCoordinator interface.
  756. func (b *InproxyBrokerClientInstance) NetworkType() inproxy.NetworkType {
  757. return getInproxyNetworkType(GetNetworkType(b.networkID))
  758. }
  759. // Implements the inproxy.BrokerDialCoordinator interface.
  760. func (b *InproxyBrokerClientInstance) CommonCompartmentIDs() []inproxy.ID {
  761. return b.commonCompartmentIDs
  762. }
  763. // Implements the inproxy.BrokerDialCoordinator interface.
  764. func (b *InproxyBrokerClientInstance) PersonalCompartmentIDs() []inproxy.ID {
  765. return b.personalCompartmentIDs
  766. }
  767. // Implements the inproxy.BrokerDialCoordinator interface.
  768. func (b *InproxyBrokerClientInstance) DisableWaitToShareSession() bool {
  769. return b.disableWaitToShareSession
  770. }
  771. // Implements the inproxy.BrokerDialCoordinator interface.
  772. func (b *InproxyBrokerClientInstance) BrokerClientPrivateKey() inproxy.SessionPrivateKey {
  773. return b.brokerClientPrivateKey
  774. }
  775. // Implements the inproxy.BrokerDialCoordinator interface.
  776. func (b *InproxyBrokerClientInstance) BrokerPublicKey() inproxy.SessionPublicKey {
  777. return b.brokerPublicKey
  778. }
  779. // Implements the inproxy.BrokerDialCoordinator interface.
  780. func (b *InproxyBrokerClientInstance) BrokerRootObfuscationSecret() inproxy.ObfuscationSecret {
  781. return b.brokerRootObfuscationSecret
  782. }
  783. // Implements the inproxy.BrokerDialCoordinator interface.
  784. func (b *InproxyBrokerClientInstance) BrokerClientRoundTripper() (inproxy.RoundTripper, error) {
  785. // Returns the same round tripper for the lifetime of the
  786. // inproxy.BrokerDialCoordinator, ensuring all requests for one in-proxy
  787. // dial or proxy relay use the same broker, as is necessary due to the
  788. // broker state for the proxy announce/answer, client broker/server
  789. // relay, etc.
  790. return b.roundTripper, nil
  791. }
  792. // Implements the inproxy.BrokerDialCoordinator interface.
  793. func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperSucceeded(roundTripper inproxy.RoundTripper) {
  794. b.mutex.Lock()
  795. defer b.mutex.Unlock()
  796. if rt, ok := roundTripper.(*InproxyBrokerRoundTripper); !ok || rt != b.roundTripper {
  797. // Passing in the round tripper obtained from BrokerClientRoundTripper
  798. // is just used for sanity check in this implementation, since each
  799. // InproxyBrokerClientInstance has exactly one round tripper.
  800. NoticeError("BrokerClientRoundTripperSucceeded: roundTripper instance mismatch")
  801. return
  802. }
  803. now := time.Now()
  804. b.lastSuccess = now
  805. // Set replay or extend the broker dial parameters replay TTL after a
  806. // success. With tunnel dial parameters, the replay TTL is extended after
  807. // every successful tunnel connection. Since there are potentially more
  808. // and more frequent broker round trips compared to tunnel dials, the TTL
  809. // is only extended after some target duration has elapsed, to avoid
  810. // excessive datastore writes.
  811. if b.replayEnabled && now.Sub(b.lastStoreReplay) > b.replayUpdateFrequency {
  812. b.brokerDialParams.LastUsedTimestamp = time.Now()
  813. err := SetNetworkReplayParameters[InproxyBrokerDialParameters](
  814. b.networkID, b.brokerDialParams.brokerSpec.BrokerPublicKey, b.brokerDialParams)
  815. if err != nil {
  816. NoticeWarning("StoreBrokerDialParameters failed: %v", errors.Trace(err))
  817. // Continue without persisting replay changes.
  818. } else {
  819. b.lastStoreReplay = now
  820. }
  821. }
  822. // Verify/extend the resolver cache entry for any resolved domain after a
  823. // success.
  824. //
  825. // Limitation: currently this re-extends regardless of how long ago the DNS
  826. // resolve happened.
  827. resolver := b.config.GetResolver()
  828. if resolver != nil {
  829. resolver.VerifyCacheExtension(b.brokerDialParams.FrontedHTTPDialParameters.FrontingDialAddress)
  830. }
  831. }
  832. // Implements the inproxy.BrokerDialCoordinator interface.
  833. func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperFailed(roundTripper inproxy.RoundTripper) {
  834. b.mutex.Lock()
  835. defer b.mutex.Unlock()
  836. if rt, ok := roundTripper.(*InproxyBrokerRoundTripper); !ok || rt != b.roundTripper {
  837. // Passing in the round tripper obtained from BrokerClientRoundTripper
  838. // is just used for sanity check in this implementation, since each
  839. // InproxyBrokerClientInstance has exactly one round tripper.
  840. NoticeError("BrokerClientRoundTripperFailed: roundTripper instance mismatch")
  841. return
  842. }
  843. // For common pairing proxies, skip both the replay deletion and the
  844. // InproxyBrokerClientInstance reset for a short duration after a recent
  845. // round trip success. In this case, subsequent broker requests will use
  846. // the existing round tripper, wired up with the same dial parameters and
  847. // fronting provider selection. If the failure was due to a transient
  848. // TLS/TCP network failure, the net/http round tripper should establish a
  849. // new connection on the next request.
  850. //
  851. // This retry is intended to retain proxy affinity with its currently
  852. // selected broker in cases such as broker service upgrades/restarts or
  853. // brief network interruptions, mitigating load balancing issues that
  854. // otherwise occur (e.g., all proxies fail over to other brokers, leaving
  855. // no supply on a restarted broker).
  856. //
  857. // In common pairing mode, clients do not perform this retry and
  858. // immediately reset, as is appropriate for the tunnel establishment
  859. // race. In personal pairing mode, neither proxies nor clients retry and
  860. // instead follow the personal pairing broker selection scheme in an
  861. // effort to rendezvous at the same broker with minimal delay.
  862. //
  863. // A delay before retrying announce requests is appropriate, but there is
  864. // no delay added here since Proxy.proxyOneClient already schedule delays
  865. // between announcements.
  866. //
  867. // Limitation: BrokerClientRoundTripperSucceeded is not invoked -- and no
  868. // recent last success time is set -- for proxies which announce, don't
  869. // match, and then hit the misaligned fronting provider request timeout
  870. // issue. See the ""unexpected response status code" case and comment in
  871. // InproxyBrokerRoundTripper.RoundTrip. This case should be mitigated by
  872. // configuring InproxyFrontingProviderServerMaxRequestTimeouts.
  873. //
  874. // TODO: also retry after initial startup, with no previous success? This
  875. // would further retain random load balancing of proxies newly starting
  876. // at the same time that their initially selected broker is restarted or
  877. // briefly unavailable.
  878. if b.brokerClientManager.isProxy &&
  879. !b.config.IsInproxyProxyPersonalPairingMode() &&
  880. b.retryOnFailedPeriod > 0 &&
  881. !b.lastSuccess.IsZero() &&
  882. time.Since(b.lastSuccess) <= b.retryOnFailedPeriod {
  883. NoticeWarning("BrokerClientRoundTripperFailed: retry roundTripper")
  884. return
  885. }
  886. // Delete any persistent replay dial parameters. Unlike with the success
  887. // case, consecutive, repeated deletes shouldn't write to storage, so
  888. // they are not avoided.
  889. if b.replayEnabled &&
  890. !prng.FlipWeightedCoin(b.replayRetainFailedProbability) {
  891. // Limitation: there's a race condition with multiple
  892. // InproxyBrokerClientInstances writing to the replay datastore for
  893. // the same broker, such as in the case where there's a dual-mode
  894. // in-proxy client and proxy; this delete could potentially clobber a
  895. // concurrent fresh replay store after a success.
  896. //
  897. // TODO: add an additional storage key distinguisher for each instance?
  898. err := DeleteNetworkReplayParameters[InproxyBrokerDialParameters](
  899. b.networkID, b.brokerDialParams.brokerSpec.BrokerPublicKey)
  900. if err != nil {
  901. NoticeWarning("DeleteBrokerDialParameters failed: %v", errors.Trace(err))
  902. // Continue without resetting replay.
  903. }
  904. }
  905. // Remove the TLS session cache entry for the broker's fronting dial address, if present.
  906. // This ensures that the next round trip establishes a new TLS session, avoiding potential issues
  907. // caused by session resumption fingerprint that may have contributed to the round tripper failure.
  908. if hardcodedCache := b.brokerDialParams.FrontedHTTPDialParameters.meekConfig.TLSClientSessionCache; hardcodedCache != nil {
  909. hardcodedCache.RemoveCacheEntry()
  910. }
  911. // Invoke resetBrokerClientOnRoundTripperFailed to signal the
  912. // InproxyBrokerClientManager to create a new
  913. // InproxyBrokerClientInstance, with new dial parameters and a new round
  914. // tripper, after a failure.
  915. //
  916. // This InproxyBrokerClientInstance doesn't change its dial parameters or
  917. // round tripper to ensure that any concurrent usage retains affinity
  918. // with the same parameters and broker.
  919. //
  920. // Limitation: a transport-level failure may unnecessarily reset the
  921. // broker session state; see comment in NewInproxyBrokerClientInstance.
  922. err := b.brokerClientManager.resetBrokerClientOnRoundTripperFailed(b)
  923. if err != nil {
  924. NoticeWarning("reset broker client failed: %v", errors.Trace(err))
  925. // Continue with old broker client instance.
  926. }
  927. }
  928. // Implements the inproxy.BrokerDialCoordinator interface.
  929. func (b *InproxyBrokerClientInstance) BrokerClientNoMatch(roundTripper inproxy.RoundTripper) {
  930. b.mutex.Lock()
  931. defer b.mutex.Unlock()
  932. if rt, ok := roundTripper.(*InproxyBrokerRoundTripper); !ok || rt != b.roundTripper {
  933. // See roundTripper check comment in BrokerClientRoundTripperFailed.
  934. NoticeError("BrokerClientNoMatch: roundTripper instance mismatch")
  935. return
  936. }
  937. // Any persistent replay dial parameters are retained and not deleted,
  938. // since the broker client successfully transacted with the broker.
  939. err := b.brokerClientManager.resetBrokerClientOnNoMatch(b)
  940. if err != nil {
  941. NoticeWarning("reset broker client failed: %v", errors.Trace(err))
  942. // Continue with old broker client instance.
  943. }
  944. }
  945. // Implements the inproxy.BrokerDialCoordinator interface.
  946. func (b *InproxyBrokerClientInstance) MetricsForBrokerRequests() common.LogFields {
  947. return b.brokerDialParams.GetMetricsForBrokerRequests()
  948. }
  949. // Implements the inproxy.BrokerDialCoordinator interface.
  950. func (b *InproxyBrokerClientInstance) AnnounceRequestTimeout() time.Duration {
  951. return b.announceRequestTimeout
  952. }
  953. // Implements the inproxy.BrokerDialCoordinator interface.
  954. func (b *InproxyBrokerClientInstance) SessionHandshakeRoundTripTimeout() time.Duration {
  955. return b.sessionHandshakeTimeout
  956. }
  957. // Implements the inproxy.BrokerDialCoordinator interface.
  958. func (b *InproxyBrokerClientInstance) AnnounceDelay() time.Duration {
  959. return b.announceDelay
  960. }
  961. // Implements the inproxy.BrokerDialCoordinator interface.
  962. func (b *InproxyBrokerClientInstance) AnnounceMaxBackoffDelay() time.Duration {
  963. return b.announceMaxBackoffDelay
  964. }
  965. // Implements the inproxy.BrokerDialCoordinator interface.
  966. func (b *InproxyBrokerClientInstance) AnnounceDelayJitter() float64 {
  967. return b.announceDelayJitter
  968. }
  969. // Implements the inproxy.BrokerDialCoordinator interface.
  970. func (b *InproxyBrokerClientInstance) AnswerRequestTimeout() time.Duration {
  971. return b.answerRequestTimeout
  972. }
  973. // Implements the inproxy.BrokerDialCoordinator interface.
  974. func (b *InproxyBrokerClientInstance) OfferRequestTimeout() time.Duration {
  975. return b.offerRequestTimeout
  976. }
  977. // Implements the inproxy.BrokerDialCoordinator interface.
  978. func (b *InproxyBrokerClientInstance) OfferRequestPersonalTimeout() time.Duration {
  979. return b.offerRequestPersonalTimeout
  980. }
  981. // Implements the inproxy.BrokerDialCoordinator interface.
  982. func (b *InproxyBrokerClientInstance) OfferRetryDelay() time.Duration {
  983. return b.offerRetryDelay
  984. }
  985. // Implements the inproxy.BrokerDialCoordinator interface.
  986. func (b *InproxyBrokerClientInstance) OfferRetryJitter() float64 {
  987. return b.offerRetryJitter
  988. }
  989. // Implements the inproxy.BrokerDialCoordinator interface.
  990. func (b *InproxyBrokerClientInstance) RelayedPacketRequestTimeout() time.Duration {
  991. return b.relayedPacketRequestTimeout
  992. }
  993. // Implements the inproxy.BrokerDialCoordinator interface.
  994. func (b *InproxyBrokerClientInstance) DSLRequestTimeout() time.Duration {
  995. return b.dslRequestTimeout
  996. }
  997. // InproxyBrokerDialParameters represents a selected broker transport and dial
  998. // paramaters.
  999. //
  1000. // InproxyBrokerDialParameters is used to configure dialers; as a persistent
  1001. // record to store successful dial parameters for replay; and to report dial
  1002. // stats in notices and Psiphon API calls.
  1003. //
  1004. // InproxyBrokerDialParameters is similar to tunnel DialParameters, but is
  1005. // specific to the in-proxy broker dial phase.
  1006. type InproxyBrokerDialParameters struct {
  1007. brokerSpec *parameters.InproxyBrokerSpec `json:"-"`
  1008. isReplay bool `json:"-"`
  1009. isReuse bool `json:"-"`
  1010. LastUsedTimestamp time.Time
  1011. LastUsedBrokerSpecHash []byte
  1012. FrontedHTTPDialParameters *FrontedMeekDialParameters
  1013. }
  1014. // MakeInproxyBrokerDialParameters creates a new InproxyBrokerDialParameters.
  1015. func MakeInproxyBrokerDialParameters(
  1016. config *Config,
  1017. p parameters.ParametersAccessor,
  1018. networkID string,
  1019. brokerSpec *parameters.InproxyBrokerSpec,
  1020. tlsCache utls.ClientSessionCache) (*InproxyBrokerDialParameters, error) {
  1021. if config.UseUpstreamProxy() {
  1022. return nil, errors.TraceNew("upstream proxy unsupported")
  1023. }
  1024. currentTimestamp := time.Now()
  1025. // Select new broker dial parameters
  1026. brokerDialParams := &InproxyBrokerDialParameters{
  1027. brokerSpec: brokerSpec,
  1028. LastUsedTimestamp: currentTimestamp,
  1029. LastUsedBrokerSpecHash: hashBrokerSpec(brokerSpec),
  1030. }
  1031. // FrontedMeekDialParameters
  1032. //
  1033. // The broker round trips use MeekModeWrappedPlaintextRoundTrip without
  1034. // meek cookies, so meek obfuscation is not configured. The in-proxy
  1035. // broker session payloads have their own obfuscation layer.
  1036. payloadSecure := true
  1037. skipVerify := false
  1038. var err error
  1039. brokerDialParams.FrontedHTTPDialParameters, err = makeFrontedMeekDialParameters(
  1040. config,
  1041. p,
  1042. nil,
  1043. brokerSpec.BrokerFrontingSpecs,
  1044. nil,
  1045. true,
  1046. skipVerify,
  1047. config.DisableSystemRootCAs,
  1048. payloadSecure,
  1049. tlsCache,
  1050. )
  1051. if err != nil {
  1052. return nil, errors.Trace(err)
  1053. }
  1054. // Initialize Dial/MeekConfigs to be passed to the corresponding dialers.
  1055. err = brokerDialParams.prepareDialConfigs(
  1056. config,
  1057. p,
  1058. false,
  1059. tlsCache)
  1060. if err != nil {
  1061. return nil, errors.Trace(err)
  1062. }
  1063. return brokerDialParams, nil
  1064. }
  1065. // prepareDialConfigs is called for both new and replayed broker dial parameters.
  1066. func (brokerDialParams *InproxyBrokerDialParameters) prepareDialConfigs(
  1067. config *Config,
  1068. p parameters.ParametersAccessor,
  1069. isReplay bool,
  1070. tlsCache utls.ClientSessionCache) error {
  1071. brokerDialParams.isReplay = isReplay
  1072. // brokerDialParams.isReuse is set only later, as this is a new broker
  1073. // client dial.
  1074. if isReplay {
  1075. // FrontedHTTPDialParameters
  1076. //
  1077. // The broker round trips use MeekModeWrappedPlaintextRoundTrip without
  1078. // meek cookies, so meek obfuscation is not configured. The in-proxy
  1079. // broker session payloads have their own obfuscation layer.
  1080. payloadSecure := true
  1081. skipVerify := false
  1082. err := brokerDialParams.FrontedHTTPDialParameters.prepareDialConfigs(
  1083. config, p, nil, nil, true, skipVerify,
  1084. config.DisableSystemRootCAs, payloadSecure, tlsCache)
  1085. if err != nil {
  1086. return errors.Trace(err)
  1087. }
  1088. }
  1089. return nil
  1090. }
  1091. // GetMetricsForBroker returns broker client dial parameter log fields to be
  1092. // reported to a broker.
  1093. func (brokerDialParams *InproxyBrokerDialParameters) GetMetricsForBrokerRequests() common.LogFields {
  1094. logFields := common.LogFields{}
  1095. // TODO: add additional broker fronting dial parameters to be logged by
  1096. // the broker -- as successful parameters might not otherwise by logged
  1097. // via server_tunnel if the subsequent WebRTC dials fail.
  1098. logFields["fronting_provider_id"] = brokerDialParams.FrontedHTTPDialParameters.FrontingProviderID
  1099. return logFields
  1100. }
  1101. // GetMetrics implements the common.MetricsSource interface and returns log
  1102. // fields detailing the broker dial parameters.
  1103. func (brokerDialParams *InproxyBrokerDialParameters) GetMetrics() common.LogFields {
  1104. logFields := common.LogFields{}
  1105. // Add underlying log fields, which must be renamed to be scoped to the
  1106. // broker.
  1107. logFields.Add(brokerDialParams.FrontedHTTPDialParameters.GetMetrics("inproxy_broker_"))
  1108. logFields["inproxy_broker_transport"] = brokerDialParams.FrontedHTTPDialParameters.FrontingTransport
  1109. isReplay := "0"
  1110. if brokerDialParams.isReplay {
  1111. isReplay = "1"
  1112. }
  1113. logFields["inproxy_broker_is_replay"] = isReplay
  1114. isReuse := "0"
  1115. if brokerDialParams.isReuse {
  1116. isReuse = "1"
  1117. }
  1118. logFields["inproxy_broker_is_reuse"] = isReuse
  1119. // TODO: include tlsConn.GetMetrics tls_did_resume/tls_sent_ticket.
  1120. // Requires a reference to the InproxyBrokerRoundTripper.
  1121. return logFields
  1122. }
  1123. // hashBrokerSpec hashes the broker spec. The hash is used to detect when
  1124. // broker spec tactics have changed.
  1125. func hashBrokerSpec(spec *parameters.InproxyBrokerSpec) []byte {
  1126. var hash [8]byte
  1127. binary.BigEndian.PutUint64(
  1128. hash[:],
  1129. uint64(xxhash.Sum64String(fmt.Sprintf("%+v", spec))))
  1130. return hash[:]
  1131. }
  1132. // InproxyBrokerRoundTripper is a broker request round trip transport
  1133. // implemented using MeekConn in MeekModePlaintextRoundTrip mode, utilizing
  1134. // MeekConn's domain fronting capabilities and using persistent and
  1135. // multiplexed connections, via HTTP/2, to support multiple concurrent
  1136. // in-flight round trips.
  1137. //
  1138. // InproxyBrokerRoundTripper implements the inproxy.RoundTripper interface.
  1139. type InproxyBrokerRoundTripper struct {
  1140. brokerDialParams *InproxyBrokerDialParameters
  1141. runCtx context.Context
  1142. stopRunning context.CancelFunc
  1143. dial int32
  1144. dialCompleted chan struct{}
  1145. dialErr error
  1146. conn *MeekConn
  1147. failureThreshold time.Duration
  1148. }
  1149. // NewInproxyBrokerRoundTripper creates a new InproxyBrokerRoundTripper. The
  1150. // initial DialMeek is defered until the first call to RoundTrip, so
  1151. // NewInproxyBrokerRoundTripper does not perform any network operations.
  1152. //
  1153. // The input brokerDialParams dial parameter and config fields must not
  1154. // modifed after NewInproxyBrokerRoundTripper is called.
  1155. func NewInproxyBrokerRoundTripper(
  1156. p parameters.ParametersAccessor,
  1157. brokerDialParams *InproxyBrokerDialParameters) *InproxyBrokerRoundTripper {
  1158. runCtx, stopRunning := context.WithCancel(context.Background())
  1159. return &InproxyBrokerRoundTripper{
  1160. brokerDialParams: brokerDialParams,
  1161. runCtx: runCtx,
  1162. stopRunning: stopRunning,
  1163. dialCompleted: make(chan struct{}),
  1164. failureThreshold: p.Duration(
  1165. parameters.InproxyBrokerRoundTripStatusCodeFailureThreshold),
  1166. }
  1167. }
  1168. // Close interrupts any in-flight request and closes the underlying
  1169. // MeekConn.
  1170. func (rt *InproxyBrokerRoundTripper) Close() error {
  1171. // Interrupt any DialMeek or RoundTrip.
  1172. rt.stopRunning()
  1173. if atomic.CompareAndSwapInt32(&rt.dial, 0, 1) {
  1174. // RoundTrip has not yet been called or has not yet kicked off
  1175. // DialMeek, so there is no MeekConn to close. Prevent any future
  1176. // DialMeek by signaling dialCompleted and fail any future round trip
  1177. // attempt by setting dialErr.
  1178. rt.dialErr = errors.TraceNew("closed")
  1179. close(rt.dialCompleted)
  1180. } else {
  1181. // Await any ongoing DialMeek or RoundTrip (stopRunning should
  1182. // interrupt either one quickly).
  1183. <-rt.dialCompleted
  1184. if rt.conn != nil {
  1185. _ = rt.conn.Close()
  1186. }
  1187. }
  1188. // As with MeekConn.Close, any Close errors from underlying conns are not
  1189. // propagated.
  1190. return nil
  1191. }
  1192. // RoundTrip transports a request to the broker endpoint and returns a
  1193. // response.
  1194. func (rt *InproxyBrokerRoundTripper) RoundTrip(
  1195. ctx context.Context,
  1196. roundTripDelay time.Duration,
  1197. roundTripTimeout time.Duration,
  1198. requestPayload []byte) (_ []byte, retErr error) {
  1199. defer func() {
  1200. // Log any error which results in invoking BrokerClientRoundTripperFailed.
  1201. var failedError *inproxy.RoundTripperFailedError
  1202. if std_errors.As(retErr, &failedError) {
  1203. NoticeWarning("RoundTripperFailedError: %v", retErr)
  1204. }
  1205. }()
  1206. // Cancel DialMeek or MeekConn.RoundTrip when:
  1207. // - Close is called
  1208. // - the input context is done
  1209. ctx, cancelFunc := common.MergeContextCancel(ctx, rt.runCtx)
  1210. defer cancelFunc()
  1211. // Apply any round trip delay. Currently, this is used to apply an
  1212. // announce request delay post-waitToShareSession, pre-network round
  1213. // trip, and cancelable by the above merged context.
  1214. if roundTripDelay > 0 {
  1215. common.SleepWithContext(ctx, roundTripDelay)
  1216. }
  1217. // Apply the round trip timeout after any delay is complete.
  1218. //
  1219. // This timeout includes any TLS handshake network round trips, as
  1220. // performed by the initial DialMeek and may be performed subsequently by
  1221. // net/http via MeekConn.RoundTrip. These extra round trips should be
  1222. // accounted for in the in the difference between client-side request
  1223. // timeouts, such as InproxyProxyAnswerRequestTimeout, and broker-side
  1224. // handler timeouts, such as InproxyBrokerProxyAnnounceTimeout, with the
  1225. // former allowing more time for network round trips.
  1226. requestCtx := ctx
  1227. if roundTripTimeout > 0 {
  1228. var requestCancelFunc context.CancelFunc
  1229. requestCtx, requestCancelFunc = context.WithTimeout(ctx, roundTripTimeout)
  1230. defer requestCancelFunc()
  1231. }
  1232. // The first RoundTrip caller will perform the DialMeek step, which
  1233. // establishes the TLS trasport connection to the fronted endpoint.
  1234. // Following callers will await that DialMeek or share an established
  1235. // connection.
  1236. //
  1237. // To accomodate using custom utls fingerprints, with varying ALPNs, with
  1238. // net/http, DialMeek completes a full TLS handshake before instantiating
  1239. // the appropriate http.Transport or http2.Transport. Until that first
  1240. // DialMeek completes, and unlike standard net/http round trips,
  1241. // InproxyBrokerRoundTripper won't spawn distinct TLS persistent
  1242. // connections for concurrent round trips. After DialMeek, concurrent
  1243. // round trips over HTTP/2 connections may simply share the one TLS
  1244. // connection, while concurrent round trips over HTTP connections may
  1245. // spawn additional TLS persistent connections.
  1246. //
  1247. // There is no retry here if DialMeek fails, as higher levels will invoke
  1248. // BrokerClientRoundTripperFailed on failure, clear any replay, select
  1249. // new dial parameters, and retry.
  1250. if atomic.CompareAndSwapInt32(&rt.dial, 0, 1) {
  1251. // DialMeek hasn't been called yet.
  1252. conn, err := DialMeek(
  1253. requestCtx,
  1254. rt.brokerDialParams.FrontedHTTPDialParameters.meekConfig,
  1255. rt.brokerDialParams.FrontedHTTPDialParameters.dialConfig)
  1256. if err != nil && ctx.Err() != context.Canceled {
  1257. // DialMeek performs an initial TLS handshake. DialMeek errors,
  1258. // excluding a cancelled context as happens on shutdown, are
  1259. // classified as as RoundTripperFailedErrors, which will invoke
  1260. // BrokerClientRoundTripperFailed, resetting the round tripper
  1261. // and clearing replay parameters.
  1262. err = inproxy.NewRoundTripperFailedError(err)
  1263. }
  1264. rt.conn = conn
  1265. rt.dialErr = err
  1266. close(rt.dialCompleted)
  1267. if err != nil {
  1268. return nil, errors.Trace(rt.dialErr)
  1269. }
  1270. } else {
  1271. // Await any ongoing DialMeek run by a concurrent RoundTrip caller.
  1272. select {
  1273. case <-rt.dialCompleted:
  1274. case <-ctx.Done():
  1275. return nil, errors.Trace(ctx.Err())
  1276. }
  1277. if rt.dialErr != nil {
  1278. // There is no NewRoundTripperFailedError wrapping here, as the
  1279. // DialMeek caller will wrap its error and
  1280. // BrokerClientRoundTripperFailed will be invoked already.
  1281. return nil, errors.Trace(rt.dialErr)
  1282. }
  1283. }
  1284. // At this point, rt.conn is an established MeekConn.
  1285. // Note that the network address portion of the URL will be ignored by
  1286. // MeekConn in favor of the MeekDialConfig, while the path will be used.
  1287. url := fmt.Sprintf(
  1288. "https://%s/%s",
  1289. rt.brokerDialParams.FrontedHTTPDialParameters.DialAddress,
  1290. inproxy.BrokerEndPointName)
  1291. request, err := http.NewRequestWithContext(
  1292. requestCtx, "POST", url, bytes.NewBuffer(requestPayload))
  1293. if err != nil {
  1294. return nil, errors.Trace(err)
  1295. }
  1296. startTime := time.Now()
  1297. response, err := rt.conn.RoundTrip(request)
  1298. roundTripDuration := time.Since(startTime)
  1299. if err == nil {
  1300. defer response.Body.Close()
  1301. if response.StatusCode != http.StatusOK {
  1302. err = fmt.Errorf(
  1303. "unexpected response status code %d after %v",
  1304. response.StatusCode,
  1305. roundTripDuration)
  1306. // Depending on the round trip duration, this case is treated as a
  1307. // temporary round tripper failure, since we received a response
  1308. // from the CDN, secured with TLS and VerifyPins, or from broker
  1309. // itself. One common scenario is the CDN returning a temporary
  1310. // timeout error, as can happen when CDN timeouts and broker
  1311. // timeouts are misaligned, especially for long-polling requests.
  1312. //
  1313. // In this scenario, we can reuse the existing round tripper and
  1314. // it may be counterproductive to return a RoundTripperFailedError
  1315. // which will trigger a clearing of any broker dial replay
  1316. // parameters as well as reseting the round tripper.
  1317. //
  1318. // When the round trip duration is sufficiently short, much
  1319. // shorter than expected round trip timeouts, this is still
  1320. // classified as a RoundTripperFailedError error, as it is more
  1321. // likely due to a more serious issue between the CDN and broker.
  1322. if rt.failureThreshold > 0 &&
  1323. roundTripDuration <= rt.failureThreshold {
  1324. err = inproxy.NewRoundTripperFailedError(err)
  1325. }
  1326. }
  1327. } else if ctx.Err() != context.Canceled {
  1328. // Other round trip errors, including TLS failures and client-side
  1329. // timeouts, but excluding a cancelled context as happens on
  1330. // shutdown, are classified as RoundTripperFailedErrors.
  1331. err = inproxy.NewRoundTripperFailedError(err)
  1332. }
  1333. if err != nil {
  1334. return nil, errors.Trace(err)
  1335. }
  1336. responsePayload, err := io.ReadAll(response.Body)
  1337. if err != nil {
  1338. err = inproxy.NewRoundTripperFailedError(err)
  1339. return nil, errors.Trace(err)
  1340. }
  1341. return responsePayload, nil
  1342. }
  1343. // InproxyWebRTCDialInstance is the network state and dial parameters for a
  1344. // single WebRTC client or proxy connection.
  1345. //
  1346. // InproxyWebRTCDialInstance implements the inproxy.WebRTCDialCoordinator
  1347. // interface, which provides the WebRTC dial configuration and support to the
  1348. // in-proxy package.
  1349. type InproxyWebRTCDialInstance struct {
  1350. config *Config
  1351. networkID string
  1352. natStateManager *InproxyNATStateManager
  1353. stunDialParameters *InproxySTUNDialParameters
  1354. webRTCDialParameters *InproxyWebRTCDialParameters
  1355. discoverNAT bool
  1356. disableSTUN bool
  1357. disablePortMapping bool
  1358. disableInboundForMobileNetworks bool
  1359. disableIPv6ICECandidates bool
  1360. discoverNATTimeout time.Duration
  1361. webRTCAnswerTimeout time.Duration
  1362. webRTCAwaitPortMappingTimeout time.Duration
  1363. awaitReadyToProxyTimeout time.Duration
  1364. proxyDestinationDialTimeout time.Duration
  1365. proxyRelayInactivityTimeout time.Duration
  1366. }
  1367. // NewInproxyWebRTCDialInstance creates a new InproxyWebRTCDialInstance.
  1368. //
  1369. // The caller provides STUN and WebRTC dial parameters that are either newly
  1370. // generated or replayed. Proxies may optionally pass in nil for either
  1371. // stunDialParameters or webRTCDialParameters, and new parameters will be
  1372. // generated.
  1373. func NewInproxyWebRTCDialInstance(
  1374. config *Config,
  1375. networkID string,
  1376. isProxy bool,
  1377. natStateManager *InproxyNATStateManager,
  1378. stunDialParameters *InproxySTUNDialParameters,
  1379. webRTCDialParameters *InproxyWebRTCDialParameters) (*InproxyWebRTCDialInstance, error) {
  1380. p := config.GetParameters().Get()
  1381. defer p.Close()
  1382. if isProxy && stunDialParameters == nil {
  1383. // Auto-generate STUN dial parameters. There's no replay in this case.
  1384. var err error
  1385. stunDialParameters, err = MakeInproxySTUNDialParameters(config, p, isProxy)
  1386. if err != nil {
  1387. return nil, errors.Trace(err)
  1388. }
  1389. }
  1390. if isProxy && webRTCDialParameters == nil {
  1391. // Auto-generate STUN dial parameters. There's no replay in this case.
  1392. var err error
  1393. webRTCDialParameters, err = MakeInproxyWebRTCDialParameters(p)
  1394. if err != nil {
  1395. return nil, errors.Trace(err)
  1396. }
  1397. }
  1398. disableSTUN := p.Bool(parameters.InproxyDisableSTUN)
  1399. disablePortMapping := p.Bool(parameters.InproxyDisablePortMapping)
  1400. disableInboundForMobileNetworks := p.Bool(parameters.InproxyDisableInboundForMobileNetworks)
  1401. disableIPv6ICECandidates := p.Bool(parameters.InproxyDisableIPv6ICECandidates)
  1402. var discoverNATTimeout, awaitReadyToProxyTimeout time.Duration
  1403. if isProxy {
  1404. disableSTUN = disableSTUN || p.Bool(parameters.InproxyProxyDisableSTUN)
  1405. disablePortMapping = disablePortMapping || p.Bool(parameters.InproxyProxyDisablePortMapping)
  1406. disableInboundForMobileNetworks = disableInboundForMobileNetworks ||
  1407. p.Bool(parameters.InproxyProxyDisableInboundForMobileNetworks)
  1408. disableIPv6ICECandidates = disableIPv6ICECandidates ||
  1409. p.Bool(parameters.InproxyProxyDisableIPv6ICECandidates)
  1410. discoverNATTimeout = p.Duration(parameters.InproxyProxyDiscoverNATTimeout)
  1411. awaitReadyToProxyTimeout = p.Duration(parameters.InproxyProxyWebRTCAwaitReadyToProxyTimeout)
  1412. } else {
  1413. disableSTUN = disableSTUN || p.Bool(parameters.InproxyClientDisableSTUN)
  1414. disablePortMapping = disablePortMapping || p.Bool(parameters.InproxyClientDisablePortMapping)
  1415. disableInboundForMobileNetworks = disableInboundForMobileNetworks ||
  1416. p.Bool(parameters.InproxyClientDisableInboundForMobileNetworks)
  1417. disableIPv6ICECandidates = disableIPv6ICECandidates ||
  1418. p.Bool(parameters.InproxyClientDisableIPv6ICECandidates)
  1419. discoverNATTimeout = p.Duration(parameters.InproxyClientDiscoverNATTimeout)
  1420. awaitReadyToProxyTimeout = p.Duration(parameters.InproxyClientWebRTCAwaitReadyToProxyTimeout)
  1421. }
  1422. if clientAPILevelDisableInproxyPortMapping.Load() {
  1423. disablePortMapping = true
  1424. }
  1425. // Parameters such as disabling certain operations and operation timeouts
  1426. // are not replayed, but snapshots are stored in the
  1427. // InproxyWebRTCDialInstance for efficient lookup.
  1428. return &InproxyWebRTCDialInstance{
  1429. config: config,
  1430. networkID: networkID,
  1431. natStateManager: natStateManager,
  1432. stunDialParameters: stunDialParameters,
  1433. webRTCDialParameters: webRTCDialParameters,
  1434. // discoverNAT is ignored by proxies, which always attempt discovery.
  1435. // webRTCAnswerTimeout, proxyDestinationDialTimeout, and
  1436. // proxyRelayInactivityTimeout are used only by proxies.
  1437. discoverNAT: p.WeightedCoinFlip(parameters.InproxyClientDiscoverNATProbability),
  1438. disableSTUN: disableSTUN,
  1439. disablePortMapping: disablePortMapping,
  1440. disableInboundForMobileNetworks: disableInboundForMobileNetworks,
  1441. disableIPv6ICECandidates: disableIPv6ICECandidates,
  1442. discoverNATTimeout: discoverNATTimeout,
  1443. webRTCAnswerTimeout: p.Duration(parameters.InproxyWebRTCAnswerTimeout),
  1444. webRTCAwaitPortMappingTimeout: p.Duration(parameters.InproxyWebRTCAwaitPortMappingTimeout),
  1445. awaitReadyToProxyTimeout: awaitReadyToProxyTimeout,
  1446. proxyDestinationDialTimeout: p.Duration(parameters.InproxyProxyDestinationDialTimeout),
  1447. proxyRelayInactivityTimeout: p.Duration(parameters.InproxyProxyRelayInactivityTimeout),
  1448. }, nil
  1449. }
  1450. // Implements the inproxy.WebRTCDialCoordinator interface.
  1451. func (w *InproxyWebRTCDialInstance) NetworkID() string {
  1452. return w.networkID
  1453. }
  1454. // Implements the inproxy.WebRTCDialCoordinator interface.
  1455. func (w *InproxyWebRTCDialInstance) NetworkType() inproxy.NetworkType {
  1456. return getInproxyNetworkType(GetNetworkType(w.networkID))
  1457. }
  1458. // Implements the inproxy.WebRTCDialCoordinator interface.
  1459. func (w *InproxyWebRTCDialInstance) ClientRootObfuscationSecret() inproxy.ObfuscationSecret {
  1460. return w.webRTCDialParameters.RootObfuscationSecret
  1461. }
  1462. // Implements the inproxy.WebRTCDialCoordinator interface.
  1463. func (w *InproxyWebRTCDialInstance) DoDTLSRandomization() bool {
  1464. return w.webRTCDialParameters.DoDTLSRandomization
  1465. }
  1466. // Implements the inproxy.WebRTCDialCoordinator interface.
  1467. func (w *InproxyWebRTCDialInstance) UseMediaStreams() bool {
  1468. return w.webRTCDialParameters.UseMediaStreams
  1469. }
  1470. // Implements the inproxy.WebRTCDialCoordinator interface.
  1471. func (w *InproxyWebRTCDialInstance) TrafficShapingParameters() *inproxy.TrafficShapingParameters {
  1472. return w.webRTCDialParameters.TrafficShapingParameters
  1473. }
  1474. // Implements the inproxy.WebRTCDialCoordinator interface.
  1475. func (w *InproxyWebRTCDialInstance) STUNServerAddress(RFC5780 bool) string {
  1476. if RFC5780 {
  1477. return w.stunDialParameters.STUNServerAddressRFC5780
  1478. } else {
  1479. return w.stunDialParameters.STUNServerAddress
  1480. }
  1481. }
  1482. // Implements the inproxy.WebRTCDialCoordinator interface.
  1483. func (w *InproxyWebRTCDialInstance) STUNServerAddressResolved(RFC5780 bool) string {
  1484. if RFC5780 {
  1485. return w.stunDialParameters.STUNServerAddressRFC5780
  1486. } else {
  1487. return w.stunDialParameters.STUNServerAddress
  1488. }
  1489. }
  1490. // Implements the inproxy.WebRTCDialCoordinator interface.
  1491. func (w *InproxyWebRTCDialInstance) STUNServerAddressSucceeded(RFC5780 bool, address string) {
  1492. // Currently, for client tunnel dials, STUN dial parameter replay is
  1493. // managed by DialParameters and DialParameters.InproxySTUNDialParameters
  1494. // are replayed only when the entire dial succeeds.
  1495. //
  1496. // Note that, for a client tunnel dial, even if the STUN step fails and
  1497. // there are no STUN ICE candidates, the subsequent WebRTC connection may
  1498. // still proceed and be successful. In this case, the failed STUN dial
  1499. // parameters may be replayed.
  1500. //
  1501. // For proxies, there is no STUN dial parameter replay.
  1502. //
  1503. // As a future enhancement, consider independent and shared replay of
  1504. // working STUN servers, similar to how broker client dial parameters are
  1505. // replayed independent of overall dials and proxy relays, and shared
  1506. // between local client and proxy instances.
  1507. // Verify/extend the resolver cache entry for any resolved domain after a
  1508. // success.
  1509. resolver := w.config.GetResolver()
  1510. if resolver != nil {
  1511. resolver.VerifyCacheExtension(address)
  1512. }
  1513. }
  1514. // Implements the inproxy.WebRTCDialCoordinator interface.
  1515. func (w *InproxyWebRTCDialInstance) STUNServerAddressFailed(RFC5780 bool, address string) {
  1516. // Currently there is no independent replay for STUN dial parameters. See
  1517. // comment in STUNServerAddressSucceeded.
  1518. }
  1519. // Implements the inproxy.WebRTCDialCoordinator interface.
  1520. func (w *InproxyWebRTCDialInstance) DiscoverNAT() bool {
  1521. return w.discoverNAT
  1522. }
  1523. // Implements the inproxy.WebRTCDialCoordinator interface.
  1524. func (w *InproxyWebRTCDialInstance) DisableSTUN() bool {
  1525. return w.disableSTUN
  1526. }
  1527. // Implements the inproxy.WebRTCDialCoordinator interface.
  1528. func (w *InproxyWebRTCDialInstance) DisablePortMapping() bool {
  1529. return w.disablePortMapping
  1530. }
  1531. // Implements the inproxy.WebRTCDialCoordinator interface.
  1532. func (w *InproxyWebRTCDialInstance) DisableInboundForMobileNetworks() bool {
  1533. return w.disableInboundForMobileNetworks
  1534. }
  1535. // Implements the inproxy.WebRTCDialCoordinator interface.
  1536. func (w *InproxyWebRTCDialInstance) DisableIPv6ICECandidates() bool {
  1537. return w.disableIPv6ICECandidates
  1538. }
  1539. // Implements the inproxy.WebRTCDialCoordinator interface.
  1540. func (w *InproxyWebRTCDialInstance) NATType() inproxy.NATType {
  1541. return w.natStateManager.getNATType(w.networkID)
  1542. }
  1543. // Implements the inproxy.WebRTCDialCoordinator interface.
  1544. func (w *InproxyWebRTCDialInstance) SetNATType(natType inproxy.NATType) {
  1545. w.natStateManager.setNATType(w.networkID, natType)
  1546. }
  1547. // Implements the inproxy.WebRTCDialCoordinator interface.
  1548. func (w *InproxyWebRTCDialInstance) PortMappingTypes() inproxy.PortMappingTypes {
  1549. return w.natStateManager.getPortMappingTypes(w.networkID)
  1550. }
  1551. // Implements the inproxy.WebRTCDialCoordinator interface.
  1552. func (w *InproxyWebRTCDialInstance) SetPortMappingTypes(
  1553. portMappingTypes inproxy.PortMappingTypes) {
  1554. w.natStateManager.setPortMappingTypes(w.networkID, portMappingTypes)
  1555. }
  1556. // Implements the inproxy.WebRTCDialCoordinator interface.
  1557. func (w *InproxyWebRTCDialInstance) PortMappingProbe() *inproxy.PortMappingProbe {
  1558. return w.natStateManager.getPortMappingProbe(w.networkID)
  1559. }
  1560. // Implements the inproxy.WebRTCDialCoordinator interface.
  1561. func (w *InproxyWebRTCDialInstance) SetPortMappingProbe(
  1562. portMappingProbe *inproxy.PortMappingProbe) {
  1563. w.natStateManager.setPortMappingProbe(w.networkID, portMappingProbe)
  1564. }
  1565. // Implements the inproxy.WebRTCDialCoordinator interface.
  1566. func (w *InproxyWebRTCDialInstance) ResolveAddress(ctx context.Context, network, address string) (string, error) {
  1567. // Use the Psiphon resolver to resolve addresses.
  1568. r := w.config.GetResolver()
  1569. if r == nil {
  1570. return "", errors.TraceNew("missing resolver")
  1571. }
  1572. // Identify when the address to be resolved is one of the configured STUN
  1573. // servers, and, in those cases, use/replay any STUN dial parameters
  1574. // ResolveParameters; and record the resolved IP address for metrics.
  1575. //
  1576. // In the in-proxy proxy case, ResolveAddress is invoked for the upstream,
  1577. // 2nd hop dial as well as for STUN server addresses.
  1578. //
  1579. // Limitation: there's no ResolveParameters, including no preresolved DNS
  1580. // tactics, for 2nd hop dials.
  1581. isSTUNServerAddress := address == w.stunDialParameters.STUNServerAddress
  1582. isSTUNServerAddressRFC5780 := address == w.stunDialParameters.STUNServerAddressRFC5780
  1583. var resolveParams *resolver.ResolveParameters
  1584. if isSTUNServerAddress || isSTUNServerAddressRFC5780 {
  1585. resolveParams = w.stunDialParameters.ResolveParameters
  1586. }
  1587. resolved, err := r.ResolveAddress(
  1588. ctx, w.networkID, resolveParams, network, address)
  1589. if err != nil {
  1590. return "", errors.Trace(err)
  1591. }
  1592. // Invoke the resolved IP callbacks only when the input is not the
  1593. // resolved IP address (this differs from the meek
  1594. // DialConfig.ResolvedIPCallback case).
  1595. if resolved != address {
  1596. if isSTUNServerAddress {
  1597. w.stunDialParameters.STUNServerResolvedIPAddress.Store(resolved)
  1598. } else if isSTUNServerAddressRFC5780 {
  1599. w.stunDialParameters.STUNServerRFC5780ResolvedIPAddress.Store(resolved)
  1600. }
  1601. }
  1602. return resolved, nil
  1603. }
  1604. // Implements the inproxy.WebRTCDialCoordinator interface.
  1605. func (w *InproxyWebRTCDialInstance) UDPListen(ctx context.Context) (net.PacketConn, error) {
  1606. // Create a new inproxyUDPConn for use as the in-proxy STUN and/ord WebRTC
  1607. // UDP socket.
  1608. conn, err := newInproxyUDPConn(ctx, w.config)
  1609. if err != nil {
  1610. return nil, errors.Trace(err)
  1611. }
  1612. return conn, nil
  1613. }
  1614. // Implements the inproxy.WebRTCDialCoordinator interface.
  1615. func (w *InproxyWebRTCDialInstance) UDPConn(
  1616. ctx context.Context, network, remoteAddress string) (net.PacketConn, error) {
  1617. // Create a new UDPConn bound to the specified remote address. This UDP
  1618. // conn is used, by the inproxy package, to determine the local address
  1619. // of the active interface the OS will select for the specified remote
  1620. // destination.
  1621. //
  1622. // Only IP address destinations are supported. ResolveIP is wired up only
  1623. // because NewUDPConn requires a non-nil resolver.
  1624. dialConfig := &DialConfig{
  1625. DeviceBinder: w.config.deviceBinder,
  1626. IPv6Synthesizer: w.config.IPv6Synthesizer,
  1627. ResolveIP: func(_ context.Context, hostname string) ([]net.IP, error) {
  1628. IP := net.ParseIP(hostname)
  1629. if IP == nil {
  1630. return nil, errors.TraceNew("not supported")
  1631. }
  1632. return []net.IP{IP}, nil
  1633. },
  1634. }
  1635. conn, _, err := NewUDPConn(ctx, network, true, "", remoteAddress, dialConfig)
  1636. if err != nil {
  1637. return nil, errors.Trace(err)
  1638. }
  1639. return conn, nil
  1640. }
  1641. // Implements the inproxy.WebRTCDialCoordinator interface.
  1642. func (w *InproxyWebRTCDialInstance) BindToDevice(fileDescriptor int) error {
  1643. if w.config.deviceBinder == nil {
  1644. return nil
  1645. }
  1646. // Use config.deviceBinder, with wired up logging, not
  1647. // config.DeviceBinder; other tunnel-core dials do this indirectly via
  1648. // psiphon.DialConfig.
  1649. _, err := w.config.deviceBinder.BindToDevice(fileDescriptor)
  1650. return errors.Trace(err)
  1651. }
  1652. func (w *InproxyWebRTCDialInstance) ProxyUpstreamDial(
  1653. ctx context.Context, network, address string) (net.Conn, error) {
  1654. // This implementation of ProxyUpstreamDial applies additional socket
  1655. // options and BindToDevice as required, but is otherwise a stock dialer.
  1656. //
  1657. // TODO: Use custom UDP and TCP dialers, and wire up TCP/UDP-level
  1658. // tactics, including BPF and the custom resolver, which may be enabled
  1659. // for the proxy's ISP or geolocation. Orchestrating preresolved DNS
  1660. // requires additional information from either from the broker, the
  1661. // FrontingProviderID, to be applied to any
  1662. // DNSResolverPreresolvedIPAddressCIDRs proxy tactics. In addition,
  1663. // replay the selected upstream dial tactics parameters.
  1664. dialer := net.Dialer{
  1665. Control: func(_, _ string, c syscall.RawConn) error {
  1666. var controlErr error
  1667. err := c.Control(func(fd uintptr) {
  1668. socketFD := int(fd)
  1669. setAdditionalSocketOptions(socketFD)
  1670. if w.config.deviceBinder != nil {
  1671. _, err := w.config.deviceBinder.BindToDevice(socketFD)
  1672. if err != nil {
  1673. controlErr = errors.Tracef("BindToDevice failed: %s", err)
  1674. return
  1675. }
  1676. }
  1677. })
  1678. if controlErr != nil {
  1679. return errors.Trace(controlErr)
  1680. }
  1681. return errors.Trace(err)
  1682. },
  1683. }
  1684. conn, err := dialer.DialContext(ctx, network, address)
  1685. if err != nil {
  1686. return nil, errors.Trace(err)
  1687. }
  1688. return conn, nil
  1689. }
  1690. // Implements the inproxy.WebRTCDialCoordinator interface.
  1691. func (w *InproxyWebRTCDialInstance) DiscoverNATTimeout() time.Duration {
  1692. return w.discoverNATTimeout
  1693. }
  1694. // Implements the inproxy.WebRTCDialCoordinator interface.
  1695. func (w *InproxyWebRTCDialInstance) WebRTCAnswerTimeout() time.Duration {
  1696. return w.webRTCAnswerTimeout
  1697. }
  1698. // Implements the inproxy.WebRTCDialCoordinator interface.
  1699. func (w *InproxyWebRTCDialInstance) WebRTCAwaitPortMappingTimeout() time.Duration {
  1700. return w.webRTCAwaitPortMappingTimeout
  1701. }
  1702. // Implements the inproxy.WebRTCDialCoordinator interface.
  1703. func (w *InproxyWebRTCDialInstance) WebRTCAwaitReadyToProxyTimeout() time.Duration {
  1704. return w.awaitReadyToProxyTimeout
  1705. }
  1706. // Implements the inproxy.WebRTCDialCoordinator interface.
  1707. func (w *InproxyWebRTCDialInstance) ProxyDestinationDialTimeout() time.Duration {
  1708. return w.proxyDestinationDialTimeout
  1709. }
  1710. // Implements the inproxy.WebRTCDialCoordinator interface.
  1711. func (w *InproxyWebRTCDialInstance) ProxyRelayInactivityTimeout() time.Duration {
  1712. return w.proxyRelayInactivityTimeout
  1713. }
  1714. // InproxySTUNDialParameters is a set of STUN dial parameters.
  1715. // InproxySTUNDialParameters is compatible with DialParameters JSON
  1716. // marshaling. For client in-proxy tunnel dials, DialParameters will manage
  1717. // STUN dial parameter selection and replay.
  1718. //
  1719. // When an instance of InproxySTUNDialParameters is unmarshaled from JSON,
  1720. // Prepare must be called to initialize the instance for use.
  1721. type InproxySTUNDialParameters struct {
  1722. ResolveParameters *resolver.ResolveParameters
  1723. STUNServerAddress string
  1724. STUNServerAddressRFC5780 string
  1725. STUNServerResolvedIPAddress atomic.Value `json:"-"`
  1726. STUNServerRFC5780ResolvedIPAddress atomic.Value `json:"-"`
  1727. }
  1728. // MakeInproxySTUNDialParameters generates new STUN dial parameters from the
  1729. // given tactics parameters.
  1730. func MakeInproxySTUNDialParameters(
  1731. config *Config,
  1732. p parameters.ParametersAccessor,
  1733. isProxy bool) (*InproxySTUNDialParameters, error) {
  1734. var stunServerAddresses, stunServerAddressesRFC5780 []string
  1735. if isProxy {
  1736. stunServerAddresses = p.Strings(
  1737. parameters.InproxyProxySTUNServerAddresses, parameters.InproxySTUNServerAddresses)
  1738. stunServerAddressesRFC5780 = p.Strings(
  1739. parameters.InproxyProxySTUNServerAddressesRFC5780, parameters.InproxySTUNServerAddressesRFC5780)
  1740. } else {
  1741. stunServerAddresses = p.Strings(
  1742. parameters.InproxyClientSTUNServerAddresses, parameters.InproxySTUNServerAddresses)
  1743. stunServerAddressesRFC5780 = p.Strings(
  1744. parameters.InproxyClientSTUNServerAddressesRFC5780, parameters.InproxySTUNServerAddressesRFC5780)
  1745. }
  1746. // Empty STUN server address lists are not an error condition. When used
  1747. // for WebRTC, the STUN ICE candidate gathering will be skipped but the
  1748. // WebRTC connection may still be established via other candidate types.
  1749. var stunServerAddress, stunServerAddressRFC5780 string
  1750. if len(stunServerAddresses) > 0 {
  1751. stunServerAddress = stunServerAddresses[prng.Range(0, len(stunServerAddresses)-1)]
  1752. }
  1753. if len(stunServerAddressesRFC5780) > 0 {
  1754. stunServerAddressRFC5780 =
  1755. stunServerAddressesRFC5780[prng.Range(0, len(stunServerAddressesRFC5780)-1)]
  1756. }
  1757. // Create DNS resolver dial parameters to use when resolving STUN server
  1758. // domain addresses. Instantiate only when there is a domain to be
  1759. // resolved; when recording DNS fields, GetMetrics will assume that a nil
  1760. // InproxySTUNDialParameters.ResolveParameters implies no resolve was
  1761. // attempted.
  1762. var resolveParameters *resolver.ResolveParameters
  1763. if (stunServerAddress != "" && net.ParseIP(stunServerAddress) == nil) ||
  1764. (stunServerAddressRFC5780 != "" && net.ParseIP(stunServerAddressRFC5780) == nil) {
  1765. // No DNSResolverPreresolvedIPAddressCIDRs will be selected since no
  1766. // fronting provider ID is specified.
  1767. //
  1768. // It would be possible to overload the meaning of the fronting
  1769. // provider ID field by using a string derived from STUN server
  1770. // address as the key.
  1771. //
  1772. // However, preresolved STUN configuration can already be achieved
  1773. // with IP addresses in the STUNServerAddresses tactics parameters.
  1774. // This approach results in slightly different metrics log fields vs.
  1775. // preresolved.
  1776. var err error
  1777. resolveParameters, err = config.GetResolver().MakeResolveParameters(p, "", "")
  1778. if err != nil {
  1779. return nil, errors.Trace(err)
  1780. }
  1781. }
  1782. dialParams := &InproxySTUNDialParameters{
  1783. ResolveParameters: resolveParameters,
  1784. STUNServerAddress: stunServerAddress,
  1785. STUNServerAddressRFC5780: stunServerAddressRFC5780,
  1786. }
  1787. dialParams.Prepare()
  1788. return dialParams, nil
  1789. }
  1790. // Prepare initializes an InproxySTUNDialParameters for use. Prepare should be
  1791. // called for any InproxySTUNDialParameters instance unmarshaled from JSON.
  1792. func (dialParams *InproxySTUNDialParameters) Prepare() {
  1793. dialParams.STUNServerResolvedIPAddress.Store("")
  1794. dialParams.STUNServerRFC5780ResolvedIPAddress.Store("")
  1795. }
  1796. // IsValidClientReplay checks that the selected STUN servers remain configured
  1797. // STUN server candidates for in-proxy clients.
  1798. func (dialParams *InproxySTUNDialParameters) IsValidClientReplay(
  1799. p parameters.ParametersAccessor) bool {
  1800. return (dialParams.STUNServerAddress == "" ||
  1801. common.Contains(
  1802. p.Strings(parameters.InproxyClientSTUNServerAddresses),
  1803. dialParams.STUNServerAddress)) &&
  1804. (dialParams.STUNServerAddressRFC5780 == "" ||
  1805. common.Contains(
  1806. p.Strings(parameters.InproxyClientSTUNServerAddressesRFC5780),
  1807. dialParams.STUNServerAddressRFC5780))
  1808. }
  1809. // GetMetrics implements the common.MetricsSource interface and returns log
  1810. // fields detailing the STUN dial parameters.
  1811. func (dialParams *InproxySTUNDialParameters) GetMetrics() common.LogFields {
  1812. // There is no is_replay-type field added here; replay is handled at a
  1813. // higher level, and, for client in-proxy tunnel dials, is part of the
  1814. // main tunnel dial parameters.
  1815. logFields := make(common.LogFields)
  1816. logFields["inproxy_webrtc_stun_server"] = dialParams.STUNServerAddress
  1817. resolvedIPAddress := dialParams.STUNServerResolvedIPAddress.Load().(string)
  1818. if resolvedIPAddress != "" {
  1819. logFields["inproxy_webrtc_stun_server_resolved_ip_address"] = resolvedIPAddress
  1820. }
  1821. // TODO: log RFC5780 selection only if used?
  1822. logFields["inproxy_webrtc_stun_server_RFC5780"] = dialParams.STUNServerAddressRFC5780
  1823. resolvedIPAddress = dialParams.STUNServerRFC5780ResolvedIPAddress.Load().(string)
  1824. if resolvedIPAddress != "" {
  1825. logFields["inproxy_webrtc_stun_server_RFC5780_resolved_ip_address"] = resolvedIPAddress
  1826. }
  1827. if dialParams.ResolveParameters != nil {
  1828. // See comment in getBaseAPIParameters regarding
  1829. // dialParams.ResolveParameters handling. As noted in
  1830. // MakeInproxySTUNDialParameters, no preresolved parameters are set,
  1831. // so none are checked for logging.
  1832. //
  1833. // Limitation: the potential use of single ResolveParameters to
  1834. // resolve multiple, different STUN server domains can skew the
  1835. // meaning of GetFirstAttemptWithAnswer.
  1836. if dialParams.ResolveParameters.PreferAlternateDNSServer {
  1837. logFields["inproxy_webrtc_dns_preferred"] = dialParams.ResolveParameters.AlternateDNSServer
  1838. }
  1839. if dialParams.ResolveParameters.ProtocolTransformName != "" {
  1840. logFields["inproxy_webrtc_dns_transform"] = dialParams.ResolveParameters.ProtocolTransformName
  1841. }
  1842. if dialParams.ResolveParameters.RandomQNameCasingSeed != nil {
  1843. logFields["inproxy_webrtc_dns_qname_random_casing"] = "1"
  1844. }
  1845. if dialParams.ResolveParameters.ResponseQNameMustMatch {
  1846. logFields["inproxy_webrtc_dns_qname_must_match"] = "1"
  1847. }
  1848. logFields["inproxy_webrtc_dns_qname_mismatches"] = strconv.Itoa(
  1849. dialParams.ResolveParameters.GetQNameMismatches())
  1850. logFields["inproxy_webrtc_dns_attempt"] = strconv.Itoa(
  1851. dialParams.ResolveParameters.GetFirstAttemptWithAnswer())
  1852. }
  1853. return logFields
  1854. }
  1855. // InproxyWebRTCDialParameters is a set of WebRTC obfuscation dial parameters.
  1856. // InproxyWebRTCDialParameters is compatible with DialParameters JSON
  1857. // marshaling. For client in-proxy tunnel dials, DialParameters will manage
  1858. // WebRTC dial parameter selection and replay.
  1859. type InproxyWebRTCDialParameters struct {
  1860. RootObfuscationSecret inproxy.ObfuscationSecret
  1861. UseMediaStreams bool
  1862. TrafficShapingParameters *inproxy.TrafficShapingParameters
  1863. DoDTLSRandomization bool
  1864. }
  1865. // MakeInproxyWebRTCDialParameters generates new InproxyWebRTCDialParameters.
  1866. func MakeInproxyWebRTCDialParameters(
  1867. p parameters.ParametersAccessor) (*InproxyWebRTCDialParameters, error) {
  1868. rootObfuscationSecret, err := inproxy.GenerateRootObfuscationSecret()
  1869. if err != nil {
  1870. return nil, errors.Trace(err)
  1871. }
  1872. useMediaStreams := p.WeightedCoinFlip(parameters.InproxyWebRTCMediaStreamsProbability)
  1873. var trafficSharingParams *inproxy.TrafficShapingParameters
  1874. if useMediaStreams {
  1875. if p.WeightedCoinFlip(parameters.InproxyWebRTCMediaStreamsTrafficShapingProbability) {
  1876. t := inproxy.TrafficShapingParameters(
  1877. p.InproxyTrafficShapingParameters(
  1878. parameters.InproxyWebRTCMediaStreamsTrafficShapingParameters))
  1879. trafficSharingParams = &t
  1880. }
  1881. } else {
  1882. if p.WeightedCoinFlip(parameters.InproxyWebRTCDataChannelTrafficShapingProbability) {
  1883. t := inproxy.TrafficShapingParameters(
  1884. p.InproxyTrafficShapingParameters(
  1885. parameters.InproxyWebRTCDataChannelTrafficShapingParameters))
  1886. trafficSharingParams = &t
  1887. }
  1888. }
  1889. doDTLSRandomization := p.WeightedCoinFlip(parameters.InproxyDTLSRandomizationProbability)
  1890. return &InproxyWebRTCDialParameters{
  1891. RootObfuscationSecret: rootObfuscationSecret,
  1892. UseMediaStreams: useMediaStreams,
  1893. TrafficShapingParameters: trafficSharingParams,
  1894. DoDTLSRandomization: doDTLSRandomization,
  1895. }, nil
  1896. }
  1897. // GetMetrics implements the common.MetricsSource interface.
  1898. func (dialParams *InproxyWebRTCDialParameters) GetMetrics() common.LogFields {
  1899. // There is no is_replay-type field added here; replay is handled at a
  1900. // higher level, and, for client in-proxy tunnel dials, is part of the
  1901. // main tunnel dial parameters.
  1902. // Currently, all WebRTC metrics are delivered via
  1903. // inproxy.ClientConn/WebRTCConn GetMetrics.
  1904. return common.LogFields{}
  1905. }
  1906. // InproxyNATStateManager manages the NAT-related network topology state for
  1907. // the current network, caching the discovered network NAT type and supported
  1908. // port mapping types, if any.
  1909. type InproxyNATStateManager struct {
  1910. config *Config
  1911. mutex sync.Mutex
  1912. networkID string
  1913. natType inproxy.NATType
  1914. portMappingTypes inproxy.PortMappingTypes
  1915. portMappingProbe *inproxy.PortMappingProbe
  1916. }
  1917. // NewInproxyNATStateManager creates a new InproxyNATStateManager.
  1918. func NewInproxyNATStateManager(config *Config) *InproxyNATStateManager {
  1919. s := &InproxyNATStateManager{
  1920. config: config,
  1921. natType: inproxy.NATTypeUnknown,
  1922. portMappingTypes: inproxy.PortMappingTypes{},
  1923. }
  1924. s.reset()
  1925. return s
  1926. }
  1927. // TacticsApplied implements the TacticsAppliedReceiver interface, and is
  1928. // called when tactics have changed, which triggers a cached NAT state reset
  1929. // in order to apply potentially changed parameters.
  1930. func (s *InproxyNATStateManager) TacticsApplied() error {
  1931. s.reset()
  1932. return nil
  1933. }
  1934. func (s *InproxyNATStateManager) reset() {
  1935. s.mutex.Lock()
  1936. defer s.mutex.Unlock()
  1937. networkID := s.config.GetNetworkID()
  1938. s.networkID = networkID
  1939. s.natType = inproxy.NATTypeUnknown
  1940. s.portMappingTypes = inproxy.PortMappingTypes{}
  1941. }
  1942. func (s *InproxyNATStateManager) getNATType(
  1943. networkID string) inproxy.NATType {
  1944. s.mutex.Lock()
  1945. defer s.mutex.Unlock()
  1946. if s.networkID != networkID {
  1947. return inproxy.NATTypeUnknown
  1948. }
  1949. return s.natType
  1950. }
  1951. func (s *InproxyNATStateManager) setNATType(
  1952. networkID string, natType inproxy.NATType) {
  1953. s.mutex.Lock()
  1954. defer s.mutex.Unlock()
  1955. if s.networkID != networkID {
  1956. return
  1957. }
  1958. s.natType = natType
  1959. }
  1960. func (s *InproxyNATStateManager) getPortMappingTypes(
  1961. networkID string) inproxy.PortMappingTypes {
  1962. s.mutex.Lock()
  1963. defer s.mutex.Unlock()
  1964. if s.networkID != networkID {
  1965. return inproxy.PortMappingTypes{}
  1966. }
  1967. return s.portMappingTypes
  1968. }
  1969. func (s *InproxyNATStateManager) setPortMappingTypes(
  1970. networkID string,
  1971. portMappingTypes inproxy.PortMappingTypes) {
  1972. s.mutex.Lock()
  1973. defer s.mutex.Unlock()
  1974. if s.networkID != networkID {
  1975. return
  1976. }
  1977. s.portMappingTypes = portMappingTypes
  1978. }
  1979. func (s *InproxyNATStateManager) getPortMappingProbe(
  1980. networkID string) *inproxy.PortMappingProbe {
  1981. s.mutex.Lock()
  1982. defer s.mutex.Unlock()
  1983. if s.networkID != networkID {
  1984. return nil
  1985. }
  1986. return s.portMappingProbe
  1987. }
  1988. func (s *InproxyNATStateManager) setPortMappingProbe(
  1989. networkID string,
  1990. portMappingProbe *inproxy.PortMappingProbe) {
  1991. s.mutex.Lock()
  1992. defer s.mutex.Unlock()
  1993. if s.networkID != networkID {
  1994. return
  1995. }
  1996. s.portMappingProbe = portMappingProbe
  1997. }
  1998. // inproxyUDPConn is based on NewUDPConn and includes the write timeout
  1999. // workaround from common.WriteTimeoutUDPConn.
  2000. //
  2001. // inproxyUDPConn expands the NewUDPConn IPv6Synthesizer to support many
  2002. // destination addresses, as the inproxyUDPConn will be used to send/receive
  2003. // packets between many remote destination addresses.
  2004. //
  2005. // inproxyUDPConn implements the net.PacketConn interface.
  2006. type inproxyUDPConn struct {
  2007. udpConn *net.UDPConn
  2008. ipv6Synthesizer IPv6Synthesizer
  2009. synthesizerMutex sync.Mutex
  2010. ipv4ToIPv6 map[netip.Addr]net.IP
  2011. ipv6ToIPv4 map[netip.Addr]net.IP
  2012. }
  2013. func newInproxyUDPConn(ctx context.Context, config *Config) (net.PacketConn, error) {
  2014. listen := &net.ListenConfig{
  2015. Control: func(_, _ string, c syscall.RawConn) error {
  2016. var controlErr error
  2017. err := c.Control(func(fd uintptr) {
  2018. socketFD := int(fd)
  2019. setAdditionalSocketOptions(socketFD)
  2020. // Use config.deviceBinder, with wired up logging, not
  2021. // config.DeviceBinder; other tunnel-core dials do this
  2022. // indirectly via psiphon.DialConfig.
  2023. if config.deviceBinder != nil {
  2024. _, err := config.deviceBinder.BindToDevice(socketFD)
  2025. if err != nil {
  2026. controlErr = errors.Tracef("BindToDevice failed: %s", err)
  2027. return
  2028. }
  2029. }
  2030. })
  2031. if controlErr != nil {
  2032. return errors.Trace(controlErr)
  2033. }
  2034. return errors.Trace(err)
  2035. },
  2036. }
  2037. // Create an "unconnected" UDP socket for use with WriteTo and listening
  2038. // on all interfaces. See the limitation comment in NewUDPConn regarding
  2039. // its equivilent mode.
  2040. packetConn, err := listen.ListenPacket(ctx, "udp", "")
  2041. if err != nil {
  2042. return nil, errors.Trace(err)
  2043. }
  2044. var ok bool
  2045. udpConn, ok := packetConn.(*net.UDPConn)
  2046. if !ok {
  2047. return nil, errors.Tracef("unexpected conn type: %T", packetConn)
  2048. }
  2049. conn := &inproxyUDPConn{
  2050. udpConn: udpConn,
  2051. ipv6Synthesizer: config.IPv6Synthesizer,
  2052. }
  2053. if conn.ipv6Synthesizer != nil {
  2054. conn.ipv4ToIPv6 = make(map[netip.Addr]net.IP)
  2055. conn.ipv6ToIPv4 = make(map[netip.Addr]net.IP)
  2056. }
  2057. return conn, nil
  2058. }
  2059. func (conn *inproxyUDPConn) ReadFrom(p []byte) (int, net.Addr, error) {
  2060. // net.UDPConn.ReadFrom currently allocates a &UDPAddr{} per call, and so
  2061. // the &net.UDPAddr{} allocations done in the following synthesizer code
  2062. // path are no more than the standard code path.
  2063. //
  2064. // TODO: avoid all address allocations in both ReadFrom and WriteTo by:
  2065. //
  2066. // - changing ipvXToIPvY to map[netip.AddrPort]*net.UDPAddr
  2067. // - using a similar lookup for the non-synthesizer code path
  2068. //
  2069. // Such a scheme would work only if the caller is guaranteed to not mutate
  2070. // the returned net.Addr.
  2071. if conn.ipv6Synthesizer == nil {
  2072. // Do not wrap any I/O err returned by UDPConn
  2073. return conn.udpConn.ReadFrom(p)
  2074. }
  2075. n, addrPort, err := conn.udpConn.ReadFromUDPAddrPort(p)
  2076. // Reverse any synthesized address before returning err.
  2077. // Reverse the IPv6 synthesizer, returning the original IPv4 address
  2078. // as expected by the caller, including pion/webrtc. This logic
  2079. // assumes that no synthesized IPv6 address will conflict with any
  2080. // real IPv6 address.
  2081. var IP net.IP
  2082. ipAddr := addrPort.Addr()
  2083. if ipAddr.Is6() {
  2084. conn.synthesizerMutex.Lock()
  2085. IP, _ = conn.ipv6ToIPv4[ipAddr]
  2086. conn.synthesizerMutex.Unlock()
  2087. }
  2088. if IP == nil {
  2089. IP = ipAddr.AsSlice()
  2090. }
  2091. // Do not wrap any I/O err returned by UDPConn
  2092. return n, &net.UDPAddr{IP: IP, Port: int(addrPort.Port())}, err
  2093. }
  2094. func (conn *inproxyUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) {
  2095. // See common.WriteTimeoutUDPConn.
  2096. err := conn.udpConn.SetWriteDeadline(
  2097. time.Now().Add(common.UDP_PACKET_WRITE_TIMEOUT))
  2098. if err != nil {
  2099. return 0, errors.Trace(err)
  2100. }
  2101. if conn.ipv6Synthesizer == nil {
  2102. // Do not wrap any I/O err returned by UDPConn
  2103. return conn.udpConn.WriteTo(b, addr)
  2104. }
  2105. // When configured, attempt to synthesize IPv6 addresses from an IPv4
  2106. // addresses for compatibility on DNS64/NAT64 networks.
  2107. //
  2108. // Store any synthesized addresses in a lookup table and reuse for
  2109. // subsequent writes to the same destination as well as reversing the
  2110. // conversion on reads.
  2111. //
  2112. // If synthesize fails, fall back to trying the original address.
  2113. // The netip.Addr type is used as the map key and the input address is
  2114. // assumed to be of the type *net.UDPAddr. This allows for more efficient
  2115. // lookup operations vs. a string key and parsing the input address via
  2116. // addr.String()/net.SplitHostPort().
  2117. udpAddr, ok := addr.(*net.UDPAddr)
  2118. if !ok {
  2119. return 0, errors.Tracef("unexpected addr type: %T", addr)
  2120. }
  2121. // Stack allocate to avoid an extra heap allocation per write.
  2122. var synthesizedAddr net.UDPAddr
  2123. if udpAddr.IP.To4() != nil {
  2124. ip4Addr, ok := netip.AddrFromSlice(udpAddr.IP)
  2125. if !ok {
  2126. return 0, errors.Tracef("invalid addr")
  2127. }
  2128. conn.synthesizerMutex.Lock()
  2129. synthesizedIP, ok := conn.ipv4ToIPv6[ip4Addr]
  2130. conn.synthesizerMutex.Unlock()
  2131. if ok {
  2132. synthesizedAddr = net.UDPAddr{IP: synthesizedIP, Port: udpAddr.Port}
  2133. } else {
  2134. synthesized := conn.ipv6Synthesizer.IPv6Synthesize(udpAddr.IP.String())
  2135. if synthesized != "" {
  2136. synthesizedIP := net.ParseIP(synthesized)
  2137. if synthesizedIP != nil {
  2138. conn.synthesizerMutex.Lock()
  2139. conn.ipv4ToIPv6[ip4Addr] = synthesizedIP
  2140. ipv6Addr, _ := netip.AddrFromSlice(synthesizedIP)
  2141. conn.ipv6ToIPv4[ipv6Addr] = udpAddr.IP
  2142. conn.synthesizerMutex.Unlock()
  2143. synthesizedAddr = net.UDPAddr{IP: synthesizedIP, Port: udpAddr.Port}
  2144. }
  2145. }
  2146. }
  2147. }
  2148. if synthesizedAddr.IP == nil {
  2149. // Do not wrap any I/O err returned by UDPConn
  2150. return conn.udpConn.WriteTo(b, addr)
  2151. }
  2152. return conn.udpConn.WriteTo(b, &synthesizedAddr)
  2153. }
  2154. func (conn *inproxyUDPConn) Close() error {
  2155. // Do not wrap any I/O err returned by UDPConn
  2156. return conn.udpConn.Close()
  2157. }
  2158. func (conn *inproxyUDPConn) LocalAddr() net.Addr {
  2159. // Do not wrap any I/O err returned by UDPConn
  2160. return conn.udpConn.LocalAddr()
  2161. }
  2162. func (conn *inproxyUDPConn) SetDeadline(t time.Time) error {
  2163. // Do not wrap any I/O err returned by UDPConn
  2164. return conn.udpConn.SetDeadline(t)
  2165. }
  2166. func (conn *inproxyUDPConn) SetReadDeadline(t time.Time) error {
  2167. // Do not wrap any I/O err returned by UDPConn
  2168. return conn.udpConn.SetReadDeadline(t)
  2169. }
  2170. func (conn *inproxyUDPConn) SetWriteDeadline(t time.Time) error {
  2171. // Do not wrap any I/O err returned by UDPConn
  2172. return conn.udpConn.SetWriteDeadline(t)
  2173. }
  2174. // getInproxyNetworkType converts a legacy string network type to an inproxy
  2175. // package type.
  2176. func getInproxyNetworkType(networkType string) inproxy.NetworkType {
  2177. // There is no VPN type conversion; clients and proxies will skip/fail
  2178. // in-proxy operations on non-Psiphon VPN networks.
  2179. switch networkType {
  2180. case "WIFI":
  2181. return inproxy.NetworkTypeWiFi
  2182. case "MOBILE":
  2183. return inproxy.NetworkTypeMobile
  2184. case "WIRED":
  2185. return inproxy.NetworkTypeWired
  2186. case "VPN":
  2187. return inproxy.NetworkTypeVPN
  2188. }
  2189. return inproxy.NetworkTypeUnknown
  2190. }