webrtc.go 73 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288
  1. //go:build PSIPHON_ENABLE_INPROXY
  2. /*
  3. * Copyright (c) 2023, Psiphon Inc.
  4. * All rights reserved.
  5. *
  6. * This program is free software: you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License as published by
  8. * the Free Software Foundation, either version 3 of the License, or
  9. * (at your option) any later version.
  10. *
  11. * This program is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. * GNU General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. *
  19. */
  20. package inproxy
  21. import (
  22. "bytes"
  23. "context"
  24. "encoding/binary"
  25. std_errors "errors"
  26. "fmt"
  27. "net"
  28. "strconv"
  29. "strings"
  30. "sync"
  31. "sync/atomic"
  32. "time"
  33. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
  34. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
  35. inproxy_dtls "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/inproxy/dtls"
  36. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
  37. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/stacktrace"
  38. "github.com/pion/datachannel"
  39. "github.com/pion/dtls/v2"
  40. "github.com/pion/ice/v2"
  41. pion_logging "github.com/pion/logging"
  42. "github.com/pion/sdp/v3"
  43. "github.com/pion/stun"
  44. "github.com/pion/transport/v2"
  45. "github.com/pion/webrtc/v3"
  46. "github.com/wlynxg/anet"
  47. )
  48. const (
  49. dataChannelAwaitTimeout = 20 * time.Second
  50. dataChannelBufferedAmountLowThreshold uint64 = 512 * 1024
  51. dataChannelMaxBufferedAmount uint64 = 1024 * 1024
  52. dataChannelMaxMessageSize = 65536
  53. dataChannelMaxLabelLength = 512
  54. // Psiphon uses a fork of github.com/pion/dtls/v2, selected with go mod
  55. // replace, which has an idential API apart from dtls.IsPsiphon. If
  56. // dtls.IsPsiphon is undefined, the build is not using the fork.
  57. //
  58. // Limitation: this doesn't check that the vendored code is exactly the
  59. // same code as the fork.
  60. assertDTLSFork = dtls.IsPsiphon
  61. // Similarly, check for the fork of github.com/pion/ice/v2.
  62. assertICEFork = ice.IsPsiphon
  63. // Note that Psiphon also uses a fork of github.com/pion/webrtc/v3, but it
  64. // has an API change which will cause builds to fail when not present.
  65. )
  66. // webRTCConn is a WebRTC connection between two peers, with a data channel
  67. // used to relay streams or packets between them. WebRTCConn implements the
  68. // net.Conn interface.
  69. type webRTCConn struct {
  70. config *webRTCConfig
  71. trafficShapingParameters *DataChannelTrafficShapingParameters
  72. mutex sync.Mutex
  73. udpConn net.PacketConn
  74. portMapper *portMapper
  75. isClosed bool
  76. closedSignal chan struct{}
  77. peerConnection *webrtc.PeerConnection
  78. dataChannel *webrtc.DataChannel
  79. dataChannelConn datachannel.ReadWriteCloser
  80. dataChannelOpenedSignal chan struct{}
  81. dataChannelOpenedOnce sync.Once
  82. dataChannelWriteBufferSignal chan struct{}
  83. decoyDone bool
  84. iceCandidatePairMetrics common.LogFields
  85. readMutex sync.Mutex
  86. readBuffer []byte
  87. readOffset int
  88. readLength int
  89. readError error
  90. peerPaddingDone bool
  91. writeMutex sync.Mutex
  92. trafficShapingPRNG *prng.PRNG
  93. trafficShapingBuffer *bytes.Buffer
  94. paddedMessageCount int
  95. decoyMessageCount int
  96. trafficShapingDone bool
  97. paddedMessagesSent int32
  98. paddedMessagesReceived int32
  99. decoyMessagesSent int32
  100. decoyMessagesReceived int32
  101. }
  102. // webRTCConfig specifies the configuration for a WebRTC dial.
  103. type webRTCConfig struct {
  104. // Logger is used to log events.
  105. Logger common.Logger
  106. // EnableDebugLogging indicates whether to log pion/webrtc debug and trace
  107. // events. When enabled, these events will be logged to the specified
  108. // Logger at a Debug log level.
  109. EnableDebugLogging bool
  110. // WebRTCDialCoordinator specifies specific WebRTC dial strategies and
  111. // settings; WebRTCDialCoordinator also facilities dial replay by
  112. // receiving callbacks when individual dial steps succeed or fail.
  113. WebRTCDialCoordinator WebRTCDialCoordinator
  114. // ClientRootObfuscationSecret is generated (or replayed) by the client
  115. // and sent to the proxy and used to drive obfuscation operations.
  116. ClientRootObfuscationSecret ObfuscationSecret
  117. // DoDTLSRandomization indicates whether to perform DTLS randomization.
  118. DoDTLSRandomization bool
  119. // TrafficShapingParameters indicates whether and how to perform data channel traffic shaping.
  120. TrafficShapingParameters *DataChannelTrafficShapingParameters
  121. // ReliableTransport indicates whether to configure the WebRTC data
  122. // channel to use reliable transport. Set ReliableTransport when proxying
  123. // a TCP stream, and unset it when proxying a UDP packets flow with its
  124. // own reliability later, such as QUIC.
  125. ReliableTransport bool
  126. }
  127. // newWebRTCConnWithOffer initiates a new WebRTC connection. An offer SDP is
  128. // returned, to be sent to the peer. After the offer SDP is forwarded and an
  129. // answer SDP received in response, call SetRemoteSDP with the answer SDP and
  130. // then call AwaitInitialDataChannel to await the eventual WebRTC connection
  131. // establishment.
  132. func newWebRTCConnWithOffer(
  133. ctx context.Context,
  134. config *webRTCConfig,
  135. hasPersonalCompartmentIDs bool) (
  136. *webRTCConn, WebRTCSessionDescription, *webRTCSDPMetrics, error) {
  137. conn, SDP, metrics, err := newWebRTCConn(
  138. ctx, config, nil, hasPersonalCompartmentIDs)
  139. if err != nil {
  140. return nil, WebRTCSessionDescription{}, nil, errors.Trace(err)
  141. }
  142. return conn, *SDP, metrics, nil
  143. }
  144. // newWebRTCConnWithAnswer creates a new WebRTC connection initiated by a peer
  145. // that provided an offer SDP. An answer SDP is returned to be sent to the
  146. // peer. After the answer SDP is forwarded, call AwaitInitialDataChannel to
  147. // await the eventual WebRTC connection establishment.
  148. func newWebRTCConnWithAnswer(
  149. ctx context.Context,
  150. config *webRTCConfig,
  151. peerSDP WebRTCSessionDescription,
  152. hasPersonalCompartmentIDs bool) (
  153. *webRTCConn, WebRTCSessionDescription, *webRTCSDPMetrics, error) {
  154. conn, SDP, metrics, err := newWebRTCConn(
  155. ctx, config, &peerSDP, hasPersonalCompartmentIDs)
  156. if err != nil {
  157. return nil, WebRTCSessionDescription{}, nil, errors.Trace(err)
  158. }
  159. return conn, *SDP, metrics, nil
  160. }
  161. func newWebRTCConn(
  162. ctx context.Context,
  163. config *webRTCConfig,
  164. peerSDP *WebRTCSessionDescription,
  165. hasPersonalCompartmentIDs bool) (
  166. retconn *webRTCConn,
  167. retSDP *WebRTCSessionDescription,
  168. retMetrics *webRTCSDPMetrics,
  169. retErr error) {
  170. isOffer := peerSDP == nil
  171. udpConn, err := config.WebRTCDialCoordinator.UDPListen(ctx)
  172. if err != nil {
  173. return nil, nil, nil, errors.Trace(err)
  174. }
  175. // Initialize WebRTC
  176. // There is no explicit anti-probing measures for the proxy side of the
  177. // WebRTC connection, since each proxy "listener" is ephemeral, and since
  178. // the WebRTC data channel protocol authenticates peers with
  179. // certificates, so even if a probe were to find an ephemeral proxy
  180. // listener, the listener can respond the same as a normal WebRTC end
  181. // point would respond to a peer that doesn't have the correct credentials.
  182. //
  183. // pion's Mux API is used, as it enables providing a pre-created UDP
  184. // socket which is configured with necessary BindToDevice settings. We do
  185. // not actually multiplex multiple client connections on a single proxy
  186. // connection. As a proxy creates a new UDP socket and Mux for each
  187. // client, this currently open issue should not impact our
  188. // implementation: "Listener doesn't process parallel handshakes",
  189. // https://github.com/pion/dtls/issues/279.
  190. //
  191. // We detach data channels in order to use the standard Read/Write APIs.
  192. // As detaching avoids using the pion DataChannel read loop, this
  193. // currently open issue should not impact our
  194. // implementation: "DataChannel.readLoop goroutine leak",
  195. // https://github.com/pion/webrtc/issues/2098.
  196. // UDPMux Limitations:
  197. //
  198. // For Psiphon, WebRTCDialCoordinator.UDPListen will call
  199. // https://pkg.go.dev/net#ListenUDP with an unspecified IP address, in
  200. // order to listen on all available interfaces, both IPv4 and IPv6.
  201. // However, using webrtc.NewICEUDPMux and a UDP conn with an unspecified
  202. // IP address results in this log warning: "UDPMuxDefault should not
  203. // listening on unspecified address, use NewMultiUDPMuxFromPort instead".
  204. //
  205. // With NewICEUDPMux and an unspecified IP address, pion currently
  206. // enumerates local, active interfaces and derives a list of listening
  207. // addresses, combining each interface's IP addresses with the assigned
  208. // port:
  209. // https://github.com/pion/ice/blob/8c5b0991ef3bb070e47afda96faf090e8bf94be6/net.go#L35.
  210. // While this works ok in many cases, this PR,
  211. // https://github.com/pion/ice/pull/475, indicates the nature of the
  212. // issue with UDPMuxDefault:
  213. //
  214. // > When we have multiple host candidates and been mux to a single port,
  215. // > if these candidates share a same conn (either tcp or udp), they
  216. // > might read other's [messages causing failure].
  217. //
  218. // This PR, https://github.com/pion/ice/pull/473, also describes the issue:
  219. //
  220. // > When using UDPMux and UniversalUDPMux, it is possible that a
  221. // > registerConnForAddress() could be called twice or more for the same
  222. // > remote candidate (endpoint) by different candidates. E.g., when
  223. // > different HOST candidates ping the same remote candidate, the
  224. // > udpMuxedConn gets stored once. The second candidate will never
  225. // > receive a response. This is also the case when a single socket is
  226. // > used for gathering SRFLX and HOST candidates.
  227. //
  228. // PR 475 introduced MultiUDPMuxDefault to address the issue. However, at
  229. // this time, https://github.com/pion/ice/releases/tag/v2.3.6, there's an
  230. // open bug with MultiUDPMuxDefault
  231. // https://github.com/pion/ice/issues/507: "Multi UDP Mux can't works
  232. // when remote also enables Multi UDP Mux". Running the test program
  233. // attached to the bug confirms that no data channel is established;
  234. // while switching the test code to use NewICEUDPMux results in a
  235. // successful data channel connection. Since we need to use a Mux API on
  236. // both clients and proxies, we can't yet use MultiUDPMux.
  237. //
  238. // We patch pion/webrtc to add the SetICEUDPMuxSrflx functionality from
  239. // the currently unmerged https://github.com/pion/webrtc/pull/2298.
  240. // Without SetICEUDPMuxSrflx, STUN operations don't use the mux.
  241. //
  242. // We patch pion/ice gatherCandidatesSrflxUDPMux vendor patch to include
  243. // only the correct network type (IPv4 or IPv6) address candidates.
  244. // Without this patch, we observed up to 2x duplicate/redundant STUN
  245. // candidates.
  246. //
  247. // TODO: implement and try using transport.Net UDP dial functions in place
  248. // of NewICEUDPMux and pre-dialed UDP conn; track all dialed UDP
  249. // connections to close on WebRTCConn.Close; this approach would require
  250. // an alternative approach to injecting port mapping candidates, which
  251. // currently depends on the mux UDP socket being available outside of pion.
  252. // Another limitation and issue with NewICEUDPMux is that its enumeration
  253. // of all local interfaces and IPs includes many IPv6 addresses for
  254. // certain interfaces. For example, on macOS,
  255. // https://apple.stackexchange.com/a/371661, there are "secured" IPv6
  256. // addresses and many "temporary" IPv6 addresses, with all but one
  257. // temporary address being "deprecated". Instead of a full enumeration,
  258. // we should select only the non-deprecated temporary IPv6 address --
  259. // both for performance (avoid excess STUN requests) and privacy.
  260. //
  261. // Go has a proposal to expose the necessary IPv6 address information:
  262. // https://github.com/golang/go/issues/42694. However, as of Android SDK
  263. // 30, Go's net.InterfaceAddrs doesn't work at all:
  264. // https://github.com/pion/transport/issues/228,
  265. // https://github.com/golang/go/issues/40569.
  266. //
  267. // Note that it's not currently possible to
  268. // webrtc.SettingEngine.SetIPFilter to limit IPv6 selection to a single
  269. // candidate; that IP filter is not passed through to localInterfaces in
  270. // the NewUDPMuxDefault case. And even if it were, there's no guarantee
  271. // that the the first IPv6 address passed to the filter would be the
  272. // non-deprecated temporary address.
  273. //
  274. // To workaround net.Interface issues, we use SettingEngine.SetNet to plug
  275. // in an alternative implementation of net.Interface which selects only
  276. // one IPv4 and one IPv6 active interface and IP address and uses the
  277. // anet package for Android. See pionNetwork for more details.
  278. deadline, _ := ctx.Deadline()
  279. TTL := time.Until(deadline)
  280. pionLoggerFactory := newPionLoggerFactory(
  281. config.Logger,
  282. config.EnableDebugLogging)
  283. pionNetwork := newPionNetwork(
  284. ctx, pionLoggerFactory.NewLogger("net"), config.WebRTCDialCoordinator)
  285. udpMux := webrtc.NewICEUniversalUDPMux(
  286. pionLoggerFactory.NewLogger("mux"), udpConn, TTL, pionNetwork)
  287. settingEngine := webrtc.SettingEngine{
  288. LoggerFactory: pionLoggerFactory,
  289. }
  290. settingEngine.SetNet(pionNetwork)
  291. settingEngine.DetachDataChannels()
  292. settingEngine.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled)
  293. settingEngine.SetICEUDPMux(udpMux)
  294. settingEngine.SetICEUDPMuxSrflx(udpMux)
  295. // Set this behavior to look like common web browser WebRTC stacks.
  296. settingEngine.SetDTLSInsecureSkipHelloVerify(true)
  297. settingEngine.EnableSCTPZeroChecksum(true)
  298. // Timeout, retry, and delay adjustments
  299. //
  300. // - Add some jitter to timed operations to avoid a trivial pion timing
  301. // fingerprint.
  302. //
  303. // - Reduce the wait time for STUN and peer reflexive candidates from the
  304. // default 500ms and 1s.
  305. //
  306. // - Reduce keepalives from the default 2s to +/-15s and increase
  307. // disconnect timeout from the default 5s to 3x15s.
  308. //
  309. // TODO:
  310. //
  311. // - Configuration via tactics.
  312. //
  313. // - While the RFC,
  314. // https://datatracker.ietf.org/doc/html/rfc5245#section-10, calls for
  315. // keep alives no less than 15s, implementations such as Chrome send
  316. // keep alives much more frequently,
  317. // https://issues.webrtc.org/issues/42221718.
  318. //
  319. // - Varying the period bewteen each keepalive, as is done with SSH via
  320. // SSHKeepAlivePeriodMin/Max, requires changes to pion/dtls.
  321. //
  322. // - Some traffic-related timeouts are not yet exposed via settingEngine,
  323. // including ice.defaultSTUNGatherTimeout, ice.maxBindingRequestTimeout.
  324. settingEngine.SetDTLSRetransmissionInterval(prng.JitterDuration(100*time.Millisecond, 0.1))
  325. settingEngine.SetHostAcceptanceMinWait(0)
  326. settingEngine.SetSrflxAcceptanceMinWait(prng.JitterDuration(100*time.Millisecond, 0.1))
  327. settingEngine.SetPrflxAcceptanceMinWait(prng.JitterDuration(200*time.Millisecond, 0.1))
  328. settingEngine.SetICETimeouts(45*time.Second, 0, prng.JitterDuration(15*time.Second, 0.2))
  329. settingEngine.SetICEMaxBindingRequests(10)
  330. // Initialize data channel obfuscation
  331. config.Logger.WithTraceFields(common.LogFields{
  332. "dtls_randomization": config.DoDTLSRandomization,
  333. "data_channel_traffic_shaping": config.TrafficShapingParameters != nil,
  334. }).Info("webrtc_data_channel_obfuscation")
  335. // Facilitate DTLS Client/ServerHello randomization. The client decides
  336. // whether to do DTLS randomization and generates and the proxy receives
  337. // ClientRootObfuscationSecret, so the client can orchestrate replay on
  338. // both ends of the connection by reusing an obfuscation secret. Derive a
  339. // secret specific to DTLS. SetDTLSSeed will futher derive a secure PRNG
  340. // seed specific to either the client or proxy end of the connection
  341. // (so each peer's randomization will be distinct).
  342. //
  343. // To avoid forking many pion repos in order to pass the seed through to
  344. // the DTLS implementation, SetDTLSSeed attaches the seed to the DTLS
  345. // dial context.
  346. //
  347. // Either SetDTLSSeed or SetNoDTLSSeed should be set for each conn, as the
  348. // pion/dtl fork treats no-seed as an error, as a check against the
  349. // context value mechanism.
  350. var dtlsCtx context.Context
  351. if config.DoDTLSRandomization {
  352. dtlsObfuscationSecret, err := deriveObfuscationSecret(
  353. config.ClientRootObfuscationSecret, "in-proxy-DTLS-seed")
  354. if err != nil {
  355. return nil, nil, nil, errors.Trace(err)
  356. }
  357. baseSeed := prng.Seed(dtlsObfuscationSecret)
  358. dtlsCtx, err = inproxy_dtls.SetDTLSSeed(ctx, &baseSeed, isOffer)
  359. if err != nil {
  360. return nil, nil, nil, errors.Trace(err)
  361. }
  362. } else {
  363. dtlsCtx = inproxy_dtls.SetNoDTLSSeed(ctx)
  364. }
  365. settingEngine.SetDTLSConnectContextMaker(func() (context.Context, func()) {
  366. return context.WithCancel(dtlsCtx)
  367. })
  368. // Configure traffic shaping, which adds random padding and decoy messages
  369. // to data channel message flows.
  370. var trafficShapingPRNG *prng.PRNG
  371. trafficShapingBuffer := new(bytes.Buffer)
  372. paddedMessageCount := 0
  373. decoyMessageCount := 0
  374. if config.TrafficShapingParameters != nil {
  375. // TODO: also use pion/dtls.Config.PaddingLengthGenerator?
  376. trafficShapingContext := "in-proxy-data-channel-traffic-shaping-offer"
  377. if !isOffer {
  378. trafficShapingContext = "in-proxy-data-channel-traffic-shaping-answer"
  379. }
  380. trafficShapingObfuscationSecret, err := deriveObfuscationSecret(
  381. config.ClientRootObfuscationSecret, trafficShapingContext)
  382. if err != nil {
  383. return nil, nil, nil, errors.Trace(err)
  384. }
  385. seed := prng.Seed(trafficShapingObfuscationSecret)
  386. trafficShapingPRNG = prng.NewPRNGWithSeed(&seed)
  387. paddedMessageCount = trafficShapingPRNG.Range(
  388. config.TrafficShapingParameters.MinPaddedMessages,
  389. config.TrafficShapingParameters.MaxPaddedMessages)
  390. decoyMessageCount = trafficShapingPRNG.Range(
  391. config.TrafficShapingParameters.MinDecoyMessages,
  392. config.TrafficShapingParameters.MaxDecoyMessages)
  393. }
  394. // NAT traversal setup
  395. // When DisableInboundForMobileNetworks is set, skip both STUN and port
  396. // mapping for mobile networks. Most mobile networks use CGNAT and
  397. // neither STUN nor port mapping will be effective. It's faster to not
  398. // wait for something that ultimately won't work.
  399. disableInbound := config.WebRTCDialCoordinator.DisableInboundForMobileNetworks() &&
  400. config.WebRTCDialCoordinator.NetworkType() == NetworkTypeMobile
  401. // Try to establish a port mapping (UPnP-IGD, PCP, or NAT-PMP). The port
  402. // mapper will attempt to identify the local gateway and query various
  403. // port mapping protocols. portMapper.start launches this process and
  404. // does not block. Port mappings are not part of the WebRTC standard, or
  405. // supported by pion/webrtc. Instead, if a port mapping is established,
  406. // it's edited into the SDP as a new host-type ICE candidate.
  407. localPort := udpConn.LocalAddr().(*net.UDPAddr).Port
  408. portMapper := newPortMapper(config.Logger, localPort)
  409. doPortMapping := !disableInbound && !config.WebRTCDialCoordinator.DisablePortMapping()
  410. if doPortMapping {
  411. portMapper.start()
  412. }
  413. // Select a STUN server for ICE hole punching. The STUN server to be used
  414. // needs only support bind and not full RFC5780 NAT discovery.
  415. //
  416. // Each dial trys only one STUN server; in Psiphon tunnel establishment,
  417. // other, concurrent in-proxy dials may select alternative STUN servers
  418. // via WebRTCDialCoordinator. When the STUN server operation is successful,
  419. // WebRTCDialCoordinator will be signaled so that it may configure the STUN
  420. // server selection for replay.
  421. //
  422. // The STUN server will observe proxy IP addresses. Enumeration is
  423. // mitigated by using various public STUN servers, including Psiphon STUN
  424. // servers for proxies in non-censored regions. Proxies are also more
  425. // ephemeral than Psiphon servers.
  426. RFC5780 := false
  427. stunServerAddress := config.WebRTCDialCoordinator.STUNServerAddress(RFC5780)
  428. // Proceed even when stunServerAddress is "" and !DisableSTUN, as ICE may
  429. // find other host candidates.
  430. doSTUN := stunServerAddress != "" && !disableInbound && !config.WebRTCDialCoordinator.DisableSTUN()
  431. var ICEServers []webrtc.ICEServer
  432. if doSTUN {
  433. // stunServerAddress domain names are resolved with the Psiphon custom
  434. // resolver via pionNetwork.ResolveUDPAddr
  435. ICEServers = []webrtc.ICEServer{{URLs: []string{"stun:" + stunServerAddress}}}
  436. }
  437. conn := &webRTCConn{
  438. config: config,
  439. udpConn: udpConn,
  440. portMapper: portMapper,
  441. closedSignal: make(chan struct{}),
  442. dataChannelOpenedSignal: make(chan struct{}),
  443. dataChannelWriteBufferSignal: make(chan struct{}, 1),
  444. // A data channel uses SCTP and is message oriented. The maximum
  445. // message size supported by pion/webrtc is 65536:
  446. // https://github.com/pion/webrtc/blob/dce970438344727af9c9965f88d958c55d32e64d/datachannel.go#L19.
  447. // This read buffer must be as large as the maximum message size or
  448. // else a read may fail with io.ErrShortBuffer.
  449. readBuffer: make([]byte, dataChannelMaxMessageSize),
  450. trafficShapingPRNG: trafficShapingPRNG,
  451. trafficShapingBuffer: trafficShapingBuffer,
  452. paddedMessageCount: paddedMessageCount,
  453. decoyMessageCount: decoyMessageCount,
  454. }
  455. defer func() {
  456. if retErr != nil {
  457. // Cleanup on early return
  458. conn.Close()
  459. // Notify the WebRTCDialCoordinator that the operation failed so
  460. // that it can clear replay for that STUN server selection.
  461. //
  462. // Limitation: the error here may be due to failures unrelated to
  463. // the STUN server.
  464. if ctx.Err() == nil && doSTUN {
  465. config.WebRTCDialCoordinator.STUNServerAddressFailed(RFC5780, stunServerAddress)
  466. }
  467. }
  468. }()
  469. settingEngine.SetICEBindingRequestHandler(conn.onICEBindingRequest)
  470. // All settingEngine configuration must be done before calling NewAPI.
  471. webRTCAPI := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine))
  472. conn.peerConnection, err = webRTCAPI.NewPeerConnection(
  473. webrtc.Configuration{
  474. ICEServers: ICEServers,
  475. })
  476. if err != nil {
  477. return nil, nil, nil, errors.Trace(err)
  478. }
  479. conn.peerConnection.OnConnectionStateChange(conn.onConnectionStateChange)
  480. conn.peerConnection.OnICECandidate(conn.onICECandidate)
  481. conn.peerConnection.OnICEConnectionStateChange(conn.onICEConnectionStateChange)
  482. conn.peerConnection.OnICEGatheringStateChange(conn.onICEGatheringStateChange)
  483. conn.peerConnection.OnNegotiationNeeded(conn.onNegotiationNeeded)
  484. conn.peerConnection.OnSignalingStateChange(conn.onSignalingStateChange)
  485. conn.peerConnection.OnDataChannel(conn.onDataChannel)
  486. // As a future enhancement, consider using media channels instead of data
  487. // channels, as media channels may be more common. Proxied QUIC would
  488. // work over an unreliable media channel. Note that a media channel is
  489. // still prefixed with STUN and DTLS exchanges before SRTP begins, so the
  490. // first few packets are the same as a data channel.
  491. // The offer sets the data channel configuration.
  492. if isOffer {
  493. dataChannelInit := &webrtc.DataChannelInit{}
  494. if !config.ReliableTransport {
  495. ordered := false
  496. dataChannelInit.Ordered = &ordered
  497. maxRetransmits := uint16(0)
  498. dataChannelInit.MaxRetransmits = &maxRetransmits
  499. }
  500. // Generate a random length label, to vary the DATA_CHANNEL_OPEN
  501. // message length. The label is derived from and replayed via
  502. // ClientRootObfuscationSecret.
  503. labelObfuscationSecret, err := deriveObfuscationSecret(
  504. config.ClientRootObfuscationSecret, "in-proxy-data-channel-label")
  505. if err != nil {
  506. return nil, nil, nil, errors.Trace(err)
  507. }
  508. seed := prng.Seed(labelObfuscationSecret)
  509. labelPRNG := prng.NewPRNGWithSeed(&seed)
  510. dataChannelLabel := labelPRNG.HexString(
  511. labelPRNG.Range(1, dataChannelMaxLabelLength/2))
  512. dataChannel, err := conn.peerConnection.CreateDataChannel(
  513. dataChannelLabel, dataChannelInit)
  514. if err != nil {
  515. return nil, nil, nil, errors.Trace(err)
  516. }
  517. conn.setDataChannel(dataChannel)
  518. }
  519. // Prepare to await full ICE completion, including STUN candidates.
  520. // Trickle ICE is not used, simplifying the broker API. It's expected
  521. // that most clients and proxies will be behind a NAT, and not have
  522. // publicly addressable host candidates. TURN is not used. So most
  523. // candidates will be STUN, or server-reflexive, candidates.
  524. //
  525. // Later, the first to complete out of ICE or port mapping is used.
  526. //
  527. // TODO: stop waiting if an IPv6 host candidate is found?
  528. iceComplete := webrtc.GatheringCompletePromise(conn.peerConnection)
  529. // Create an offer, or input a peer's offer to create an answer.
  530. if isOffer {
  531. offer, err := conn.peerConnection.CreateOffer(nil)
  532. if err != nil {
  533. return nil, nil, nil, errors.Trace(err)
  534. }
  535. err = conn.peerConnection.SetLocalDescription(offer)
  536. if err != nil {
  537. return nil, nil, nil, errors.Trace(err)
  538. }
  539. } else {
  540. SDP := peerSDP.SDP
  541. if hasPersonalCompartmentIDs {
  542. // In personal pairing mode, the peer SDP may include private IP
  543. // addresses. To avoid unnecessary network traffic, filter out
  544. // any peer private IP addresses for which there is no
  545. // corresponding local, active interface.
  546. errorOnNoCandidates := false
  547. allowPrivateIPAddressCandidates := true
  548. filterPrivateIPAddressCandidates := true
  549. adjustedSDP, _, err := filterSDPAddresses(
  550. []byte(peerSDP.SDP),
  551. errorOnNoCandidates,
  552. nil,
  553. common.GeoIPData{},
  554. allowPrivateIPAddressCandidates,
  555. filterPrivateIPAddressCandidates)
  556. if err != nil {
  557. return nil, nil, nil, errors.Trace(err)
  558. }
  559. SDP = string(adjustedSDP)
  560. }
  561. pionSessionDescription := webrtc.SessionDescription{
  562. Type: webrtc.SDPType(peerSDP.Type),
  563. SDP: SDP,
  564. }
  565. err = conn.peerConnection.SetRemoteDescription(pionSessionDescription)
  566. if err != nil {
  567. return nil, nil, nil, errors.Trace(err)
  568. }
  569. answer, err := conn.peerConnection.CreateAnswer(nil)
  570. if err != nil {
  571. return nil, nil, nil, errors.Trace(err)
  572. }
  573. err = conn.peerConnection.SetLocalDescription(answer)
  574. if err != nil {
  575. return nil, nil, nil, errors.Trace(err)
  576. }
  577. }
  578. // Await either ICE or port mapping completion.
  579. // As a future enhancement, track which of ICE or port mapping succeeds
  580. // and is then followed by a failed WebRTC dial; stop trying the method
  581. // that often fails.
  582. iceCompleted := false
  583. portMappingExternalAddr := ""
  584. select {
  585. case <-iceComplete:
  586. iceCompleted = true
  587. case portMappingExternalAddr = <-portMapper.portMappingExternalAddress():
  588. // Set responding port mapping types for metrics.
  589. //
  590. // Limitation: if there are multiple responding protocol types, it's
  591. // not known here which was used for this dial.
  592. config.WebRTCDialCoordinator.SetPortMappingTypes(
  593. getRespondingPortMappingTypes(config.WebRTCDialCoordinator.NetworkID()))
  594. case <-ctx.Done():
  595. return nil, nil, nil, errors.Trace(ctx.Err())
  596. }
  597. // Release any port mapping resources when not using it.
  598. if portMapper != nil && portMappingExternalAddr == "" {
  599. portMapper.close()
  600. conn.portMapper = nil
  601. }
  602. config.Logger.WithTraceFields(common.LogFields{
  603. "ice_completed": iceCompleted,
  604. "port_mapping": portMappingExternalAddr != "",
  605. }).Info("webrtc_candidates_gathered")
  606. // Get the offer or answer, now populated with any ICE candidates.
  607. localDescription := conn.peerConnection.LocalDescription()
  608. // Adjust the SDP, removing local network addresses and adding any
  609. // port mapping candidate. Clients (offer) are permitted to have
  610. // no ICE candidates but proxies (answer) must have at least one
  611. //candidate.
  612. errorOnNoCandidates := !isOffer
  613. adjustedSDP, metrics, err := prepareSDPAddresses(
  614. []byte(localDescription.SDP),
  615. errorOnNoCandidates,
  616. portMappingExternalAddr,
  617. config.WebRTCDialCoordinator.DisableIPv6ICECandidates(),
  618. hasPersonalCompartmentIDs)
  619. if err != nil {
  620. return nil, nil, nil, errors.Trace(err)
  621. }
  622. // When STUN was attempted, ICE completed, and a STUN server-reflexive
  623. // candidate is present, notify the WebRTCDialCoordinator so that it can
  624. // set replay for that STUN server selection.
  625. if iceCompleted && doSTUN {
  626. hasServerReflexive := false
  627. for _, candidateType := range metrics.iceCandidateTypes {
  628. if candidateType == ICECandidateServerReflexive {
  629. hasServerReflexive = true
  630. }
  631. }
  632. if hasServerReflexive {
  633. config.WebRTCDialCoordinator.STUNServerAddressSucceeded(RFC5780, stunServerAddress)
  634. } else {
  635. config.WebRTCDialCoordinator.STUNServerAddressFailed(RFC5780, stunServerAddress)
  636. }
  637. }
  638. // The WebRTCConn is prepared, but the data channel is not yet connected.
  639. // On the offer end, the peer's following answer must be input to
  640. // SetRemoteSDP. And both ends must call AwaitInitialDataChannel to await
  641. // the data channel establishment.
  642. return conn,
  643. &WebRTCSessionDescription{
  644. Type: int(localDescription.Type),
  645. SDP: string(adjustedSDP),
  646. },
  647. metrics,
  648. nil
  649. }
  650. func (conn *webRTCConn) setDataChannel(dataChannel *webrtc.DataChannel) {
  651. // Assumes the caller holds conn.mutex, or is newWebRTCConn, creating the
  652. // conn.
  653. conn.dataChannel = dataChannel
  654. conn.dataChannel.OnOpen(conn.onDataChannelOpen)
  655. conn.dataChannel.OnClose(conn.onDataChannelClose)
  656. // Set up flow control (see comment in conn.Write)
  657. conn.dataChannel.SetBufferedAmountLowThreshold(dataChannelBufferedAmountLowThreshold)
  658. conn.dataChannel.OnBufferedAmountLow(func() {
  659. select {
  660. case conn.dataChannelWriteBufferSignal <- struct{}{}:
  661. default:
  662. }
  663. })
  664. }
  665. // SetRemoteSDP takes the answer SDP that is received in response to an offer
  666. // SDP. SetRemoteSDP initiates the WebRTC connection establishment on the
  667. // offer end.
  668. func (conn *webRTCConn) SetRemoteSDP(
  669. peerSDP WebRTCSessionDescription,
  670. hasPersonalCompartmentIDs bool) error {
  671. conn.mutex.Lock()
  672. defer conn.mutex.Unlock()
  673. SDP := peerSDP.SDP
  674. if hasPersonalCompartmentIDs {
  675. // In personal pairing mode, the peer SDP may include private IP
  676. // addresses. To avoid unnecessary network traffic, filter out any
  677. // peer private IP addresses for which there is no corresponding
  678. // local, active interface.
  679. errorOnNoCandidates := false
  680. allowPrivateIPAddressCandidates := true
  681. filterPrivateIPAddressCandidates := true
  682. adjustedSDP, _, err := filterSDPAddresses(
  683. []byte(peerSDP.SDP),
  684. errorOnNoCandidates,
  685. nil,
  686. common.GeoIPData{},
  687. allowPrivateIPAddressCandidates,
  688. filterPrivateIPAddressCandidates)
  689. if err != nil {
  690. return errors.Trace(err)
  691. }
  692. SDP = string(adjustedSDP)
  693. }
  694. pionSessionDescription := webrtc.SessionDescription{
  695. Type: webrtc.SDPType(peerSDP.Type),
  696. SDP: SDP,
  697. }
  698. err := conn.peerConnection.SetRemoteDescription(pionSessionDescription)
  699. if err != nil {
  700. return errors.Trace(err)
  701. }
  702. return nil
  703. }
  704. // AwaitInitialDataChannel returns when the data channel is established, or
  705. // when an error has occured.
  706. func (conn *webRTCConn) AwaitInitialDataChannel(ctx context.Context) error {
  707. // Don't lock the mutex, or else necessary operations will deadlock.
  708. select {
  709. case <-conn.dataChannelOpenedSignal:
  710. // The data channel is connected.
  711. err := conn.recordSelectedICECandidateStats()
  712. if err != nil {
  713. conn.config.Logger.WithTraceFields(common.LogFields{
  714. "error": err.Error()}).Warning("recordCandidateStats failed")
  715. // Continue without log
  716. }
  717. case <-ctx.Done():
  718. return errors.Tracef("with ICE candidate pairs %s: %w",
  719. conn.getICECandidatePairsSummary(),
  720. ctx.Err())
  721. case <-conn.closedSignal:
  722. return errors.TraceNew("connection has closed")
  723. }
  724. return nil
  725. }
  726. func (conn *webRTCConn) getICECandidatePairsSummary() string {
  727. conn.mutex.Lock()
  728. defer conn.mutex.Unlock()
  729. stateCounts := map[webrtc.StatsICECandidatePairState]int{}
  730. statsReport := conn.peerConnection.GetStats()
  731. for key, stats := range statsReport {
  732. // Uses the pion StatsReport key formats "candidate:<ID>"
  733. // and "candidate:<ID>-candidate:<ID>"
  734. key, found := strings.CutPrefix(key, "candidate:")
  735. if !found {
  736. continue
  737. }
  738. candidateIDs := strings.Split(key, "-candidate:")
  739. if len(candidateIDs) != 2 {
  740. continue
  741. }
  742. candidatePairStats, ok := stats.(webrtc.ICECandidatePairStats)
  743. if !ok {
  744. continue
  745. }
  746. stateCounts[candidatePairStats.State] += 1
  747. }
  748. if len(stateCounts) == 0 {
  749. return "(none)"
  750. }
  751. var strs []string
  752. for state, count := range stateCounts {
  753. strs = append(strs, fmt.Sprintf("%s(%d)", state, count))
  754. }
  755. return strings.Join(strs, ", ")
  756. }
  757. func (conn *webRTCConn) recordSelectedICECandidateStats() error {
  758. conn.mutex.Lock()
  759. defer conn.mutex.Unlock()
  760. statsReport := conn.peerConnection.GetStats()
  761. foundNominatedPair := false
  762. for key, stats := range statsReport {
  763. // Uses the pion StatsReport key formats "candidate:<ID>"
  764. // and "candidate:<ID>-candidate:<ID>"
  765. key, found := strings.CutPrefix(key, "candidate:")
  766. if !found {
  767. continue
  768. }
  769. candidateIDs := strings.Split(key, "-candidate:")
  770. if len(candidateIDs) != 2 {
  771. continue
  772. }
  773. candidatePairStats, ok := stats.(webrtc.ICECandidatePairStats)
  774. if !ok ||
  775. candidatePairStats.State != webrtc.StatsICECandidatePairStateSucceeded ||
  776. !candidatePairStats.Nominated {
  777. continue
  778. }
  779. localKey := fmt.Sprintf("candidate:%s", candidateIDs[0])
  780. stats, ok := statsReport[localKey]
  781. if !ok {
  782. return errors.TraceNew("missing local ICECandidateStats")
  783. }
  784. localCandidateStats, ok := stats.(webrtc.ICECandidateStats)
  785. if !ok {
  786. return errors.TraceNew("unexpected local ICECandidateStats")
  787. }
  788. remoteKey := fmt.Sprintf("candidate:%s", candidateIDs[1])
  789. stats, ok = statsReport[remoteKey]
  790. if !ok {
  791. return errors.TraceNew("missing remote ICECandidateStats")
  792. }
  793. remoteCandidateStats, ok := stats.(webrtc.ICECandidateStats)
  794. if !ok {
  795. return errors.TraceNew("unexpected remote ICECandidateStats")
  796. }
  797. // Use the same ICE candidate type names as logged in broker logs.
  798. logCandidateType := func(
  799. iceCandidateType webrtc.ICECandidateType) string {
  800. logType := ICECandidateUnknown
  801. switch iceCandidateType {
  802. case webrtc.ICECandidateTypeHost:
  803. logType = ICECandidateHost
  804. case webrtc.ICECandidateTypeSrflx:
  805. logType = ICECandidateServerReflexive
  806. case webrtc.ICECandidateTypePrflx:
  807. logType = ICECandidatePeerReflexive
  808. }
  809. return logType.String()
  810. }
  811. conn.iceCandidatePairMetrics = common.LogFields{}
  812. // TODO: log which of local/remote candidate is initiator
  813. conn.iceCandidatePairMetrics["inproxy_webrtc_local_ice_candidate_type"] =
  814. logCandidateType(localCandidateStats.CandidateType)
  815. localIP := net.ParseIP(localCandidateStats.IP)
  816. isIPv6 := "0"
  817. if localIP != nil && localIP.To4() == nil {
  818. isIPv6 = "1"
  819. }
  820. isPrivate := "0"
  821. if localIP != nil && localIP.IsPrivate() {
  822. isPrivate = "1"
  823. }
  824. conn.iceCandidatePairMetrics["inproxy_webrtc_local_ice_candidate_is_IPv6"] =
  825. isIPv6
  826. conn.iceCandidatePairMetrics["inproxy_webrtc_local_ice_candidate_is_private_IP"] =
  827. isPrivate
  828. conn.iceCandidatePairMetrics["inproxy_webrtc_local_ice_candidate_port"] =
  829. localCandidateStats.Port
  830. conn.iceCandidatePairMetrics["inproxy_webrtc_remote_ice_candidate_type"] =
  831. logCandidateType(remoteCandidateStats.CandidateType)
  832. remoteIP := net.ParseIP(remoteCandidateStats.IP)
  833. isIPv6 = "0"
  834. if remoteIP != nil && remoteIP.To4() == nil {
  835. isIPv6 = "1"
  836. }
  837. isPrivate = "0"
  838. if remoteIP != nil && remoteIP.IsPrivate() {
  839. isPrivate = "1"
  840. }
  841. conn.iceCandidatePairMetrics["inproxy_webrtc_remote_ice_candidate_is_IPv6"] =
  842. isIPv6
  843. conn.iceCandidatePairMetrics["inproxy_webrtc_remote_ice_candidate_is_private_IP"] =
  844. isPrivate
  845. conn.iceCandidatePairMetrics["inproxy_webrtc_remote_ice_candidate_port"] =
  846. remoteCandidateStats.Port
  847. foundNominatedPair = true
  848. break
  849. }
  850. if !foundNominatedPair {
  851. return errors.TraceNew("missing nominated ICECandidateStatsPair")
  852. }
  853. return nil
  854. }
  855. func (conn *webRTCConn) Close() error {
  856. conn.mutex.Lock()
  857. defer conn.mutex.Unlock()
  858. if conn.isClosed {
  859. return nil
  860. }
  861. if conn.portMapper != nil {
  862. conn.portMapper.close()
  863. }
  864. if conn.dataChannelConn != nil {
  865. conn.dataChannelConn.Close()
  866. }
  867. if conn.dataChannel != nil {
  868. conn.dataChannel.Close()
  869. }
  870. if conn.peerConnection != nil {
  871. conn.peerConnection.Close()
  872. }
  873. // Close the udpConn to interrupt any blocking DTLS handshake:
  874. // https://github.com/pion/webrtc/blob/c1467e4871c78ee3f463b50d858d13dc6f2874a4/dtlstransport.go#L334-L340
  875. //
  876. // Limitation: there is no guarantee that pion sends any closing packets
  877. // before the UDP socket is closed here.
  878. if conn.udpConn != nil {
  879. conn.udpConn.Close()
  880. }
  881. close(conn.closedSignal)
  882. conn.isClosed = true
  883. return nil
  884. }
  885. func (conn *webRTCConn) IsClosed() bool {
  886. conn.mutex.Lock()
  887. defer conn.mutex.Unlock()
  888. return conn.isClosed
  889. }
  890. func (conn *webRTCConn) Read(p []byte) (int, error) {
  891. for {
  892. n, err := conn.readMessage(p)
  893. if err != nil || n > 0 {
  894. return n, err
  895. }
  896. // A decoy message was read; discard and read again.
  897. }
  898. }
  899. func (conn *webRTCConn) readMessage(p []byte) (int, error) {
  900. // Don't hold this lock, or else concurrent Writes will be blocked.
  901. conn.mutex.Lock()
  902. isClosed := conn.isClosed
  903. dataChannelConn := conn.dataChannelConn
  904. decoyDone := conn.decoyDone
  905. conn.mutex.Unlock()
  906. if isClosed {
  907. return 0, errors.TraceNew("closed")
  908. }
  909. if dataChannelConn == nil {
  910. return 0, errors.TraceNew("no data channel")
  911. }
  912. // The input read buffer, p, may not be the same length as the message
  913. // read from the data channel. Buffer the read message if another Read
  914. // call is necessary to consume it. As per https://pkg.go.dev/io#Reader,
  915. // dataChannelConn bytes read are processed even when
  916. // dataChannelConn.Read returns an error; the error value is stored and
  917. // returned with the Read call that consumes the end of the message buffer.
  918. conn.readMutex.Lock()
  919. defer conn.readMutex.Unlock()
  920. if conn.readOffset == conn.readLength {
  921. n, err := dataChannelConn.Read(conn.readBuffer)
  922. conn.readOffset = 0
  923. conn.readLength = n
  924. conn.readError = err
  925. // Skip over padding.
  926. if n > 0 && !conn.peerPaddingDone {
  927. paddingSize, n := binary.Varint(conn.readBuffer[0:conn.readLength])
  928. if (paddingSize == 0 && n <= 0) || paddingSize >= int64(conn.readLength) {
  929. return 0, errors.TraceNew("invalid padding")
  930. }
  931. if paddingSize < 0 {
  932. // When the padding header indicates a padding size of -1, the
  933. // peer is indicating that padding is done. Subsequent
  934. // messages will have no padding header or padding bytes.
  935. conn.peerPaddingDone = true
  936. conn.readOffset += n
  937. } else {
  938. conn.readOffset += n + int(paddingSize)
  939. atomic.AddInt32(&conn.paddedMessagesReceived, 1)
  940. if conn.readOffset == conn.readLength {
  941. atomic.AddInt32(&conn.decoyMessagesReceived, 1)
  942. }
  943. }
  944. }
  945. }
  946. n := copy(p, conn.readBuffer[conn.readOffset:conn.readLength])
  947. conn.readOffset += n
  948. var err error
  949. if conn.readOffset == conn.readLength {
  950. err = conn.readError
  951. }
  952. // When decoy messages are enabled, periodically response to an incoming
  953. // messages with an immediate outbound decoy message. This is similar to
  954. // the design here:
  955. // https://github.com/Psiphon-Labs/psiphon-tunnel-core/blob/c4f6a593a645db4479a7032a9e97d3c0b905cdfc/psiphon/common/quic/obfuscator.go#L361-L409
  956. //
  957. // writeMessage handles conn.decoyMessageCount, which is syncronized with
  958. // conn.WriteMutex, as well as other specific logic. Here we just signal
  959. // writeMessage based on the read event.
  960. //
  961. // When the data channel already has buffered writes in excess of a decoy
  962. // message size, the writeMessage skips the decoy message and returns
  963. // without blocking, so Read calls will not block.
  964. if !decoyDone {
  965. _, _ = conn.writeMessage(nil, true)
  966. }
  967. return n, errors.Trace(err)
  968. }
  969. func (conn *webRTCConn) Write(p []byte) (int, error) {
  970. return conn.writeMessage(p, false)
  971. }
  972. func (conn *webRTCConn) writeMessage(p []byte, decoy bool) (int, error) {
  973. if p != nil && decoy {
  974. return 0, errors.TraceNew("invalid write parameters")
  975. }
  976. // pion/sctp doesn't handle 0-byte writes correctly, so drop/skip at this level.
  977. //
  978. // Testing shows that the SCTP connection stalls after a 0-byte write. In
  979. // the pion/sctp implementation,
  980. // https://github.com/pion/sctp/blob/v1.8.8/stream.go#L254-L278 and
  981. // https://github.com/pion/sctp/blob/v1.8.8/stream.go#L280-L336, it
  982. // appears that a zero-byte write won't send an SCTP messages but does
  983. // increment a sequence number.
  984. if len(p) == 0 && !decoy {
  985. return 0, nil
  986. }
  987. // Don't hold this lock, or else concurrent Reads will be blocked.
  988. conn.mutex.Lock()
  989. isClosed := conn.isClosed
  990. bufferedAmount := conn.dataChannel.BufferedAmount()
  991. dataChannelConn := conn.dataChannelConn
  992. conn.mutex.Unlock()
  993. if isClosed {
  994. return 0, errors.TraceNew("closed")
  995. }
  996. if dataChannelConn == nil {
  997. return 0, errors.TraceNew("no data channel")
  998. }
  999. // Only proceed with a decoy message when no pending writes are buffered.
  1000. //
  1001. // This check is made before acquiring conn.writeMutex so that, in most
  1002. // cases, writeMessage won't block Read calls when a concurrent Write is
  1003. // holding conn.writeMutex and potentially blocking on flow control.
  1004. // There's still a chance that this test passes, and a concurrent Write
  1005. // arrives at the same time.
  1006. if decoy && bufferedAmount > 0 {
  1007. return 0, nil
  1008. }
  1009. conn.writeMutex.Lock()
  1010. defer conn.writeMutex.Unlock()
  1011. writeSize := len(p)
  1012. // Determine padding size and padding header size.
  1013. doPadding := false
  1014. paddingSize := 0
  1015. var paddingHeader [binary.MaxVarintLen32]byte
  1016. paddingHeaderSize := 0
  1017. if decoy {
  1018. if conn.decoyMessageCount < 1 {
  1019. return 0, nil
  1020. }
  1021. if !conn.trafficShapingPRNG.FlipWeightedCoin(
  1022. conn.config.TrafficShapingParameters.DecoyMessageProbability) {
  1023. return 0, nil
  1024. }
  1025. conn.decoyMessageCount -= 1
  1026. decoySize := conn.trafficShapingPRNG.Range(
  1027. conn.config.TrafficShapingParameters.MinDecoySize,
  1028. conn.config.TrafficShapingParameters.MaxDecoySize)
  1029. // When sending a decoy message, the entire message is padding.
  1030. doPadding = true
  1031. paddingSize = decoySize
  1032. if conn.decoyMessageCount == 0 {
  1033. // Set the shared flag that readMessage uses to stop invoking
  1034. // writeMessage for decoy events.
  1035. conn.mutex.Lock()
  1036. conn.decoyDone = true
  1037. conn.mutex.Unlock()
  1038. }
  1039. } else if conn.paddedMessageCount > 0 {
  1040. // Add padding to a normal write.
  1041. conn.paddedMessageCount -= 1
  1042. doPadding = true
  1043. paddingSize = prng.Range(
  1044. conn.config.TrafficShapingParameters.MinPaddingSize,
  1045. conn.config.TrafficShapingParameters.MaxPaddingSize)
  1046. } else if conn.decoyMessageCount > 0 {
  1047. // Padding normal messages is done, but there are still outstanding
  1048. // decoy messages, so add a padding header indicating padding size 0
  1049. // to this normal message.
  1050. doPadding = true
  1051. paddingSize = 0
  1052. } else if !conn.trafficShapingDone {
  1053. // Padding normal messages is done and all decoy messages are sent, so
  1054. // send a special padding header with padding size -1, signaling the
  1055. // peer that no additional padding will be performed and no
  1056. // subsequent messages will contain a padding header.
  1057. doPadding = true
  1058. paddingSize = -1
  1059. }
  1060. if doPadding {
  1061. if paddingSize > 0 {
  1062. // Reduce, if necessary, to stay within the maximum data channel
  1063. // message size. This is not expected to happen for the io.Copy use
  1064. // case, with 32K message size, plus reasonable padding sizes.
  1065. if writeSize+binary.MaxVarintLen32+paddingSize > dataChannelMaxMessageSize {
  1066. paddingSize -= (writeSize + binary.MaxVarintLen32 + paddingSize) - dataChannelMaxMessageSize
  1067. if paddingSize < 0 {
  1068. paddingSize = 0
  1069. }
  1070. }
  1071. // Add padding overhead to total writeSize before the flow control check.
  1072. writeSize += paddingSize
  1073. }
  1074. paddingHeaderSize = binary.PutVarint(paddingHeader[:], int64(paddingSize))
  1075. writeSize += paddingHeaderSize
  1076. }
  1077. if writeSize > dataChannelMaxMessageSize {
  1078. return 0, errors.TraceNew("write too large")
  1079. }
  1080. // Flow control is required to ensure that Write calls don't result in
  1081. // unbounded buffering in pion/webrtc. Use similar logic and the same
  1082. // buffer size thresholds as the pion sample code.
  1083. //
  1084. // https://github.com/pion/webrtc/tree/master/examples/data-channels-flow-control#when-do-we-need-it:
  1085. // > Send or SendText methods are called on DataChannel to send data to
  1086. // > the connected peer. The methods return immediately, but it does not
  1087. // > mean the data was actually sent onto the wire. Instead, it is
  1088. // > queued in a buffer until it actually gets sent out to the wire.
  1089. // >
  1090. // > When you have a large amount of data to send, it is an application's
  1091. // > responsibility to control the buffered amount in order not to
  1092. // > indefinitely grow the buffer size to eventually exhaust the memory.
  1093. // If the pion write buffer is too full, wait for a signal that sufficient
  1094. // write data has been consumed before writing more.
  1095. if !isClosed && bufferedAmount+uint64(writeSize) > dataChannelMaxBufferedAmount {
  1096. select {
  1097. case <-conn.dataChannelWriteBufferSignal:
  1098. case <-conn.closedSignal:
  1099. return 0, errors.TraceNew("connection has closed")
  1100. }
  1101. }
  1102. if conn.trafficShapingDone {
  1103. // When traffic shaping is done, p is written directly without the
  1104. // additional trafficShapingBuffer copy.
  1105. // Limitation: if len(p) > 65536, the dataChannelConn.Write will fail. In
  1106. // practise, this is not expected to happen with typical use cases such
  1107. // as io.Copy, which uses a 32K buffer.
  1108. n, err := dataChannelConn.Write(p)
  1109. return n, errors.Trace(err)
  1110. }
  1111. conn.trafficShapingBuffer.Reset()
  1112. conn.trafficShapingBuffer.Write(paddingHeader[:paddingHeaderSize])
  1113. if paddingSize > 0 {
  1114. conn.trafficShapingBuffer.Write(prng.Bytes(paddingSize))
  1115. }
  1116. conn.trafficShapingBuffer.Write(p)
  1117. // Limitation: see above; len(p) + padding must be <= 65536.
  1118. _, err := dataChannelConn.Write(conn.trafficShapingBuffer.Bytes())
  1119. if decoy {
  1120. atomic.AddInt32(&conn.decoyMessagesSent, 1)
  1121. } else if doPadding && paddingSize > 0 {
  1122. atomic.AddInt32(&conn.paddedMessagesSent, 1)
  1123. }
  1124. if conn.paddedMessageCount == 0 && conn.decoyMessageCount == 0 && paddingSize == -1 {
  1125. // Set flag indicating -1 padding size was sent and release traffic
  1126. // shaping resources.
  1127. conn.trafficShapingDone = true
  1128. conn.trafficShapingPRNG = nil
  1129. conn.trafficShapingBuffer = nil
  1130. }
  1131. return len(p), errors.Trace(err)
  1132. }
  1133. func (conn *webRTCConn) LocalAddr() net.Addr {
  1134. conn.mutex.Lock()
  1135. defer conn.mutex.Unlock()
  1136. // This is the local UDP socket address, not the external, public address.
  1137. return conn.udpConn.LocalAddr()
  1138. }
  1139. func (conn *webRTCConn) RemoteAddr() net.Addr {
  1140. conn.mutex.Lock()
  1141. defer conn.mutex.Unlock()
  1142. // Not supported.
  1143. return nil
  1144. }
  1145. func (conn *webRTCConn) SetDeadline(t time.Time) error {
  1146. conn.mutex.Lock()
  1147. defer conn.mutex.Unlock()
  1148. return errors.TraceNew("not supported")
  1149. }
  1150. func (conn *webRTCConn) SetReadDeadline(t time.Time) error {
  1151. conn.mutex.Lock()
  1152. defer conn.mutex.Unlock()
  1153. if conn.isClosed {
  1154. return errors.TraceNew("closed")
  1155. }
  1156. readDeadliner, ok := conn.dataChannelConn.(datachannel.ReadDeadliner)
  1157. if !ok {
  1158. return errors.TraceNew("no data channel")
  1159. }
  1160. return readDeadliner.SetReadDeadline(t)
  1161. }
  1162. func (conn *webRTCConn) SetWriteDeadline(t time.Time) error {
  1163. conn.mutex.Lock()
  1164. defer conn.mutex.Unlock()
  1165. return errors.TraceNew("not supported")
  1166. }
  1167. // GetMetrics implements the common.MetricsSource interface and returns log
  1168. // fields detailing the WebRTC dial parameters.
  1169. func (conn *webRTCConn) GetMetrics() common.LogFields {
  1170. conn.mutex.Lock()
  1171. defer conn.mutex.Unlock()
  1172. logFields := make(common.LogFields)
  1173. logFields.Add(conn.iceCandidatePairMetrics)
  1174. randomizeDTLS := "0"
  1175. if conn.config.DoDTLSRandomization {
  1176. randomizeDTLS = "1"
  1177. }
  1178. logFields["inproxy_webrtc_randomize_dtls"] = randomizeDTLS
  1179. logFields["inproxy_webrtc_padded_messages_sent"] = atomic.LoadInt32(&conn.paddedMessagesSent)
  1180. logFields["inproxy_webrtc_padded_messages_received"] = atomic.LoadInt32(&conn.paddedMessagesReceived)
  1181. logFields["inproxy_webrtc_decoy_messages_sent"] = atomic.LoadInt32(&conn.decoyMessagesSent)
  1182. logFields["inproxy_webrtc_decoy_messages_received"] = atomic.LoadInt32(&conn.decoyMessagesReceived)
  1183. return logFields
  1184. }
  1185. func (conn *webRTCConn) onConnectionStateChange(state webrtc.PeerConnectionState) {
  1186. // Close the WebRTCConn when the connection is no longer connected. Close
  1187. // will lock conn.mutex, so do lot aquire the lock here.
  1188. //
  1189. // Currently, ICE Restart is not used, and there is no transition from
  1190. // Disconnected back to Connected.
  1191. switch state {
  1192. case webrtc.PeerConnectionStateDisconnected,
  1193. webrtc.PeerConnectionStateFailed,
  1194. webrtc.PeerConnectionStateClosed:
  1195. conn.Close()
  1196. }
  1197. conn.config.Logger.WithTraceFields(common.LogFields{
  1198. "state": state.String(),
  1199. }).Info("peer connection state changed")
  1200. }
  1201. func (conn *webRTCConn) onICECandidate(candidate *webrtc.ICECandidate) {
  1202. if candidate == nil {
  1203. return
  1204. }
  1205. conn.config.Logger.WithTraceFields(common.LogFields{
  1206. "candidate": candidate.String(),
  1207. }).Info("new ICE candidate")
  1208. }
  1209. func (conn *webRTCConn) onICEBindingRequest(m *stun.Message, local, remote ice.Candidate, pair *ice.CandidatePair) bool {
  1210. // SetICEBindingRequestHandler is used to hook onICEBindingRequest into
  1211. // STUN bind events for logging. The return values is always false as
  1212. // this callback makes no adjustments to ICE candidate selection. When
  1213. // the data channel has already opened, skip logging events, as this
  1214. // callback appears to be invoked for keepalive pings.
  1215. if local == nil || remote == nil {
  1216. return false
  1217. }
  1218. select {
  1219. case <-conn.dataChannelOpenedSignal:
  1220. return false
  1221. default:
  1222. }
  1223. conn.config.Logger.WithTraceFields(common.LogFields{
  1224. "local_candidate": local.String(),
  1225. "remote_candidate": remote.String(),
  1226. }).Info("new ICE STUN binding request")
  1227. return false
  1228. }
  1229. func (conn *webRTCConn) onICEConnectionStateChange(state webrtc.ICEConnectionState) {
  1230. conn.config.Logger.WithTraceFields(common.LogFields{
  1231. "state": state.String(),
  1232. }).Info("ICE connection state changed")
  1233. }
  1234. func (conn *webRTCConn) onICEGatheringStateChange(state webrtc.ICEGathererState) {
  1235. conn.config.Logger.WithTraceFields(common.LogFields{
  1236. "state": state.String(),
  1237. }).Info("ICE gathering state changed")
  1238. }
  1239. func (conn *webRTCConn) onNegotiationNeeded() {
  1240. conn.config.Logger.WithTrace().Info("negotiation needed")
  1241. }
  1242. func (conn *webRTCConn) onSignalingStateChange(state webrtc.SignalingState) {
  1243. conn.config.Logger.WithTraceFields(common.LogFields{
  1244. "state": state.String(),
  1245. }).Info("signaling state changed")
  1246. }
  1247. func (conn *webRTCConn) onDataChannel(dataChannel *webrtc.DataChannel) {
  1248. conn.mutex.Lock()
  1249. defer conn.mutex.Unlock()
  1250. conn.setDataChannel(dataChannel)
  1251. conn.config.Logger.WithTraceFields(common.LogFields{
  1252. "label": dataChannel.Label(),
  1253. "ID": dataChannel.ID(),
  1254. }).Info("new data channel")
  1255. }
  1256. func (conn *webRTCConn) onDataChannelOpen() {
  1257. conn.mutex.Lock()
  1258. defer conn.mutex.Unlock()
  1259. dataChannelConn, err := conn.dataChannel.Detach()
  1260. if err == nil {
  1261. conn.dataChannelConn = dataChannelConn
  1262. // TODO: can a data channel be connected, disconnected, and then
  1263. // reestablished in one session?
  1264. conn.dataChannelOpenedOnce.Do(func() { close(conn.dataChannelOpenedSignal) })
  1265. }
  1266. conn.config.Logger.WithTraceFields(common.LogFields{
  1267. "detachError": err,
  1268. }).Info("data channel open")
  1269. }
  1270. func (conn *webRTCConn) onDataChannelClose() {
  1271. // Close the WebRTCConn when the data channel is closed. Close will lock
  1272. // conn.mutex, so do lot aquire the lock here.
  1273. conn.Close()
  1274. conn.config.Logger.WithTrace().Info("data channel closed")
  1275. }
  1276. // prepareSDPAddresses adjusts the SDP, pruning local network addresses and
  1277. // adding any port mapping as a host candidate.
  1278. func prepareSDPAddresses(
  1279. encodedSDP []byte,
  1280. errorOnNoCandidates bool,
  1281. portMappingExternalAddr string,
  1282. disableIPv6Candidates bool,
  1283. allowPrivateIPAddressCandidates bool) ([]byte, *webRTCSDPMetrics, error) {
  1284. modifiedSDP, metrics, err := processSDPAddresses(
  1285. encodedSDP,
  1286. errorOnNoCandidates,
  1287. portMappingExternalAddr,
  1288. disableIPv6Candidates,
  1289. allowPrivateIPAddressCandidates,
  1290. false,
  1291. nil,
  1292. common.GeoIPData{})
  1293. return modifiedSDP, metrics, errors.Trace(err)
  1294. }
  1295. // filterSDPAddresses checks that the SDP does not contain an empty list of
  1296. // candidates, bogon candidates, or candidates outside of the country and ASN
  1297. // for the specified expectedGeoIPData. Invalid candidates are stripped and a
  1298. // filtered SDP is returned.
  1299. func filterSDPAddresses(
  1300. encodedSDP []byte,
  1301. errorOnNoCandidates bool,
  1302. lookupGeoIP LookupGeoIP,
  1303. expectedGeoIPData common.GeoIPData,
  1304. allowPrivateIPAddressCandidates bool,
  1305. filterPrivateIPAddressCandidates bool) ([]byte, *webRTCSDPMetrics, error) {
  1306. filteredSDP, metrics, err := processSDPAddresses(
  1307. encodedSDP,
  1308. errorOnNoCandidates,
  1309. "",
  1310. false,
  1311. allowPrivateIPAddressCandidates,
  1312. filterPrivateIPAddressCandidates,
  1313. lookupGeoIP,
  1314. expectedGeoIPData)
  1315. return filteredSDP, metrics, errors.Trace(err)
  1316. }
  1317. // webRTCSDPMetrics are network capability metrics values for an SDP.
  1318. type webRTCSDPMetrics struct {
  1319. iceCandidateTypes []ICECandidateType
  1320. hasIPv6 bool
  1321. hasPrivateIP bool
  1322. filteredICECandidates []string
  1323. }
  1324. // processSDPAddresses is based on snowflake/common/util.StripLocalAddresses
  1325. // https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/blob/v2.5.1/common/util/util.go#L70-99
  1326. /*
  1327. This file contains the license for "Snowflake"
  1328. a free software project which provides a WebRTC pluggable transport.
  1329. ================================================================================
  1330. Copyright (c) 2016, Serene Han, Arlo Breault
  1331. Copyright (c) 2019-2020, The Tor Project, Inc
  1332. Redistribution and use in source and binary forms, with or without modification,
  1333. are permitted provided that the following conditions are met:
  1334. * Redistributions of source code must retain the above copyright notice, this
  1335. list of conditions and the following disclaimer.
  1336. * Redistributions in binary form must reproduce the above copyright notice,
  1337. this list of conditions and the following disclaimer in the documentation and/or
  1338. other materials provided with the distribution.
  1339. * Neither the names of the copyright owners nor the names of its
  1340. contributors may be used to endorse or promote products derived from this
  1341. software without specific prior written permission.
  1342. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  1343. ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  1344. WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  1345. DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
  1346. ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  1347. (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  1348. LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
  1349. ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  1350. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  1351. SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  1352. ================================================================================
  1353. */
  1354. func processSDPAddresses(
  1355. encodedSDP []byte,
  1356. errorOnNoCandidates bool,
  1357. portMappingExternalAddr string,
  1358. disableIPv6Candidates bool,
  1359. allowPrivateIPAddressCandidates bool,
  1360. filterPrivateIPAddressCandidates bool,
  1361. lookupGeoIP LookupGeoIP,
  1362. expectedGeoIPData common.GeoIPData) ([]byte, *webRTCSDPMetrics, error) {
  1363. var sessionDescription sdp.SessionDescription
  1364. err := sessionDescription.Unmarshal(encodedSDP)
  1365. if err != nil {
  1366. return nil, nil, errors.Trace(err)
  1367. }
  1368. candidateTypes := map[ICECandidateType]bool{}
  1369. hasIPv6 := false
  1370. hasPrivateIP := true
  1371. filteredCandidateReasons := make(map[string]int)
  1372. var portMappingICECandidates []sdp.Attribute
  1373. if portMappingExternalAddr != "" {
  1374. // Prepare ICE candidate attibute pair for the port mapping, modeled
  1375. // after the definition of host candidates.
  1376. host, portStr, err := net.SplitHostPort(portMappingExternalAddr)
  1377. if err != nil {
  1378. return nil, nil, errors.Trace(err)
  1379. }
  1380. port, err := strconv.Atoi(portStr)
  1381. if err != nil {
  1382. return nil, nil, errors.Trace(err)
  1383. }
  1384. // Only IPv4 port mapping addresses are supported due to the
  1385. // NewCandidateHost limitation noted below. It is expected that port
  1386. // mappings will be IPv4, as NAT and IPv6 is not a typical combination.
  1387. hostIP := net.ParseIP(host)
  1388. if hostIP != nil && hostIP.To4() != nil {
  1389. for _, component := range []webrtc.ICEComponent{webrtc.ICEComponentRTP, webrtc.ICEComponentRTCP} {
  1390. // The candidate ID is generated and the priority and foundation
  1391. // use the default for hosts.
  1392. //
  1393. // Limitation: NewCandidateHost initializes the networkType to
  1394. // NetworkTypeUDP4, and this field is not-exported.
  1395. // https://github.com/pion/ice/blob/6d301287654b05a36248842c278d58d501454bff/candidate_host.go#L27-L64
  1396. iceCandidate, err := ice.NewCandidateHost(&ice.CandidateHostConfig{
  1397. Network: "udp",
  1398. Address: host,
  1399. Port: port,
  1400. Component: uint16(component),
  1401. })
  1402. if err != nil {
  1403. return nil, nil, errors.Trace(err)
  1404. }
  1405. portMappingICECandidates = append(
  1406. portMappingICECandidates,
  1407. sdp.Attribute{Key: "candidate", Value: iceCandidate.Marshal()})
  1408. }
  1409. candidateTypes[ICECandidatePortMapping] = true
  1410. }
  1411. }
  1412. candidateCount := len(portMappingICECandidates)
  1413. for _, mediaDescription := range sessionDescription.MediaDescriptions {
  1414. addPortMappingCandidates := len(portMappingICECandidates) > 0
  1415. var attributes []sdp.Attribute
  1416. for _, attribute := range mediaDescription.Attributes {
  1417. // Insert the port mapping candidate either before the
  1418. // first "a=candidate", or before "a=end-of-candidates"(there may
  1419. // be no "a=candidate" attributes).
  1420. if addPortMappingCandidates &&
  1421. (attribute.IsICECandidate() || attribute.Key == sdp.AttrKeyEndOfCandidates) {
  1422. attributes = append(attributes, portMappingICECandidates...)
  1423. addPortMappingCandidates = false
  1424. }
  1425. if attribute.IsICECandidate() {
  1426. candidate, err := ice.UnmarshalCandidate(attribute.Value)
  1427. if err != nil {
  1428. return nil, nil, errors.Trace(err)
  1429. }
  1430. candidateIP := net.ParseIP(candidate.Address())
  1431. if candidateIP == nil {
  1432. return nil, nil, errors.TraceNew("unexpected non-IP")
  1433. }
  1434. candidateIsIPv6 := false
  1435. if candidateIP.To4() == nil {
  1436. if disableIPv6Candidates {
  1437. reason := fmt.Sprintf("disabled %s IPv6",
  1438. candidate.Type().String())
  1439. filteredCandidateReasons[reason] += 1
  1440. continue
  1441. }
  1442. candidateIsIPv6 = true
  1443. }
  1444. // Strip non-routable bogons, including RFC 1918/4193 private
  1445. // IP addresses. Same-LAN client/proxy hops are not expected
  1446. // to be useful, and this also avoids unnecessary network traffic.
  1447. //
  1448. // Well-behaved clients and proxies should strip these values;
  1449. // the broker enforces this with filtering.
  1450. //
  1451. // In personal pairing mode, private IP addresses are allowed,
  1452. // as connection may be made between devices the same LAN and
  1453. // not all routers support NAT hairpinning.
  1454. candidateIsPrivateIP := candidateIP.IsPrivate()
  1455. if !GetAllowBogonWebRTCConnections() &&
  1456. !(candidateIsPrivateIP && allowPrivateIPAddressCandidates) &&
  1457. common.IsBogon(candidateIP) {
  1458. version := "IPv4"
  1459. if candidateIsIPv6 {
  1460. version = "IPv6"
  1461. }
  1462. reason := fmt.Sprintf("bogon %s %s",
  1463. candidate.Type().String(), version)
  1464. filteredCandidateReasons[reason] += 1
  1465. continue
  1466. }
  1467. // In personal pairing mode, filter out any private IP
  1468. // addresses for which there is no corresponding local,
  1469. // active interface. This avoids unnecessary network traffic.
  1470. // This filtering option is applied post-broker exchange,
  1471. // with the SDP received, via the broker, from the peer.
  1472. if candidateIsPrivateIP && filterPrivateIPAddressCandidates {
  1473. if !hasInterfaceForPrivateIPAddress(candidateIP) {
  1474. continue
  1475. }
  1476. }
  1477. // The broker will check that clients and proxies specify only
  1478. // candidates that map to the same GeoIP country and ASN as
  1479. // the client/proxy connection to the broker. This limits
  1480. // misuse of candidates to connect to other locations.
  1481. // Legitimate candidates will not all have the exact same IP
  1482. // address, as there could be a mix of IPv4 and IPv6, as well
  1483. // as potentially different NAT paths.
  1484. //
  1485. // In some cases, legitimate clients and proxies may
  1486. // unintentionally submit candidates with mismatching GeoIP.
  1487. // This can occur, for example, when a STUN candidate is only
  1488. // a partial hole punch through double NAT, and when internal
  1489. // network addresses misuse non-private IP ranges (so are
  1490. // technically not bogons). Instead of outright rejecting
  1491. // SDPs containing unexpected GeoIP candidates, they are
  1492. // instead stripped out and the resulting filtered SDP is
  1493. // used.
  1494. if lookupGeoIP != nil {
  1495. candidateGeoIPData := lookupGeoIP(candidate.Address())
  1496. if candidateGeoIPData.Country != expectedGeoIPData.Country ||
  1497. candidateGeoIPData.ASN != expectedGeoIPData.ASN {
  1498. version := "IPv4"
  1499. if candidateIsIPv6 {
  1500. version = "IPv6"
  1501. }
  1502. reason := fmt.Sprintf(
  1503. "unexpected GeoIP %s %s: %s/%s",
  1504. candidate.Type().String(),
  1505. version,
  1506. candidateGeoIPData.Country,
  1507. candidateGeoIPData.ASN)
  1508. filteredCandidateReasons[reason] += 1
  1509. continue
  1510. }
  1511. }
  1512. if candidateIsIPv6 {
  1513. hasIPv6 = true
  1514. }
  1515. if candidateIsPrivateIP {
  1516. hasPrivateIP = true
  1517. }
  1518. // These types are not reported:
  1519. // - CandidateTypeRelay: TURN servers are not used.
  1520. // - CandidateTypePeerReflexive: this candidate type only
  1521. // emerges later in the connection process.
  1522. switch candidate.Type() {
  1523. case ice.CandidateTypeHost:
  1524. candidateTypes[ICECandidateHost] = true
  1525. case ice.CandidateTypeServerReflexive:
  1526. candidateTypes[ICECandidateServerReflexive] = true
  1527. }
  1528. candidateCount += 1
  1529. }
  1530. attributes = append(attributes, attribute)
  1531. }
  1532. mediaDescription.Attributes = attributes
  1533. }
  1534. if errorOnNoCandidates && candidateCount == 0 {
  1535. return nil, nil, errors.TraceNew("no candidates")
  1536. }
  1537. encodedSDP, err = sessionDescription.Marshal()
  1538. if err != nil {
  1539. return nil, nil, errors.Trace(err)
  1540. }
  1541. metrics := &webRTCSDPMetrics{
  1542. hasIPv6: hasIPv6,
  1543. hasPrivateIP: hasPrivateIP,
  1544. }
  1545. for candidateType := range candidateTypes {
  1546. metrics.iceCandidateTypes = append(metrics.iceCandidateTypes, candidateType)
  1547. }
  1548. for reason, count := range filteredCandidateReasons {
  1549. metrics.filteredICECandidates = append(metrics.filteredICECandidates,
  1550. fmt.Sprintf("%s: %d", reason, count))
  1551. }
  1552. return encodedSDP, metrics, nil
  1553. }
  1554. type pionLoggerFactory struct {
  1555. logger common.Logger
  1556. debugLogging bool
  1557. }
  1558. func newPionLoggerFactory(logger common.Logger, debugLogging bool) *pionLoggerFactory {
  1559. return &pionLoggerFactory{
  1560. logger: logger,
  1561. debugLogging: debugLogging,
  1562. }
  1563. }
  1564. func (f *pionLoggerFactory) NewLogger(scope string) pion_logging.LeveledLogger {
  1565. return newPionLogger(scope, f.logger, f.debugLogging)
  1566. }
  1567. // pionLogger wraps common.Logger and implements
  1568. // https://pkg.go.dev/github.com/pion/logging#LeveledLogger for passing into
  1569. // pion.
  1570. type pionLogger struct {
  1571. scope string
  1572. logger common.Logger
  1573. debugLogging bool
  1574. warnNoPairs int32
  1575. }
  1576. func newPionLogger(scope string, logger common.Logger, debugLogging bool) *pionLogger {
  1577. return &pionLogger{
  1578. scope: scope,
  1579. logger: logger,
  1580. debugLogging: debugLogging,
  1581. }
  1582. }
  1583. func (l *pionLogger) Trace(msg string) {
  1584. if !l.debugLogging {
  1585. return
  1586. }
  1587. l.logger.WithTrace().Debug(fmt.Sprintf("webRTC: %s: %s", l.scope, msg))
  1588. }
  1589. func (l *pionLogger) Tracef(format string, args ...interface{}) {
  1590. if !l.debugLogging {
  1591. return
  1592. }
  1593. l.logger.WithTrace().Debug(fmt.Sprintf("webRTC: %s: %s", l.scope, fmt.Sprintf(format, args...)))
  1594. }
  1595. func (l *pionLogger) Debug(msg string) {
  1596. if !l.debugLogging {
  1597. return
  1598. }
  1599. l.logger.WithTrace().Debug(fmt.Sprintf("[webRTC: %s: %s", l.scope, msg))
  1600. }
  1601. func (l *pionLogger) Debugf(format string, args ...interface{}) {
  1602. if !l.debugLogging {
  1603. return
  1604. }
  1605. l.logger.WithTrace().Debug(fmt.Sprintf("webRTC: %s: %s", l.scope, fmt.Sprintf(format, args...)))
  1606. }
  1607. func (l *pionLogger) Info(msg string) {
  1608. l.logger.WithTrace().Info(fmt.Sprintf("webRTC: %s: %s", l.scope, msg))
  1609. }
  1610. func (l *pionLogger) Infof(format string, args ...interface{}) {
  1611. l.logger.WithTrace().Info(fmt.Sprintf("webRTC: %s: %s", l.scope, fmt.Sprintf(format, args...)))
  1612. }
  1613. func (l *pionLogger) Warn(msg string) {
  1614. // To reduce diagnostic log noise, only log this message once per dial attempt.
  1615. if msg == "Failed to ping without candidate pairs. Connection is not possible yet." &&
  1616. !atomic.CompareAndSwapInt32(&l.warnNoPairs, 0, 1) {
  1617. return
  1618. }
  1619. l.logger.WithTrace().Warning(fmt.Sprintf("webRTC: %s: %s", l.scope, msg))
  1620. }
  1621. func (l *pionLogger) Warnf(format string, args ...interface{}) {
  1622. l.logger.WithTrace().Warning(fmt.Sprintf("webRTC: %s: %s", l.scope, fmt.Sprintf(format, args...)))
  1623. }
  1624. func (l *pionLogger) Error(msg string) {
  1625. l.logger.WithTrace().Error(fmt.Sprintf("webRTC: %s: %s", l.scope, msg))
  1626. }
  1627. func (l *pionLogger) Errorf(format string, args ...interface{}) {
  1628. l.logger.WithTrace().Error(fmt.Sprintf("webRTC: %s: %s", l.scope, fmt.Sprintf(format, args...)))
  1629. }
  1630. func hasInterfaceForPrivateIPAddress(IP net.IP) bool {
  1631. if !IP.IsPrivate() {
  1632. return false
  1633. }
  1634. // The anet package is used to work around net.Interfaces not working on
  1635. // Android at this time: https://github.com/golang/go/issues/40569.
  1636. //
  1637. // Any errors are silently dropped; the caller will proceed without using
  1638. // the input private IP; and equivilent anet calls are made in
  1639. // pionNetwork.Interfaces, with errors logged.
  1640. netInterfaces, err := anet.Interfaces()
  1641. if err != nil {
  1642. return false
  1643. }
  1644. for _, netInterface := range netInterfaces {
  1645. // Note: don't exclude interfaces with the net.FlagPointToPoint flag,
  1646. // which is set for certain mobile networks
  1647. if netInterface.Flags&net.FlagUp == 0 {
  1648. continue
  1649. }
  1650. addrs, err := anet.InterfaceAddrsByInterface(&netInterface)
  1651. if err != nil {
  1652. continue
  1653. }
  1654. for _, addr := range addrs {
  1655. _, IPNet, err := net.ParseCIDR(addr.String())
  1656. if err != nil {
  1657. continue
  1658. }
  1659. if IPNet.Contains(IP) {
  1660. return true
  1661. }
  1662. }
  1663. }
  1664. return false
  1665. }
  1666. // pionNetwork implements pion/transport.Net.
  1667. //
  1668. // Via the SettingsEngine, pion is configured to use a pionNetwork instance,
  1669. // which providing alternative implementations for various network functions.
  1670. // The Interfaces implementation provides a workaround for Android
  1671. // net.Interfaces issues and reduces the number of IPv6 candidates to avoid
  1672. // excess STUN requests; and the ResolveUDPAddr implementation hooks into the
  1673. // Psiphon custom resolver.
  1674. type pionNetwork struct {
  1675. dialCtx context.Context
  1676. logger pion_logging.LeveledLogger
  1677. webRTCDialCoordinator WebRTCDialCoordinator
  1678. }
  1679. func newPionNetwork(
  1680. dialCtx context.Context,
  1681. logger pion_logging.LeveledLogger,
  1682. webRTCDialCoordinator WebRTCDialCoordinator) *pionNetwork {
  1683. return &pionNetwork{
  1684. dialCtx: dialCtx,
  1685. logger: logger,
  1686. webRTCDialCoordinator: webRTCDialCoordinator,
  1687. }
  1688. }
  1689. func (p *pionNetwork) Interfaces() ([]*transport.Interface, error) {
  1690. // To determine the active IPv4 and IPv6 interfaces, let the OS bind IPv4
  1691. // and IPv6 UDP sockets with a specified external destination address.
  1692. // Then iterate over all interfaces, but return interface info for only
  1693. // the interfaces those sockets were bound to.
  1694. //
  1695. // The destination IPs are the IPs that currently resolve for example.com.
  1696. // No actual traffic to these IPs or example.com is sent, as the UDP
  1697. // sockets are not used to send any packets.
  1698. //
  1699. // This scheme should select just one IPv4 and one IPv6 address, which
  1700. // should be the active, externally routable addresses, and the IPv6
  1701. // address should be the preferred, non-deprecated temporary IPv6 address.
  1702. //
  1703. // In post-ICE gathering processing, processSDPAddresses will also strip
  1704. // all bogon addresses, so there is no explicit bogon check here.
  1705. //
  1706. // Limitations:
  1707. //
  1708. // - The active interface could change between the socket operation and
  1709. // iterating over all interfaces. Higher-level code is expected to
  1710. // react to active network changes.
  1711. //
  1712. // - The public IPs for example.com may not be robust in all routing
  1713. // situations. Alternatively, we could use the configured STUN server
  1714. // as the test destination, but the STUN server domain is not resolved
  1715. // at this point and STUN is not always configured and used.
  1716. //
  1717. // - The results could be cached and reused.
  1718. var defaultIPv4, defaultIPv6 net.IP
  1719. udpConnIPv4, err := p.webRTCDialCoordinator.UDPConn(
  1720. context.Background(), "udp4", "93.184.216.34:3478")
  1721. if err == nil {
  1722. defaultIPv4 = udpConnIPv4.LocalAddr().(*net.UDPAddr).IP
  1723. udpConnIPv4.Close()
  1724. }
  1725. udpConnIPv6, err := p.webRTCDialCoordinator.UDPConn(
  1726. context.Background(), "udp6", "[2606:2800:220:1:248:1893:25c8:1946]:3478")
  1727. if err == nil {
  1728. defaultIPv6 = udpConnIPv6.LocalAddr().(*net.UDPAddr).IP
  1729. udpConnIPv6.Close()
  1730. }
  1731. // The anet package is used to work around net.Interfaces not working on
  1732. // Android at this time: https://github.com/golang/go/issues/40569.
  1733. transportInterfaces := []*transport.Interface{}
  1734. netInterfaces, err := anet.Interfaces()
  1735. if err != nil {
  1736. return nil, errors.Trace(err)
  1737. }
  1738. for _, netInterface := range netInterfaces {
  1739. // Note: don't exclude interfaces with the net.FlagPointToPoint flag,
  1740. // which is set for certain mobile networks
  1741. if (netInterface.Flags&net.FlagUp == 0) ||
  1742. (!GetAllowBogonWebRTCConnections() && (netInterface.Flags&net.FlagLoopback != 0)) {
  1743. continue
  1744. }
  1745. addrs, err := anet.InterfaceAddrsByInterface(&netInterface)
  1746. if err != nil {
  1747. return nil, errors.Trace(err)
  1748. }
  1749. var transportInterface *transport.Interface
  1750. for _, addr := range addrs {
  1751. IP, _, err := net.ParseCIDR(addr.String())
  1752. if err != nil {
  1753. return nil, errors.Trace(err)
  1754. }
  1755. if IP.Equal(defaultIPv4) || IP.Equal(defaultIPv6) ||
  1756. (GetAllowBogonWebRTCConnections() && (netInterface.Flags&net.FlagLoopback != 0)) {
  1757. if transportInterface == nil {
  1758. transportInterface = transport.NewInterface(netInterface)
  1759. }
  1760. transportInterface.AddAddress(addr)
  1761. }
  1762. }
  1763. if transportInterface != nil {
  1764. transportInterfaces = append(transportInterfaces, transportInterface)
  1765. }
  1766. }
  1767. return transportInterfaces, nil
  1768. }
  1769. func (p *pionNetwork) ResolveUDPAddr(network, address string) (retAddr *net.UDPAddr, retErr error) {
  1770. defer func() {
  1771. if retErr != nil {
  1772. // Explicitly log an error since certain pion operations -- e.g.,
  1773. // ICE gathering -- don't propagate all pion/transport.Net errors.
  1774. p.logger.Errorf("pionNetwork.ResolveUDPAddr failed: %v", retErr)
  1775. }
  1776. }()
  1777. // Currently, pion appears to call ResolveUDPAddr with "udp4"/udp6"
  1778. // instead of "ip4"/"ip6", as expected by, e.g., net.Resolver.LookupIP.
  1779. // Convert to "ip4"/"ip6".
  1780. // Specifying v4/v6 ensures that the resolved IP address is the correct
  1781. // type. In the case of STUN servers, the correct type is required in
  1782. // order to create the correct IPv4 or IPv6 whole punch address.
  1783. switch network {
  1784. case "udp4", "tcp4":
  1785. network = "ip4"
  1786. case "udp6", "tcp6":
  1787. network = "ip6"
  1788. default:
  1789. network = "ip"
  1790. }
  1791. // Currently, pion appears to call ResolveUDPAddr with an improperly
  1792. // formatted address, <IPv6>:443 not [<IPv6>]:443; handle this case.
  1793. index := strings.LastIndex(address, ":")
  1794. if index != -1 {
  1795. address = net.JoinHostPort(address[:index], address[index+1:])
  1796. }
  1797. // Use the Psiphon custom resolver to resolve any STUN server domains.
  1798. resolvedAddress, err := p.webRTCDialCoordinator.ResolveAddress(
  1799. p.dialCtx, network, address)
  1800. if err != nil {
  1801. return nil, errors.Trace(err)
  1802. }
  1803. IPStr, portStr, err := net.SplitHostPort(resolvedAddress)
  1804. if err != nil {
  1805. return nil, errors.Trace(err)
  1806. }
  1807. IP := net.ParseIP(IPStr)
  1808. if IP == nil {
  1809. return nil, errors.TraceNew("invalid IP address")
  1810. }
  1811. port, err := strconv.Atoi(portStr)
  1812. if err != nil {
  1813. return nil, errors.Trace(err)
  1814. }
  1815. return &net.UDPAddr{IP: IP, Port: port}, nil
  1816. }
  1817. var errNotSupported = std_errors.New("not supported")
  1818. func (p *pionNetwork) ListenPacket(network string, address string) (net.PacketConn, error) {
  1819. // Explicitly log an error since certain pion operations -- e.g., ICE
  1820. // gathering -- don't propagate all pion/transport.Net errors.
  1821. p.logger.Errorf("unexpected pionNetwork.ListenPacket call from %s", stacktrace.GetParentFunctionName())
  1822. return nil, errors.Trace(errNotSupported)
  1823. }
  1824. func (p *pionNetwork) ListenUDP(network string, locAddr *net.UDPAddr) (transport.UDPConn, error) {
  1825. p.logger.Errorf("unexpected pionNetwork.ListenUDP call from %s", stacktrace.GetParentFunctionName())
  1826. return nil, errors.Trace(errNotSupported)
  1827. }
  1828. func (p *pionNetwork) ListenTCP(network string, laddr *net.TCPAddr) (transport.TCPListener, error) {
  1829. p.logger.Errorf("unexpected pionNetwork.ListenTCP call from %s", stacktrace.GetParentFunctionName())
  1830. return nil, errors.Trace(errNotSupported)
  1831. }
  1832. func (p *pionNetwork) Dial(network, address string) (net.Conn, error) {
  1833. p.logger.Errorf("unexpected pionNetwork.Dial call from %s", stacktrace.GetParentFunctionName())
  1834. return nil, errors.Trace(errNotSupported)
  1835. }
  1836. func (p *pionNetwork) DialUDP(network string, laddr, raddr *net.UDPAddr) (transport.UDPConn, error) {
  1837. p.logger.Errorf("unexpected pionNetwork.DialUDP call from %s", stacktrace.GetParentFunctionName())
  1838. return nil, errors.Trace(errNotSupported)
  1839. }
  1840. func (p *pionNetwork) DialTCP(network string, laddr, raddr *net.TCPAddr) (transport.TCPConn, error) {
  1841. p.logger.Errorf("unexpected pionNetwork.DialTCP call from %s", stacktrace.GetParentFunctionName())
  1842. return nil, errors.Trace(errNotSupported)
  1843. }
  1844. func (p *pionNetwork) ResolveIPAddr(network, address string) (*net.IPAddr, error) {
  1845. p.logger.Errorf("unexpected pionNetwork.ResolveIPAddr call from %s", stacktrace.GetParentFunctionName())
  1846. return nil, errors.Trace(errNotSupported)
  1847. }
  1848. func (p *pionNetwork) ResolveTCPAddr(network, address string) (*net.TCPAddr, error) {
  1849. p.logger.Errorf("unexpected pionNetwork.ResolveTCPAddr call from %s", stacktrace.GetParentFunctionName())
  1850. return nil, errors.Trace(errNotSupported)
  1851. }
  1852. func (p *pionNetwork) InterfaceByIndex(index int) (*transport.Interface, error) {
  1853. p.logger.Errorf("unexpected pionNetwork.InterfaceByIndex call from %s", stacktrace.GetParentFunctionName())
  1854. return nil, errors.Trace(errNotSupported)
  1855. }
  1856. func (p *pionNetwork) InterfaceByName(name string) (*transport.Interface, error) {
  1857. p.logger.Errorf("unexpected pionNetwork.InterfaceByName call from %s", stacktrace.GetParentFunctionName())
  1858. return nil, errors.Trace(errNotSupported)
  1859. }
  1860. func (p *pionNetwork) CreateDialer(dialer *net.Dialer) transport.Dialer {
  1861. return &pionNetworkDialer{pionNetwork: p}
  1862. }
  1863. type pionNetworkDialer struct {
  1864. pionNetwork *pionNetwork
  1865. }
  1866. func (d pionNetworkDialer) Dial(network, address string) (net.Conn, error) {
  1867. d.pionNetwork.logger.Errorf("unexpected pionNetworkDialer.Dial call from %s", stacktrace.GetParentFunctionName())
  1868. return nil, errors.Trace(errNotSupported)
  1869. }