serverEntry.go 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761
  1. /*
  2. * Copyright (c) 2015, Psiphon Inc.
  3. * All rights reserved.
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation, either version 3 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. *
  18. */
  19. package protocol
  20. import (
  21. "bufio"
  22. "bytes"
  23. "crypto/hmac"
  24. "crypto/rand"
  25. "crypto/sha256"
  26. "encoding/base64"
  27. "encoding/hex"
  28. "encoding/json"
  29. "errors"
  30. "fmt"
  31. "io"
  32. "net"
  33. "strings"
  34. "time"
  35. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
  36. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/ed25519"
  37. )
  38. // ServerEntry represents a Psiphon server. It contains information
  39. // about how to establish a tunnel connection to the server through
  40. // several protocols. Server entries are JSON records downloaded from
  41. // various sources.
  42. type ServerEntry struct {
  43. Tag string `json:"tag"`
  44. IpAddress string `json:"ipAddress"`
  45. WebServerPort string `json:"webServerPort"` // not an int
  46. WebServerSecret string `json:"webServerSecret"`
  47. WebServerCertificate string `json:"webServerCertificate"`
  48. SshPort int `json:"sshPort"`
  49. SshUsername string `json:"sshUsername"`
  50. SshPassword string `json:"sshPassword"`
  51. SshHostKey string `json:"sshHostKey"`
  52. SshObfuscatedPort int `json:"sshObfuscatedPort"`
  53. SshObfuscatedQUICPort int `json:"sshObfuscatedQUICPort"`
  54. SshObfuscatedTapdancePort int `json:"sshObfuscatedTapdancePort"`
  55. SshObfuscatedKey string `json:"sshObfuscatedKey"`
  56. Capabilities []string `json:"capabilities"`
  57. Region string `json:"region"`
  58. MeekServerPort int `json:"meekServerPort"`
  59. MeekCookieEncryptionPublicKey string `json:"meekCookieEncryptionPublicKey"`
  60. MeekObfuscatedKey string `json:"meekObfuscatedKey"`
  61. MeekFrontingHost string `json:"meekFrontingHost"`
  62. MeekFrontingHosts []string `json:"meekFrontingHosts"`
  63. MeekFrontingDomain string `json:"meekFrontingDomain"`
  64. MeekFrontingAddresses []string `json:"meekFrontingAddresses"`
  65. MeekFrontingAddressesRegex string `json:"meekFrontingAddressesRegex"`
  66. MeekFrontingDisableSNI bool `json:"meekFrontingDisableSNI"`
  67. TacticsRequestPublicKey string `json:"tacticsRequestPublicKey"`
  68. TacticsRequestObfuscatedKey string `json:"tacticsRequestObfuscatedKey"`
  69. MarionetteFormat string `json:"marionetteFormat"`
  70. ConfigurationVersion int `json:"configurationVersion"`
  71. Signature string `json:"signature"`
  72. // These local fields are not expected to be present in downloaded server
  73. // entries. They are added by the client to record and report stats about
  74. // how and when server entries are obtained.
  75. // All local fields should be included the list of fields in RemoveUnsignedFields.
  76. LocalSource string `json:"localSource,omitempty"`
  77. LocalTimestamp string `json:"localTimestamp,omitempty"`
  78. IsLocalDerivedTag bool `json:"isLocalDerivedTag,omitempty"`
  79. }
  80. // ServerEntryFields is an alternate representation of ServerEntry which
  81. // enables future compatibility when unmarshaling and persisting new server
  82. // entries which may contain new, unrecognized fields not in the ServerEntry
  83. // type for a particular client version.
  84. //
  85. // When new JSON server entries with new fields are unmarshaled to ServerEntry
  86. // types, unrecognized fields are discarded. When unmarshaled to
  87. // ServerEntryFields, unrecognized fields are retained and may be persisted
  88. // and available when the client is upgraded and unmarshals to an updated
  89. // ServerEntry type.
  90. type ServerEntryFields map[string]interface{}
  91. // GetServerEntry converts a ServerEntryFields into a ServerEntry.
  92. func (fields ServerEntryFields) GetServerEntry() (*ServerEntry, error) {
  93. marshaledServerEntry, err := json.Marshal(fields)
  94. if err != nil {
  95. return nil, common.ContextError(err)
  96. }
  97. var serverEntry *ServerEntry
  98. err = json.Unmarshal(marshaledServerEntry, &serverEntry)
  99. if err != nil {
  100. return nil, common.ContextError(err)
  101. }
  102. return serverEntry, nil
  103. }
  104. func (fields ServerEntryFields) GetTag() string {
  105. tag, ok := fields["tag"]
  106. if !ok {
  107. return ""
  108. }
  109. tagStr, ok := tag.(string)
  110. if !ok {
  111. return ""
  112. }
  113. return tagStr
  114. }
  115. // SetTag sets a local, derived server entry tag. A tag is an identifier used
  116. // in server entry pruning and potentially other use cases. An explict tag,
  117. // set by the Psiphon Network, may be present in a server entry that is
  118. // imported; otherwise, the client will set a derived tag. The tag should be
  119. // generated using GenerateServerEntryTag. When SetTag finds a explicit tag,
  120. // the new, derived tag is ignored. The isLocalTag local field is set to
  121. // distinguish explict and derived tags and is used in signature verification
  122. // to determine if the tag field is part of the signature.
  123. func (fields ServerEntryFields) SetTag(tag string) {
  124. // Don't replace explicit tag
  125. if tag, ok := fields["tag"]; ok {
  126. tagStr, ok := tag.(string)
  127. if ok && tagStr != "" {
  128. isLocalDerivedTag, ok := fields["isLocalDerivedTag"]
  129. if !ok {
  130. return
  131. }
  132. isLocalDerivedTagBool, ok := isLocalDerivedTag.(bool)
  133. if ok && !isLocalDerivedTagBool {
  134. return
  135. }
  136. }
  137. }
  138. fields["tag"] = tag
  139. // Mark this tag as local
  140. fields["isLocalDerivedTag"] = true
  141. }
  142. func (fields ServerEntryFields) GetIPAddress() string {
  143. ipAddress, ok := fields["ipAddress"]
  144. if !ok {
  145. return ""
  146. }
  147. ipAddressStr, ok := ipAddress.(string)
  148. if !ok {
  149. return ""
  150. }
  151. return ipAddressStr
  152. }
  153. func (fields ServerEntryFields) GetWebServerPort() string {
  154. webServerPort, ok := fields["webServerPort"]
  155. if !ok {
  156. return ""
  157. }
  158. webServerPortStr, ok := webServerPort.(string)
  159. if !ok {
  160. return ""
  161. }
  162. return webServerPortStr
  163. }
  164. func (fields ServerEntryFields) GetWebServerSecret() string {
  165. webServerSecret, ok := fields["webServerSecret"]
  166. if !ok {
  167. return ""
  168. }
  169. webServerSecretStr, ok := webServerSecret.(string)
  170. if !ok {
  171. return ""
  172. }
  173. return webServerSecretStr
  174. }
  175. func (fields ServerEntryFields) GetWebServerCertificate() string {
  176. webServerCertificate, ok := fields["webServerCertificate"]
  177. if !ok {
  178. return ""
  179. }
  180. webServerCertificateStr, ok := webServerCertificate.(string)
  181. if !ok {
  182. return ""
  183. }
  184. return webServerCertificateStr
  185. }
  186. func (fields ServerEntryFields) GetConfigurationVersion() int {
  187. configurationVersion, ok := fields["configurationVersion"]
  188. if !ok {
  189. return 0
  190. }
  191. configurationVersionFloat, ok := configurationVersion.(float64)
  192. if !ok {
  193. return 0
  194. }
  195. return int(configurationVersionFloat)
  196. }
  197. func (fields ServerEntryFields) GetLocalSource() string {
  198. localSource, ok := fields["localSource"]
  199. if !ok {
  200. return ""
  201. }
  202. localSourceStr, ok := localSource.(string)
  203. if !ok {
  204. return ""
  205. }
  206. return localSourceStr
  207. }
  208. func (fields ServerEntryFields) SetLocalSource(source string) {
  209. fields["localSource"] = source
  210. }
  211. func (fields ServerEntryFields) GetLocalTimestamp() string {
  212. localTimestamp, ok := fields["localTimestamp"]
  213. if !ok {
  214. return ""
  215. }
  216. localTimestampStr, ok := localTimestamp.(string)
  217. if !ok {
  218. return ""
  219. }
  220. return localTimestampStr
  221. }
  222. func (fields ServerEntryFields) SetLocalTimestamp(timestamp string) {
  223. fields["localTimestamp"] = timestamp
  224. }
  225. func (fields ServerEntryFields) HasSignature() bool {
  226. signature, ok := fields["signature"]
  227. if !ok {
  228. return false
  229. }
  230. signatureStr, ok := signature.(string)
  231. if !ok {
  232. return false
  233. }
  234. return signatureStr != ""
  235. }
  236. const signaturePublicKeyDigestSize = 8
  237. // AddSignature signs a server entry and attaches a new field containing the
  238. // signature. Any existing "signature" field will be replaced.
  239. //
  240. // The signature incudes a public key ID that is derived from a digest of the
  241. // public key value. This ID is intended for future use when multiple signing
  242. // keys may be deployed.
  243. func (fields ServerEntryFields) AddSignature(publicKey, privateKey string) error {
  244. // Make a copy so that removing unsigned fields will have no side effects
  245. copyFields := make(ServerEntryFields)
  246. for k, v := range fields {
  247. copyFields[k] = v
  248. }
  249. copyFields.RemoveUnsignedFields()
  250. delete(copyFields, "signature")
  251. marshaledFields, err := json.Marshal(copyFields)
  252. if err != nil {
  253. return common.ContextError(err)
  254. }
  255. decodedPublicKey, err := base64.StdEncoding.DecodeString(publicKey)
  256. if err != nil {
  257. return common.ContextError(err)
  258. }
  259. publicKeyDigest := sha256.Sum256(decodedPublicKey)
  260. publicKeyID := publicKeyDigest[:signaturePublicKeyDigestSize]
  261. decodedPrivateKey, err := base64.StdEncoding.DecodeString(privateKey)
  262. if err != nil {
  263. return common.ContextError(err)
  264. }
  265. signature := ed25519.Sign(decodedPrivateKey, marshaledFields)
  266. fields["signature"] = base64.StdEncoding.EncodeToString(
  267. append(publicKeyID, signature...))
  268. return nil
  269. }
  270. // VerifySignature verifies the signature set by AddSignature.
  271. //
  272. // VerifySignature must be called before using any server entry that is
  273. // imported from an untrusted source, such as client-to-client exchange.
  274. func (fields ServerEntryFields) VerifySignature(publicKey string) error {
  275. // Make a copy so that removing unsigned fields will have no side effects
  276. copyFields := make(ServerEntryFields)
  277. for k, v := range fields {
  278. copyFields[k] = v
  279. }
  280. signatureField, ok := copyFields["signature"]
  281. if !ok {
  282. return common.ContextError(errors.New("missing signature field"))
  283. }
  284. signatureFieldStr, ok := signatureField.(string)
  285. if !ok {
  286. return common.ContextError(errors.New("invalid signature field"))
  287. }
  288. decodedSignatureField, err := base64.StdEncoding.DecodeString(signatureFieldStr)
  289. if err != nil {
  290. return common.ContextError(err)
  291. }
  292. if len(decodedSignatureField) < signaturePublicKeyDigestSize {
  293. return common.ContextError(errors.New("invalid signature field length"))
  294. }
  295. publicKeyID := decodedSignatureField[:signaturePublicKeyDigestSize]
  296. signature := decodedSignatureField[signaturePublicKeyDigestSize:]
  297. if len(signature) != ed25519.SignatureSize {
  298. return common.ContextError(errors.New("invalid signature length"))
  299. }
  300. decodedPublicKey, err := base64.StdEncoding.DecodeString(publicKey)
  301. if err != nil {
  302. return common.ContextError(err)
  303. }
  304. publicKeyDigest := sha256.Sum256(decodedPublicKey)
  305. expectedPublicKeyID := publicKeyDigest[:signaturePublicKeyDigestSize]
  306. if bytes.Compare(expectedPublicKeyID, publicKeyID) != 0 {
  307. return common.ContextError(errors.New("unexpected public key ID"))
  308. }
  309. copyFields.RemoveUnsignedFields()
  310. delete(copyFields, "signature")
  311. marshaledFields, err := json.Marshal(copyFields)
  312. if err != nil {
  313. return common.ContextError(err)
  314. }
  315. if !ed25519.Verify(decodedPublicKey, marshaledFields, signature) {
  316. return common.ContextError(errors.New("invalid signature"))
  317. }
  318. return nil
  319. }
  320. // RemoveUnsignedFields prepares a server entry for signing or signature
  321. // verification by removing unsigned fields. The JSON marshalling of the
  322. // remaining fields is the data that is signed.
  323. func (fields ServerEntryFields) RemoveUnsignedFields() {
  324. delete(fields, "localSource")
  325. delete(fields, "localTimestamp")
  326. // Only non-local, explicit tags are part of the signature
  327. isLocalDerivedTag, _ := fields["isLocalDerivedTag"]
  328. isLocalDerivedTagBool, ok := isLocalDerivedTag.(bool)
  329. if ok && isLocalDerivedTagBool {
  330. delete(fields, "tag")
  331. }
  332. delete(fields, "isLocalDerivedTag")
  333. }
  334. // NewServerEntrySignatureKeyPair creates an ed25519 key pair for use in
  335. // server entry signing and verification.
  336. func NewServerEntrySignatureKeyPair() (string, string, error) {
  337. publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
  338. if err != nil {
  339. return "", "", common.ContextError(err)
  340. }
  341. return base64.StdEncoding.EncodeToString(publicKey),
  342. base64.StdEncoding.EncodeToString(privateKey),
  343. nil
  344. }
  345. // GetCapability returns the server capability corresponding
  346. // to the tunnel protocol.
  347. func GetCapability(protocol string) string {
  348. return strings.TrimSuffix(protocol, "-OSSH")
  349. }
  350. // GetTacticsCapability returns the server tactics capability
  351. // corresponding to the tunnel protocol.
  352. func GetTacticsCapability(protocol string) string {
  353. return GetCapability(protocol) + "-TACTICS"
  354. }
  355. // SupportsProtocol returns true if and only if the ServerEntry has
  356. // the necessary capability to support the specified tunnel protocol.
  357. func (serverEntry *ServerEntry) SupportsProtocol(protocol string) bool {
  358. requiredCapability := GetCapability(protocol)
  359. return common.Contains(serverEntry.Capabilities, requiredCapability)
  360. }
  361. // GetSupportedProtocols returns a list of tunnel protocols supported
  362. // by the ServerEntry's capabilities.
  363. func (serverEntry *ServerEntry) GetSupportedProtocols(
  364. useUpstreamProxy bool,
  365. limitTunnelProtocols []string,
  366. excludeIntensive bool) []string {
  367. supportedProtocols := make([]string, 0)
  368. for _, protocol := range SupportedTunnelProtocols {
  369. // TODO: Marionette UDP formats are incompatible with
  370. // useUpstreamProxy, but not currently supported
  371. if useUpstreamProxy && TunnelProtocolUsesQUIC(protocol) {
  372. continue
  373. }
  374. if len(limitTunnelProtocols) > 0 {
  375. if !common.Contains(limitTunnelProtocols, protocol) {
  376. continue
  377. }
  378. } else {
  379. if common.Contains(DefaultDisabledTunnelProtocols, protocol) {
  380. continue
  381. }
  382. }
  383. if excludeIntensive && TunnelProtocolIsResourceIntensive(protocol) {
  384. continue
  385. }
  386. if serverEntry.SupportsProtocol(protocol) {
  387. supportedProtocols = append(supportedProtocols, protocol)
  388. }
  389. }
  390. return supportedProtocols
  391. }
  392. // GetSupportedTacticsProtocols returns a list of tunnel protocols,
  393. // supported by the ServerEntry's capabilities, that may be used
  394. // for tactics requests.
  395. func (serverEntry *ServerEntry) GetSupportedTacticsProtocols() []string {
  396. supportedProtocols := make([]string, 0)
  397. for _, protocol := range SupportedTunnelProtocols {
  398. if !TunnelProtocolUsesMeek(protocol) {
  399. continue
  400. }
  401. requiredCapability := GetTacticsCapability(protocol)
  402. if !common.Contains(serverEntry.Capabilities, requiredCapability) {
  403. continue
  404. }
  405. supportedProtocols = append(supportedProtocols, protocol)
  406. }
  407. return supportedProtocols
  408. }
  409. // SupportsSSHAPIRequests returns true when the server supports
  410. // SSH API requests.
  411. func (serverEntry *ServerEntry) SupportsSSHAPIRequests() bool {
  412. return common.Contains(serverEntry.Capabilities, CAPABILITY_SSH_API_REQUESTS)
  413. }
  414. func (serverEntry *ServerEntry) GetUntunneledWebRequestPorts() []string {
  415. ports := make([]string, 0)
  416. if common.Contains(serverEntry.Capabilities, CAPABILITY_UNTUNNELED_WEB_API_REQUESTS) {
  417. // Server-side configuration quirk: there's a port forward from
  418. // port 443 to the web server, which we can try, except on servers
  419. // running FRONTED_MEEK, which listens on port 443.
  420. if !serverEntry.SupportsProtocol(TUNNEL_PROTOCOL_FRONTED_MEEK) {
  421. ports = append(ports, "443")
  422. }
  423. ports = append(ports, serverEntry.WebServerPort)
  424. }
  425. return ports
  426. }
  427. func (serverEntry *ServerEntry) HasSignature() bool {
  428. return serverEntry.Signature != ""
  429. }
  430. // GenerateServerEntryTag creates a server entry tag value that is
  431. // cryptographically derived from the IP address and web server secret in a
  432. // way that is difficult to reverse the IP address value from the tag or
  433. // compute the tag without having the web server secret, a 256-bit random
  434. // value which is unique per server, in addition to the IP address. A database
  435. // consisting only of server entry tags should be resistent to an attack that
  436. // attempts to reverse all the server IPs, even given a small IP space (IPv4),
  437. // or some subset of the web server secrets.
  438. func GenerateServerEntryTag(ipAddress, webServerSecret string) string {
  439. h := hmac.New(sha256.New, []byte(webServerSecret))
  440. h.Write([]byte(ipAddress))
  441. return base64.StdEncoding.EncodeToString(h.Sum(nil))
  442. }
  443. // EncodeServerEntry returns a string containing the encoding of
  444. // a ServerEntry following Psiphon conventions.
  445. func EncodeServerEntry(serverEntry *ServerEntry) (string, error) {
  446. return encodeServerEntry(
  447. serverEntry.IpAddress,
  448. serverEntry.WebServerPort,
  449. serverEntry.WebServerSecret,
  450. serverEntry.WebServerCertificate,
  451. serverEntry)
  452. }
  453. // EncodeServerEntryFields returns a string containing the encoding of
  454. // ServerEntryFields following Psiphon conventions.
  455. func EncodeServerEntryFields(serverEntryFields ServerEntryFields) (string, error) {
  456. return encodeServerEntry(
  457. serverEntryFields.GetIPAddress(),
  458. serverEntryFields.GetWebServerPort(),
  459. serverEntryFields.GetWebServerSecret(),
  460. serverEntryFields.GetWebServerCertificate(),
  461. serverEntryFields)
  462. }
  463. func encodeServerEntry(
  464. IPAddress, webServerPort, webServerSecret, webServerCertificate string,
  465. serverEntry interface{}) (string, error) {
  466. serverEntryJSON, err := json.Marshal(serverEntry)
  467. if err != nil {
  468. return "", common.ContextError(err)
  469. }
  470. // Legacy clients expect the space-delimited fields.
  471. return hex.EncodeToString([]byte(fmt.Sprintf(
  472. "%s %s %s %s %s",
  473. IPAddress,
  474. webServerPort,
  475. webServerSecret,
  476. webServerCertificate,
  477. serverEntryJSON))), nil
  478. }
  479. // DecodeServerEntry extracts a server entry from the encoding
  480. // used by remote server lists and Psiphon server handshake requests.
  481. //
  482. // The resulting ServerEntry.LocalSource is populated with serverEntrySource,
  483. // which should be one of SERVER_ENTRY_SOURCE_EMBEDDED, SERVER_ENTRY_SOURCE_REMOTE,
  484. // SERVER_ENTRY_SOURCE_DISCOVERY, SERVER_ENTRY_SOURCE_TARGET,
  485. // SERVER_ENTRY_SOURCE_OBFUSCATED.
  486. // ServerEntry.LocalTimestamp is populated with the provided timestamp, which
  487. // should be a RFC 3339 formatted string. These local fields are stored with the
  488. // server entry and reported to the server as stats (a coarse granularity timestamp
  489. // is reported).
  490. func DecodeServerEntry(
  491. encodedServerEntry, timestamp, serverEntrySource string) (*ServerEntry, error) {
  492. serverEntry := new(ServerEntry)
  493. err := decodeServerEntry(encodedServerEntry, timestamp, serverEntrySource, serverEntry)
  494. if err != nil {
  495. return nil, common.ContextError(err)
  496. }
  497. // NOTE: if the source JSON happens to have values in these fields, they get clobbered.
  498. serverEntry.LocalSource = serverEntrySource
  499. serverEntry.LocalTimestamp = timestamp
  500. return serverEntry, nil
  501. }
  502. // DecodeServerEntryFields extracts an encoded server entry into a
  503. // ServerEntryFields type, much like DecodeServerEntry. Unrecognized fields
  504. // not in ServerEntry are retained in the ServerEntryFields.
  505. //
  506. // LocalSource/LocalTimestamp map entries are set only when the corresponding
  507. // inputs are non-blank.
  508. func DecodeServerEntryFields(
  509. encodedServerEntry, timestamp, serverEntrySource string) (ServerEntryFields, error) {
  510. serverEntryFields := make(ServerEntryFields)
  511. err := decodeServerEntry(encodedServerEntry, timestamp, serverEntrySource, &serverEntryFields)
  512. if err != nil {
  513. return nil, common.ContextError(err)
  514. }
  515. // NOTE: if the source JSON happens to have values in these fields, they get clobbered.
  516. if serverEntrySource != "" {
  517. serverEntryFields.SetLocalSource(serverEntrySource)
  518. }
  519. if timestamp != "" {
  520. serverEntryFields.SetLocalTimestamp(timestamp)
  521. }
  522. return serverEntryFields, nil
  523. }
  524. func decodeServerEntry(
  525. encodedServerEntry, timestamp, serverEntrySource string,
  526. target interface{}) error {
  527. hexDecodedServerEntry, err := hex.DecodeString(encodedServerEntry)
  528. if err != nil {
  529. return common.ContextError(err)
  530. }
  531. // Skip past legacy format (4 space delimited fields) and just parse the JSON config
  532. fields := bytes.SplitN(hexDecodedServerEntry, []byte(" "), 5)
  533. if len(fields) != 5 {
  534. return common.ContextError(errors.New("invalid encoded server entry"))
  535. }
  536. err = json.Unmarshal(fields[4], target)
  537. if err != nil {
  538. return common.ContextError(err)
  539. }
  540. return nil
  541. }
  542. // ValidateServerEntryFields checks for malformed server entries.
  543. func ValidateServerEntryFields(serverEntryFields ServerEntryFields) error {
  544. // Checks for a valid ipAddress. This is important since the IP
  545. // address is the key used to store/lookup the server entry.
  546. ipAddress := serverEntryFields.GetIPAddress()
  547. if net.ParseIP(ipAddress) == nil {
  548. return common.ContextError(
  549. fmt.Errorf("server entry has invalid ipAddress: %s", ipAddress))
  550. }
  551. // TODO: validate more fields?
  552. // Ensure locally initialized fields have been set.
  553. source := serverEntryFields.GetLocalSource()
  554. if !common.Contains(
  555. SupportedServerEntrySources, source) {
  556. return common.ContextError(
  557. fmt.Errorf("server entry has invalid source: %s", source))
  558. }
  559. timestamp := serverEntryFields.GetLocalTimestamp()
  560. _, err := time.Parse(time.RFC3339, timestamp)
  561. if err != nil {
  562. return common.ContextError(
  563. fmt.Errorf("server entry has invalid timestamp: %s", err))
  564. }
  565. return nil
  566. }
  567. // DecodeServerEntryList extracts server entries from the list encoding
  568. // used by remote server lists and Psiphon server handshake requests.
  569. // Each server entry is validated and invalid entries are skipped.
  570. // See DecodeServerEntry for note on serverEntrySource/timestamp.
  571. func DecodeServerEntryList(
  572. encodedServerEntryList, timestamp,
  573. serverEntrySource string) ([]ServerEntryFields, error) {
  574. serverEntries := make([]ServerEntryFields, 0)
  575. for _, encodedServerEntry := range strings.Split(encodedServerEntryList, "\n") {
  576. if len(encodedServerEntry) == 0 {
  577. continue
  578. }
  579. // TODO: skip this entry and continue if can't decode?
  580. serverEntryFields, err := DecodeServerEntryFields(encodedServerEntry, timestamp, serverEntrySource)
  581. if err != nil {
  582. return nil, common.ContextError(err)
  583. }
  584. if ValidateServerEntryFields(serverEntryFields) != nil {
  585. // Skip this entry and continue with the next one
  586. // TODO: invoke a logging callback
  587. continue
  588. }
  589. serverEntries = append(serverEntries, serverEntryFields)
  590. }
  591. return serverEntries, nil
  592. }
  593. // StreamingServerEntryDecoder performs the DecodeServerEntryList
  594. // operation, loading only one server entry into memory at a time.
  595. type StreamingServerEntryDecoder struct {
  596. scanner *bufio.Scanner
  597. timestamp string
  598. serverEntrySource string
  599. }
  600. // NewStreamingServerEntryDecoder creates a new StreamingServerEntryDecoder.
  601. func NewStreamingServerEntryDecoder(
  602. encodedServerEntryListReader io.Reader,
  603. timestamp, serverEntrySource string) *StreamingServerEntryDecoder {
  604. return &StreamingServerEntryDecoder{
  605. scanner: bufio.NewScanner(encodedServerEntryListReader),
  606. timestamp: timestamp,
  607. serverEntrySource: serverEntrySource,
  608. }
  609. }
  610. // Next reads and decodes, and validates the next server entry from the
  611. // input stream, returning a nil server entry when the stream is complete.
  612. //
  613. // Limitations:
  614. // - Each encoded server entry line cannot exceed bufio.MaxScanTokenSize,
  615. // the default buffer size which this decoder uses. This is 64K.
  616. // - DecodeServerEntry is called on each encoded server entry line, which
  617. // will allocate memory to hex decode and JSON deserialze the server
  618. // entry. As this is not presently reusing a fixed buffer, each call
  619. // will allocate additional memory; garbage collection is necessary to
  620. // reclaim that memory for reuse for the next server entry.
  621. //
  622. func (decoder *StreamingServerEntryDecoder) Next() (ServerEntryFields, error) {
  623. for {
  624. if !decoder.scanner.Scan() {
  625. return nil, common.ContextError(decoder.scanner.Err())
  626. }
  627. // TODO: use scanner.Bytes which doesn't allocate, instead of scanner.Text
  628. // TODO: skip this entry and continue if can't decode?
  629. serverEntryFields, err := DecodeServerEntryFields(
  630. decoder.scanner.Text(), decoder.timestamp, decoder.serverEntrySource)
  631. if err != nil {
  632. return nil, common.ContextError(err)
  633. }
  634. if ValidateServerEntryFields(serverEntryFields) != nil {
  635. // Skip this entry and continue with the next one
  636. // TODO: invoke a logging callback
  637. continue
  638. }
  639. return serverEntryFields, nil
  640. }
  641. }