api.go 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957
  1. /*
  2. * Copyright (c) 2016, Psiphon Inc.
  3. * All rights reserved.
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation, either version 3 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  17. *
  18. */
  19. package server
  20. import (
  21. "crypto/subtle"
  22. "encoding/json"
  23. "errors"
  24. "fmt"
  25. "net"
  26. "regexp"
  27. "runtime/debug"
  28. "strconv"
  29. "strings"
  30. "unicode"
  31. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
  32. "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
  33. )
  34. const (
  35. MAX_API_PARAMS_SIZE = 256 * 1024 // 256KB
  36. CLIENT_VERIFICATION_TTL_SECONDS = 60 * 60 * 24 * 7 // 7 days
  37. CLIENT_PLATFORM_ANDROID = "Android"
  38. CLIENT_PLATFORM_WINDOWS = "Windows"
  39. CLIENT_PLATFORM_IOS = "iOS"
  40. )
  41. var CLIENT_VERIFICATION_REQUIRED = false
  42. type requestJSONObject map[string]interface{}
  43. // sshAPIRequestHandler routes Psiphon API requests transported as
  44. // JSON objects via the SSH request mechanism.
  45. //
  46. // The API request handlers, handshakeAPIRequestHandler, etc., are
  47. // reused by webServer which offers the Psiphon API via web transport.
  48. //
  49. // The API request parameters and event log values follow the legacy
  50. // psi_web protocol and naming conventions. The API is compatible with
  51. // all tunnel-core clients but are not backwards compatible with all
  52. // legacy clients.
  53. //
  54. func sshAPIRequestHandler(
  55. support *SupportServices,
  56. geoIPData GeoIPData,
  57. authorizedAccessTypes []string,
  58. name string,
  59. requestPayload []byte) ([]byte, error) {
  60. // Notes:
  61. //
  62. // - For SSH requests, MAX_API_PARAMS_SIZE is implicitly enforced
  63. // by max SSH request packet size.
  64. //
  65. // - The param protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS is an
  66. // array of base64-encoded strings; the base64 representation should
  67. // not be decoded to []byte values. The default behavior of
  68. // https://golang.org/pkg/encoding/json/#Unmarshal for a target of
  69. // type map[string]interface{} will unmarshal a base64-encoded string
  70. // to a string, not a decoded []byte, as required.
  71. var params requestJSONObject
  72. err := json.Unmarshal(requestPayload, &params)
  73. if err != nil {
  74. return nil, common.ContextError(
  75. fmt.Errorf("invalid payload for request name: %s: %s", name, err))
  76. }
  77. return dispatchAPIRequestHandler(
  78. support,
  79. protocol.PSIPHON_SSH_API_PROTOCOL,
  80. geoIPData,
  81. authorizedAccessTypes,
  82. name,
  83. params)
  84. }
  85. // dispatchAPIRequestHandler is the common dispatch point for both
  86. // web and SSH API requests.
  87. func dispatchAPIRequestHandler(
  88. support *SupportServices,
  89. apiProtocol string,
  90. geoIPData GeoIPData,
  91. authorizedAccessTypes []string,
  92. name string,
  93. params requestJSONObject) (response []byte, reterr error) {
  94. // Recover from and log any unexpected panics caused by user input
  95. // handling bugs. User inputs should be properly validated; this
  96. // mechanism is only a last resort to prevent the process from
  97. // terminating in the case of a bug.
  98. defer func() {
  99. if e := recover(); e != nil {
  100. if intentionalPanic, ok := e.(IntentionalPanicError); ok {
  101. panic(intentionalPanic)
  102. } else {
  103. log.LogPanicRecover(e, debug.Stack())
  104. reterr = common.ContextError(errors.New("request handler panic"))
  105. }
  106. }
  107. }()
  108. // Before invoking the handlers, enforce some preconditions:
  109. //
  110. // - A handshake request must precede any other requests.
  111. // - When the handshake results in a traffic rules state where
  112. // the client is immediately exhausted, no requests
  113. // may succeed. This case ensures that blocked clients do
  114. // not log "connected", etc.
  115. //
  116. // Only one handshake request may be made. There is no check here
  117. // to enforce that handshakeAPIRequestHandler will be called at
  118. // most once. The SetHandshakeState call in handshakeAPIRequestHandler
  119. // enforces that only a single handshake is made; enforcing that there
  120. // ensures no race condition even if concurrent requests are
  121. // in flight.
  122. if name != protocol.PSIPHON_API_HANDSHAKE_REQUEST_NAME {
  123. // TODO: same session-ID-lookup TODO in handshakeAPIRequestHandler
  124. // applies here.
  125. sessionID, err := getStringRequestParam(params, "client_session_id")
  126. if err == nil {
  127. // Note: follows/duplicates baseRequestParams validation
  128. if !isHexDigits(support, sessionID) {
  129. err = errors.New("invalid param: client_session_id")
  130. }
  131. }
  132. if err != nil {
  133. return nil, common.ContextError(err)
  134. }
  135. completed, exhausted, err := support.TunnelServer.GetClientHandshaked(sessionID)
  136. if err != nil {
  137. return nil, common.ContextError(err)
  138. }
  139. if !completed {
  140. return nil, common.ContextError(errors.New("handshake not completed"))
  141. }
  142. if exhausted {
  143. return nil, common.ContextError(errors.New("exhausted after handshake"))
  144. }
  145. }
  146. switch name {
  147. case protocol.PSIPHON_API_HANDSHAKE_REQUEST_NAME:
  148. return handshakeAPIRequestHandler(support, apiProtocol, geoIPData, params)
  149. case protocol.PSIPHON_API_CONNECTED_REQUEST_NAME:
  150. return connectedAPIRequestHandler(support, geoIPData, authorizedAccessTypes, params)
  151. case protocol.PSIPHON_API_STATUS_REQUEST_NAME:
  152. return statusAPIRequestHandler(support, geoIPData, authorizedAccessTypes, params)
  153. case protocol.PSIPHON_API_CLIENT_VERIFICATION_REQUEST_NAME:
  154. return clientVerificationAPIRequestHandler(support, geoIPData, authorizedAccessTypes, params)
  155. }
  156. return nil, common.ContextError(fmt.Errorf("invalid request name: %s", name))
  157. }
  158. // handshakeAPIRequestHandler implements the "handshake" API request.
  159. // Clients make the handshake immediately after establishing a tunnel
  160. // connection; the response tells the client what homepage to open, what
  161. // stats to record, etc.
  162. func handshakeAPIRequestHandler(
  163. support *SupportServices,
  164. apiProtocol string,
  165. geoIPData GeoIPData,
  166. params requestJSONObject) ([]byte, error) {
  167. // Note: ignoring "known_servers" params
  168. err := validateRequestParams(support, params, baseRequestParams)
  169. if err != nil {
  170. return nil, common.ContextError(err)
  171. }
  172. sessionID, _ := getStringRequestParam(params, "client_session_id")
  173. sponsorID, _ := getStringRequestParam(params, "sponsor_id")
  174. clientVersion, _ := getStringRequestParam(params, "client_version")
  175. clientPlatform, _ := getStringRequestParam(params, "client_platform")
  176. isMobile := isMobileClientPlatform(clientPlatform)
  177. normalizedPlatform := normalizeClientPlatform(clientPlatform)
  178. var authorizations []string
  179. if params[protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS] != nil {
  180. authorizations, err = getStringArrayRequestParam(params, protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS)
  181. if err != nil {
  182. return nil, common.ContextError(err)
  183. }
  184. }
  185. // Flag the SSH client as having completed its handshake. This
  186. // may reselect traffic rules and starts allowing port forwards.
  187. // TODO: in the case of SSH API requests, the actual sshClient could
  188. // be passed in and used here. The session ID lookup is only strictly
  189. // necessary to support web API requests.
  190. activeAuthorizationIDs, authorizedAccessTypes, err := support.TunnelServer.SetClientHandshakeState(
  191. sessionID,
  192. handshakeState{
  193. completed: true,
  194. apiProtocol: apiProtocol,
  195. apiParams: copyBaseRequestParams(params),
  196. },
  197. authorizations)
  198. if err != nil {
  199. return nil, common.ContextError(err)
  200. }
  201. // The log comes _after_ SetClientHandshakeState, in case that call rejects
  202. // the state change (for example, if a second handshake is performed)
  203. //
  204. // The handshake event is no longer shipped to log consumers, so this is
  205. // simply a diagnostic log.
  206. log.WithContextFields(
  207. getRequestLogFields(
  208. "",
  209. geoIPData,
  210. authorizedAccessTypes,
  211. params,
  212. baseRequestParams)).Info("handshake")
  213. // Note: no guarantee that PsinetDatabase won't reload between database calls
  214. db := support.PsinetDatabase
  215. handshakeResponse := protocol.HandshakeResponse{
  216. SSHSessionID: sessionID,
  217. Homepages: db.GetRandomizedHomepages(sponsorID, geoIPData.Country, isMobile),
  218. UpgradeClientVersion: db.GetUpgradeClientVersion(clientVersion, normalizedPlatform),
  219. PageViewRegexes: make([]map[string]string, 0),
  220. HttpsRequestRegexes: db.GetHttpsRequestRegexes(sponsorID),
  221. EncodedServerList: db.DiscoverServers(geoIPData.DiscoveryValue),
  222. ClientRegion: geoIPData.Country,
  223. ServerTimestamp: common.GetCurrentTimestamp(),
  224. ActiveAuthorizationIDs: activeAuthorizationIDs,
  225. }
  226. responsePayload, err := json.Marshal(handshakeResponse)
  227. if err != nil {
  228. return nil, common.ContextError(err)
  229. }
  230. return responsePayload, nil
  231. }
  232. var connectedRequestParams = append(
  233. []requestParamSpec{
  234. {"session_id", isHexDigits, 0},
  235. {"last_connected", isLastConnected, 0},
  236. {"establishment_duration", isIntString, requestParamOptional}},
  237. baseRequestParams...)
  238. // connectedAPIRequestHandler implements the "connected" API request.
  239. // Clients make the connected request once a tunnel connection has been
  240. // established and at least once per day. The last_connected input value,
  241. // which should be a connected_timestamp output from a previous connected
  242. // response, is used to calculate unique user stats.
  243. func connectedAPIRequestHandler(
  244. support *SupportServices,
  245. geoIPData GeoIPData,
  246. authorizedAccessTypes []string,
  247. params requestJSONObject) ([]byte, error) {
  248. err := validateRequestParams(support, params, connectedRequestParams)
  249. if err != nil {
  250. return nil, common.ContextError(err)
  251. }
  252. log.LogRawFieldsWithTimestamp(
  253. getRequestLogFields(
  254. "connected",
  255. geoIPData,
  256. authorizedAccessTypes,
  257. params,
  258. connectedRequestParams))
  259. connectedResponse := protocol.ConnectedResponse{
  260. ConnectedTimestamp: common.TruncateTimestampToHour(common.GetCurrentTimestamp()),
  261. }
  262. responsePayload, err := json.Marshal(connectedResponse)
  263. if err != nil {
  264. return nil, common.ContextError(err)
  265. }
  266. return responsePayload, nil
  267. }
  268. var statusRequestParams = append(
  269. []requestParamSpec{
  270. {"session_id", isHexDigits, 0},
  271. {"connected", isBooleanFlag, 0}},
  272. baseRequestParams...)
  273. // statusAPIRequestHandler implements the "status" API request.
  274. // Clients make periodic status requests which deliver client-side
  275. // recorded data transfer and tunnel duration stats.
  276. // Note from psi_web implementation: no input validation on domains;
  277. // any string is accepted (regex transform may result in arbitrary
  278. // string). Stats processor must handle this input with care.
  279. func statusAPIRequestHandler(
  280. support *SupportServices,
  281. geoIPData GeoIPData,
  282. authorizedAccessTypes []string,
  283. params requestJSONObject) ([]byte, error) {
  284. err := validateRequestParams(support, params, statusRequestParams)
  285. if err != nil {
  286. return nil, common.ContextError(err)
  287. }
  288. statusData, err := getJSONObjectRequestParam(params, "statusData")
  289. if err != nil {
  290. return nil, common.ContextError(err)
  291. }
  292. // Logs are queued until the input is fully validated. Otherwise, stats
  293. // could be double counted if the client has a bug in its request
  294. // formatting: partial stats would be logged (counted), the request would
  295. // fail, and clients would then resend all the same stats again.
  296. logQueue := make([]LogFields, 0)
  297. // Domain bytes transferred stats
  298. // Older clients may not submit this data
  299. // TODO: ignore these stats if regexes for client were empty; in this
  300. // case, clients are expected to send no host_bytes, but older clients
  301. // may still send.
  302. if statusData["host_bytes"] != nil {
  303. hostBytes, err := getMapStringInt64RequestParam(statusData, "host_bytes")
  304. if err != nil {
  305. return nil, common.ContextError(err)
  306. }
  307. for domain, bytes := range hostBytes {
  308. domainBytesFields := getRequestLogFields(
  309. "domain_bytes",
  310. geoIPData,
  311. authorizedAccessTypes,
  312. params,
  313. statusRequestParams)
  314. domainBytesFields["domain"] = domain
  315. domainBytesFields["bytes"] = bytes
  316. logQueue = append(logQueue, domainBytesFields)
  317. }
  318. }
  319. // Remote server list download stats
  320. // Older clients may not submit this data
  321. if statusData["remote_server_list_stats"] != nil {
  322. remoteServerListStats, err := getJSONObjectArrayRequestParam(statusData, "remote_server_list_stats")
  323. if err != nil {
  324. return nil, common.ContextError(err)
  325. }
  326. for _, remoteServerListStat := range remoteServerListStats {
  327. remoteServerListFields := getRequestLogFields(
  328. "remote_server_list",
  329. geoIPData,
  330. authorizedAccessTypes,
  331. params,
  332. statusRequestParams)
  333. clientDownloadTimestamp, err := getStringRequestParam(remoteServerListStat, "client_download_timestamp")
  334. if err != nil {
  335. return nil, common.ContextError(err)
  336. }
  337. remoteServerListFields["client_download_timestamp"] = clientDownloadTimestamp
  338. url, err := getStringRequestParam(remoteServerListStat, "url")
  339. if err != nil {
  340. return nil, common.ContextError(err)
  341. }
  342. remoteServerListFields["url"] = url
  343. etag, err := getStringRequestParam(remoteServerListStat, "etag")
  344. if err != nil {
  345. return nil, common.ContextError(err)
  346. }
  347. remoteServerListFields["etag"] = etag
  348. logQueue = append(logQueue, remoteServerListFields)
  349. }
  350. }
  351. for _, logItem := range logQueue {
  352. log.LogRawFieldsWithTimestamp(logItem)
  353. }
  354. return make([]byte, 0), nil
  355. }
  356. // clientVerificationAPIRequestHandler implements the
  357. // "client verification" API request. Clients make the client
  358. // verification request once per tunnel connection. The payload
  359. // attests that client is a legitimate Psiphon client.
  360. func clientVerificationAPIRequestHandler(
  361. support *SupportServices,
  362. geoIPData GeoIPData,
  363. authorizedAccessTypes []string,
  364. params requestJSONObject) ([]byte, error) {
  365. err := validateRequestParams(support, params, baseRequestParams)
  366. if err != nil {
  367. return nil, common.ContextError(err)
  368. }
  369. // Ignoring error as params are validated
  370. clientPlatform, _ := getStringRequestParam(params, "client_platform")
  371. // Client sends empty payload to receive TTL
  372. // NOTE: these events are not currently logged
  373. if params["verificationData"] == nil {
  374. if CLIENT_VERIFICATION_REQUIRED {
  375. var clientVerificationResponse struct {
  376. ClientVerificationTTLSeconds int `json:"client_verification_ttl_seconds"`
  377. }
  378. clientVerificationResponse.ClientVerificationTTLSeconds = CLIENT_VERIFICATION_TTL_SECONDS
  379. responsePayload, err := json.Marshal(clientVerificationResponse)
  380. if err != nil {
  381. return nil, common.ContextError(err)
  382. }
  383. return responsePayload, nil
  384. }
  385. return make([]byte, 0), nil
  386. } else {
  387. verificationData, err := getJSONObjectRequestParam(params, "verificationData")
  388. if err != nil {
  389. return nil, common.ContextError(err)
  390. }
  391. logFields := getRequestLogFields(
  392. "client_verification",
  393. geoIPData,
  394. authorizedAccessTypes,
  395. params,
  396. baseRequestParams)
  397. var verified bool
  398. var safetyNetCheckLogs LogFields
  399. switch normalizeClientPlatform(clientPlatform) {
  400. case CLIENT_PLATFORM_ANDROID:
  401. verified, safetyNetCheckLogs = verifySafetyNetPayload(verificationData)
  402. logFields["safetynet_check"] = safetyNetCheckLogs
  403. }
  404. log.LogRawFieldsWithTimestamp(logFields)
  405. if verified {
  406. // TODO: change throttling treatment
  407. }
  408. return make([]byte, 0), nil
  409. }
  410. }
  411. type requestParamSpec struct {
  412. name string
  413. validator func(*SupportServices, string) bool
  414. flags uint32
  415. }
  416. const (
  417. requestParamOptional = 1
  418. requestParamNotLogged = 2
  419. requestParamArray = 4
  420. )
  421. // baseRequestParams is the list of required and optional
  422. // request parameters; derived from COMMON_INPUTS and
  423. // OPTIONAL_COMMON_INPUTS in psi_web.
  424. // Each param is expected to be a string, unless requestParamArray
  425. // is specified, in which case an array of string is expected.
  426. var baseRequestParams = []requestParamSpec{
  427. {"server_secret", isServerSecret, requestParamNotLogged},
  428. {"client_session_id", isHexDigits, requestParamNotLogged},
  429. {"propagation_channel_id", isHexDigits, 0},
  430. {"sponsor_id", isHexDigits, 0},
  431. {"client_version", isIntString, 0},
  432. {"client_platform", isClientPlatform, 0},
  433. {"client_build_rev", isHexDigits, requestParamOptional},
  434. {"relay_protocol", isRelayProtocol, 0},
  435. {"tunnel_whole_device", isBooleanFlag, requestParamOptional},
  436. {"device_region", isRegionCode, requestParamOptional},
  437. {"ssh_client_version", isAnyString, requestParamOptional},
  438. {"upstream_proxy_type", isUpstreamProxyType, requestParamOptional},
  439. {"upstream_proxy_custom_header_names", isAnyString, requestParamOptional | requestParamArray},
  440. {"meek_dial_address", isDialAddress, requestParamOptional},
  441. {"meek_resolved_ip_address", isIPAddress, requestParamOptional},
  442. {"meek_sni_server_name", isDomain, requestParamOptional},
  443. {"meek_host_header", isHostHeader, requestParamOptional},
  444. {"meek_transformed_host_name", isBooleanFlag, requestParamOptional},
  445. {"user_agent", isAnyString, requestParamOptional},
  446. {"tls_profile", isAnyString, requestParamOptional},
  447. {"server_entry_region", isRegionCode, requestParamOptional},
  448. {"server_entry_source", isServerEntrySource, requestParamOptional},
  449. {"server_entry_timestamp", isISO8601Date, requestParamOptional},
  450. }
  451. func validateRequestParams(
  452. support *SupportServices,
  453. params requestJSONObject,
  454. expectedParams []requestParamSpec) error {
  455. for _, expectedParam := range expectedParams {
  456. value := params[expectedParam.name]
  457. if value == nil {
  458. if expectedParam.flags&requestParamOptional != 0 {
  459. continue
  460. }
  461. return common.ContextError(
  462. fmt.Errorf("missing param: %s", expectedParam.name))
  463. }
  464. var err error
  465. if expectedParam.flags&requestParamArray != 0 {
  466. err = validateStringArrayRequestParam(support, expectedParam, value)
  467. } else {
  468. err = validateStringRequestParam(support, expectedParam, value)
  469. }
  470. if err != nil {
  471. return common.ContextError(err)
  472. }
  473. }
  474. return nil
  475. }
  476. // copyBaseRequestParams makes a copy of the params which
  477. // includes only the baseRequestParams.
  478. func copyBaseRequestParams(params requestJSONObject) requestJSONObject {
  479. // Note: not a deep copy; assumes baseRequestParams values
  480. // are all scalar types (int, string, etc.)
  481. paramsCopy := make(requestJSONObject)
  482. for _, baseParam := range baseRequestParams {
  483. value := params[baseParam.name]
  484. if value == nil {
  485. continue
  486. }
  487. paramsCopy[baseParam.name] = value
  488. }
  489. return paramsCopy
  490. }
  491. func validateStringRequestParam(
  492. support *SupportServices,
  493. expectedParam requestParamSpec,
  494. value interface{}) error {
  495. strValue, ok := value.(string)
  496. if !ok {
  497. return common.ContextError(
  498. fmt.Errorf("unexpected string param type: %s", expectedParam.name))
  499. }
  500. if !expectedParam.validator(support, strValue) {
  501. return common.ContextError(
  502. fmt.Errorf("invalid param: %s", expectedParam.name))
  503. }
  504. return nil
  505. }
  506. func validateStringArrayRequestParam(
  507. support *SupportServices,
  508. expectedParam requestParamSpec,
  509. value interface{}) error {
  510. arrayValue, ok := value.([]interface{})
  511. if !ok {
  512. return common.ContextError(
  513. fmt.Errorf("unexpected string param type: %s", expectedParam.name))
  514. }
  515. for _, value := range arrayValue {
  516. err := validateStringRequestParam(support, expectedParam, value)
  517. if err != nil {
  518. return common.ContextError(err)
  519. }
  520. }
  521. return nil
  522. }
  523. // getRequestLogFields makes LogFields to log the API event following
  524. // the legacy psi_web and current ELK naming conventions.
  525. func getRequestLogFields(
  526. eventName string,
  527. geoIPData GeoIPData,
  528. authorizedAccessTypes []string,
  529. params requestJSONObject,
  530. expectedParams []requestParamSpec) LogFields {
  531. logFields := make(LogFields)
  532. if eventName != "" {
  533. logFields["event_name"] = eventName
  534. }
  535. // In psi_web, the space replacement was done to accommodate space
  536. // delimited logging, which is no longer required; we retain the
  537. // transformation so that stats aggregation isn't impacted.
  538. logFields["client_region"] = strings.Replace(geoIPData.Country, " ", "_", -1)
  539. logFields["client_city"] = strings.Replace(geoIPData.City, " ", "_", -1)
  540. logFields["client_isp"] = strings.Replace(geoIPData.ISP, " ", "_", -1)
  541. if len(authorizedAccessTypes) > 0 {
  542. logFields["authorized_access_types"] = authorizedAccessTypes
  543. }
  544. if params == nil {
  545. return logFields
  546. }
  547. for _, expectedParam := range expectedParams {
  548. if expectedParam.flags&requestParamNotLogged != 0 {
  549. continue
  550. }
  551. value := params[expectedParam.name]
  552. if value == nil {
  553. // Special case: older clients don't send this value,
  554. // so log a default.
  555. if expectedParam.name == "tunnel_whole_device" {
  556. value = "0"
  557. } else {
  558. // Skip omitted, optional params
  559. continue
  560. }
  561. }
  562. switch v := value.(type) {
  563. case string:
  564. strValue := v
  565. // Special cases:
  566. // - Number fields are encoded as integer types.
  567. // - For ELK performance we record these domain-or-IP
  568. // fields as one of two different values based on type;
  569. // we also omit port from host:port fields for now.
  570. switch expectedParam.name {
  571. case "client_version":
  572. intValue, _ := strconv.Atoi(strValue)
  573. logFields[expectedParam.name] = intValue
  574. case "meek_dial_address":
  575. host, _, _ := net.SplitHostPort(strValue)
  576. if isIPAddress(nil, host) {
  577. logFields["meek_dial_ip_address"] = host
  578. } else {
  579. logFields["meek_dial_domain"] = host
  580. }
  581. case "meek_host_header":
  582. host, _, _ := net.SplitHostPort(strValue)
  583. logFields[expectedParam.name] = host
  584. case "upstream_proxy_type":
  585. // Submitted value could be e.g., "SOCKS5" or "socks5"; log lowercase
  586. logFields[expectedParam.name] = strings.ToLower(strValue)
  587. default:
  588. logFields[expectedParam.name] = strValue
  589. }
  590. case []interface{}:
  591. // Note: actually validated as an array of strings
  592. logFields[expectedParam.name] = v
  593. default:
  594. // This type assertion should be checked already in
  595. // validateRequestParams, so failure is unexpected.
  596. continue
  597. }
  598. }
  599. return logFields
  600. }
  601. func getStringRequestParam(params requestJSONObject, name string) (string, error) {
  602. if params[name] == nil {
  603. return "", common.ContextError(fmt.Errorf("missing param: %s", name))
  604. }
  605. value, ok := params[name].(string)
  606. if !ok {
  607. return "", common.ContextError(fmt.Errorf("invalid param: %s", name))
  608. }
  609. return value, nil
  610. }
  611. func getInt64RequestParam(params requestJSONObject, name string) (int64, error) {
  612. if params[name] == nil {
  613. return 0, common.ContextError(fmt.Errorf("missing param: %s", name))
  614. }
  615. value, ok := params[name].(float64)
  616. if !ok {
  617. return 0, common.ContextError(fmt.Errorf("invalid param: %s", name))
  618. }
  619. return int64(value), nil
  620. }
  621. func getJSONObjectRequestParam(params requestJSONObject, name string) (requestJSONObject, error) {
  622. if params[name] == nil {
  623. return nil, common.ContextError(fmt.Errorf("missing param: %s", name))
  624. }
  625. // Note: generic unmarshal of JSON produces map[string]interface{}, not requestJSONObject
  626. value, ok := params[name].(map[string]interface{})
  627. if !ok {
  628. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  629. }
  630. return requestJSONObject(value), nil
  631. }
  632. func getJSONObjectArrayRequestParam(params requestJSONObject, name string) ([]requestJSONObject, error) {
  633. if params[name] == nil {
  634. return nil, common.ContextError(fmt.Errorf("missing param: %s", name))
  635. }
  636. value, ok := params[name].([]interface{})
  637. if !ok {
  638. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  639. }
  640. result := make([]requestJSONObject, len(value))
  641. for i, item := range value {
  642. // Note: generic unmarshal of JSON produces map[string]interface{}, not requestJSONObject
  643. resultItem, ok := item.(map[string]interface{})
  644. if !ok {
  645. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  646. }
  647. result[i] = requestJSONObject(resultItem)
  648. }
  649. return result, nil
  650. }
  651. func getMapStringInt64RequestParam(params requestJSONObject, name string) (map[string]int64, error) {
  652. if params[name] == nil {
  653. return nil, common.ContextError(fmt.Errorf("missing param: %s", name))
  654. }
  655. // TODO: can't use requestJSONObject type?
  656. value, ok := params[name].(map[string]interface{})
  657. if !ok {
  658. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  659. }
  660. result := make(map[string]int64)
  661. for k, v := range value {
  662. numValue, ok := v.(float64)
  663. if !ok {
  664. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  665. }
  666. result[k] = int64(numValue)
  667. }
  668. return result, nil
  669. }
  670. func getStringArrayRequestParam(params requestJSONObject, name string) ([]string, error) {
  671. if params[name] == nil {
  672. return nil, common.ContextError(fmt.Errorf("missing param: %s", name))
  673. }
  674. value, ok := params[name].([]interface{})
  675. if !ok {
  676. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  677. }
  678. result := make([]string, len(value))
  679. for i, v := range value {
  680. strValue, ok := v.(string)
  681. if !ok {
  682. return nil, common.ContextError(fmt.Errorf("invalid param: %s", name))
  683. }
  684. result[i] = strValue
  685. }
  686. return result, nil
  687. }
  688. // Normalize reported client platform. Android clients, for example, report
  689. // OS version, rooted status, and Google Play build status in the clientPlatform
  690. // string along with "Android".
  691. func normalizeClientPlatform(clientPlatform string) string {
  692. if strings.Contains(strings.ToLower(clientPlatform), strings.ToLower(CLIENT_PLATFORM_ANDROID)) {
  693. return CLIENT_PLATFORM_ANDROID
  694. } else if strings.HasPrefix(clientPlatform, CLIENT_PLATFORM_IOS) {
  695. return CLIENT_PLATFORM_IOS
  696. }
  697. return CLIENT_PLATFORM_WINDOWS
  698. }
  699. func isAnyString(support *SupportServices, value string) bool {
  700. return true
  701. }
  702. func isMobileClientPlatform(clientPlatform string) bool {
  703. normalizedClientPlatform := normalizeClientPlatform(clientPlatform)
  704. return normalizedClientPlatform == CLIENT_PLATFORM_ANDROID ||
  705. normalizedClientPlatform == CLIENT_PLATFORM_IOS
  706. }
  707. // Input validators follow the legacy validations rules in psi_web.
  708. func isServerSecret(support *SupportServices, value string) bool {
  709. return subtle.ConstantTimeCompare(
  710. []byte(value),
  711. []byte(support.Config.WebServerSecret)) == 1
  712. }
  713. func isHexDigits(_ *SupportServices, value string) bool {
  714. // Allows both uppercase in addition to lowercase, for legacy support.
  715. return -1 == strings.IndexFunc(value, func(c rune) bool {
  716. return !unicode.Is(unicode.ASCII_Hex_Digit, c)
  717. })
  718. }
  719. func isDigits(_ *SupportServices, value string) bool {
  720. return -1 == strings.IndexFunc(value, func(c rune) bool {
  721. return c < '0' || c > '9'
  722. })
  723. }
  724. func isIntString(_ *SupportServices, value string) bool {
  725. _, err := strconv.Atoi(value)
  726. return err == nil
  727. }
  728. func isClientPlatform(_ *SupportServices, value string) bool {
  729. return -1 == strings.IndexFunc(value, func(c rune) bool {
  730. // Note: stricter than psi_web's Python string.whitespace
  731. return unicode.Is(unicode.White_Space, c)
  732. })
  733. }
  734. func isRelayProtocol(_ *SupportServices, value string) bool {
  735. return common.Contains(protocol.SupportedTunnelProtocols, value)
  736. }
  737. func isBooleanFlag(_ *SupportServices, value string) bool {
  738. return value == "0" || value == "1"
  739. }
  740. func isUpstreamProxyType(_ *SupportServices, value string) bool {
  741. value = strings.ToLower(value)
  742. return value == "http" || value == "socks5" || value == "socks4a"
  743. }
  744. func isRegionCode(_ *SupportServices, value string) bool {
  745. if len(value) != 2 {
  746. return false
  747. }
  748. return -1 == strings.IndexFunc(value, func(c rune) bool {
  749. return c < 'A' || c > 'Z'
  750. })
  751. }
  752. func isDialAddress(_ *SupportServices, value string) bool {
  753. // "<host>:<port>", where <host> is a domain or IP address
  754. parts := strings.Split(value, ":")
  755. if len(parts) != 2 {
  756. return false
  757. }
  758. if !isIPAddress(nil, parts[0]) && !isDomain(nil, parts[0]) {
  759. return false
  760. }
  761. if !isDigits(nil, parts[1]) {
  762. return false
  763. }
  764. port, err := strconv.Atoi(parts[1])
  765. if err != nil {
  766. return false
  767. }
  768. return port > 0 && port < 65536
  769. }
  770. func isIPAddress(_ *SupportServices, value string) bool {
  771. return net.ParseIP(value) != nil
  772. }
  773. var isDomainRegex = regexp.MustCompile("[a-zA-Z\\d-]{1,63}$")
  774. func isDomain(_ *SupportServices, value string) bool {
  775. // From: http://stackoverflow.com/questions/2532053/validate-a-hostname-string
  776. //
  777. // "ensures that each segment
  778. // * contains at least one character and a maximum of 63 characters
  779. // * consists only of allowed characters
  780. // * doesn't begin or end with a hyphen"
  781. //
  782. if len(value) > 255 {
  783. return false
  784. }
  785. value = strings.TrimSuffix(value, ".")
  786. for _, part := range strings.Split(value, ".") {
  787. // Note: regexp doesn't support the following Perl expression which
  788. // would check for '-' prefix/suffix: "(?!-)[a-zA-Z\\d-]{1,63}(?<!-)$"
  789. if strings.HasPrefix(part, "-") || strings.HasSuffix(part, "-") {
  790. return false
  791. }
  792. if !isDomainRegex.Match([]byte(part)) {
  793. return false
  794. }
  795. }
  796. return true
  797. }
  798. func isHostHeader(_ *SupportServices, value string) bool {
  799. // "<host>:<port>", where <host> is a domain or IP address and ":<port>" is optional
  800. if strings.Contains(value, ":") {
  801. return isDialAddress(nil, value)
  802. }
  803. return isIPAddress(nil, value) || isDomain(nil, value)
  804. }
  805. func isServerEntrySource(_ *SupportServices, value string) bool {
  806. return common.Contains(protocol.SupportedServerEntrySources, value)
  807. }
  808. var isISO8601DateRegex = regexp.MustCompile(
  809. "(?P<year>[0-9]{4})-(?P<month>[0-9]{1,2})-(?P<day>[0-9]{1,2})T(?P<hour>[0-9]{2}):(?P<minute>[0-9]{2}):(?P<second>[0-9]{2})(\\.(?P<fraction>[0-9]+))?(?P<timezone>Z|(([-+])([0-9]{2}):([0-9]{2})))")
  810. func isISO8601Date(_ *SupportServices, value string) bool {
  811. return isISO8601DateRegex.Match([]byte(value))
  812. }
  813. func isLastConnected(_ *SupportServices, value string) bool {
  814. return value == "None" || value == "Unknown" || isISO8601Date(nil, value)
  815. }