geoip.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. var fs = require('fs');
  2. var net = require('net');
  3. var path = require('path');
  4. fs.existsSync = fs.existsSync || path.existsSync;
  5. var utils = require('./utils');
  6. var fsWatcher = require('./fsWatcher');
  7. var async = require('async');
  8. var watcherName = 'dataWatcher';
  9. var geodatadir;
  10. if (typeof global.geodatadir === 'undefined'){
  11. geodatadir = path.join(__dirname, '/../data/');
  12. } else {
  13. geodatadir = global.geodatadir;
  14. }
  15. var dataFiles = {
  16. city: path.join(geodatadir, 'geoip-city.dat'),
  17. city6: path.join(geodatadir, 'geoip-city6.dat'),
  18. cityNames: path.join(geodatadir, 'geoip-city-names.dat'),
  19. country: path.join(geodatadir, 'geoip-country.dat'),
  20. country6: path.join(geodatadir, 'geoip-country6.dat')
  21. };
  22. var privateRange4 = [
  23. [utils.aton4('10.0.0.0'), utils.aton4('10.255.255.255')],
  24. [utils.aton4('172.16.0.0'), utils.aton4('172.31.255.255')],
  25. [utils.aton4('192.168.0.0'), utils.aton4('192.168.255.255')]
  26. ]
  27. var cache4 = {
  28. firstIP: null,
  29. lastIP: null,
  30. lastLine: 0,
  31. locationBuffer: null,
  32. locationRecordSize: 64,
  33. mainBuffer: null,
  34. recordSize: 12
  35. };
  36. var cache6 = {
  37. firstIP: null,
  38. lastIP: null,
  39. lastLine: 0,
  40. mainBuffer: null,
  41. recordSize: 58
  42. };
  43. var RECORD_SIZE = 10;
  44. var RECORD_SIZE6 = 34
  45. function lookup4(ip) {
  46. var fline = 0;
  47. var floor = cache4.lastIP;
  48. var cline = cache4.lastLine;
  49. var ceil = cache4.firstIP;
  50. var line;
  51. var locId;
  52. var buffer = cache4.mainBuffer;
  53. var locBuffer = cache4.locationBuffer;
  54. var privateRange = privateRange4;
  55. var recordSize = cache4.recordSize;
  56. var locRecordSize = cache4.locationRecordSize;
  57. var i;
  58. var geodata = {
  59. range: '',
  60. country: '',
  61. region: '',
  62. city: '',
  63. ll: [0, 0]
  64. };
  65. // outside IPv4 range
  66. if (ip > cache4.lastIP || ip < cache4.firstIP) {
  67. return null;
  68. }
  69. // private IP
  70. for (i = 0; i < privateRange.length; i++) {
  71. if (ip >= privateRange[i][0] && ip <= privateRange[i][1]) {
  72. return null;
  73. }
  74. }
  75. do {
  76. line = Math.round((cline - fline) / 2) + fline;
  77. floor = buffer.readUInt32BE(line * recordSize);
  78. ceil = buffer.readUInt32BE((line * recordSize) + 4);
  79. if (floor <= ip && ceil >= ip) {
  80. geodata.range = [floor, ceil];
  81. if (recordSize === RECORD_SIZE) {
  82. geodata.country = buffer.toString('utf8', (line * recordSize) + 8, (line * recordSize) + 10);
  83. } else {
  84. locId = buffer.readUInt32BE((line * recordSize) + 8) - 1;
  85. geodata.country = locBuffer.toString('utf8', (locId * locRecordSize) + 0, (locId * locRecordSize) + 2).replace(/\u0000.*/, '');
  86. geodata.region = locBuffer.toString('utf8', (locId * locRecordSize) + 2, (locId * locRecordSize) + 4).replace(/\u0000.*/, '');
  87. geodata.ll = [locBuffer.readInt32BE((locId * locRecordSize) + 4) / 10000, locBuffer.readInt32BE((locId * locRecordSize) + 8) / 10000];
  88. geodata.metro = locBuffer.readInt32BE((locId * locRecordSize) + 12);
  89. geodata.city = locBuffer.toString('utf8', (locId * locRecordSize) + 16, (locId * locRecordSize) + locRecordSize).replace(/\u0000.*/, '');
  90. }
  91. return geodata;
  92. } else if (fline === cline) {
  93. return null;
  94. } else if (fline === (cline - 1)) {
  95. if (line === fline) {
  96. fline = cline;
  97. } else {
  98. cline = fline;
  99. }
  100. } else if (floor > ip) {
  101. cline = line;
  102. } else if (ceil < ip) {
  103. fline = line;
  104. }
  105. } while(1);
  106. }
  107. function lookup6(ip) {
  108. var buffer = cache6.mainBuffer;
  109. var recordSize = cache6.recordSize;
  110. var geodata = {
  111. range: '',
  112. country: '',
  113. region: '',
  114. city: '',
  115. ll: [0, 0]
  116. };
  117. // XXX We only use the first 8 bytes of an IPv6 address
  118. // This identifies the network, but not the host within
  119. // the network. Unless at some point of time we have a
  120. // global peace treaty and single subnets span multiple
  121. // countries, this should not be a problem.
  122. function readip(line, offset) {
  123. var ii = 0;
  124. var ip = [];
  125. for (ii = 0; ii < 2; ii++) {
  126. ip.push(buffer.readUInt32BE((line * recordSize) + (offset * 16) + (ii * 4)));
  127. }
  128. return ip;
  129. }
  130. cache6.lastIP = readip(cache6.lastLine, 1);
  131. cache6.firstIP = readip(0, 0);
  132. var fline = 0;
  133. var floor = cache6.lastIP;
  134. var cline = cache6.lastLine;
  135. var ceil = cache6.firstIP;
  136. var line;
  137. if (utils.cmp6(ip, cache6.lastIP) > 0 || utils.cmp6(ip, cache6.firstIP) < 0) {
  138. return null;
  139. }
  140. do {
  141. line = Math.round((cline - fline) / 2) + fline;
  142. floor = readip(line, 0);
  143. ceil = readip(line, 1);
  144. if (utils.cmp6(floor, ip) <= 0 && utils.cmp6(ceil, ip) >= 0) {
  145. if (recordSize === RECORD_SIZE6) {
  146. geodata.country = buffer.toString('utf8', (line * recordSize) + 32, (line * recordSize) + 34).replace(/\u0000.*/, '');
  147. } else {
  148. geodata.range = [floor, ceil];
  149. geodata.country = buffer.toString('utf8', (line * recordSize) + 32, (line * recordSize) + 34).replace(/\u0000.*/, '');
  150. geodata.region = buffer.toString('utf8', (line * recordSize) + 34, (line * recordSize) + 36).replace(/\u0000.*/, '');
  151. geodata.ll = [buffer.readInt32BE((line * recordSize) + 36) / 10000, buffer.readInt32BE((line * recordSize) + 40) / 10000];
  152. geodata.city = buffer.toString('utf8', (line * recordSize) + 44, (line * recordSize) + recordSize).replace(/\u0000.*/, '');
  153. }
  154. // We do not currently have detailed region/city info for IPv6, but finally have coords
  155. return geodata;
  156. } else if (fline === cline) {
  157. return null;
  158. } else if (fline === (cline - 1)) {
  159. if (line === fline) {
  160. fline = cline;
  161. } else {
  162. cline = fline;
  163. }
  164. } else if (utils.cmp6(floor, ip) > 0) {
  165. cline = line;
  166. } else if (utils.cmp6(ceil, ip) < 0) {
  167. fline = line;
  168. }
  169. } while(1);
  170. }
  171. function get4mapped(ip) {
  172. var ipv6 = ip.toUpperCase();
  173. var v6prefixes = ['0:0:0:0:0:FFFF:', '::FFFF:'];
  174. for (var i = 0; i < v6prefixes.length; i++) {
  175. var v6prefix = v6prefixes[i];
  176. if (ipv6.indexOf(v6prefix) == 0) {
  177. return ipv6.substring(v6prefix.length);
  178. }
  179. }
  180. return null;
  181. }
  182. function preload(callback) {
  183. var datFile;
  184. var datSize;
  185. var asyncCache = {
  186. firstIP: null,
  187. lastIP: null,
  188. lastLine: 0,
  189. locationBuffer: null,
  190. locationRecordSize: 64,
  191. mainBuffer: null,
  192. recordSize: 12
  193. };
  194. //when the preload function receives a callback, do the task asynchronously
  195. if (typeof arguments[0] === 'function') {
  196. async.series([
  197. function (cb) {
  198. async.series([
  199. function (cb2) {
  200. fs.open(dataFiles.cityNames, 'r', function (err, file) {
  201. datFile = file;
  202. cb2(err);
  203. });
  204. },
  205. function (cb2) {
  206. fs.fstat(datFile, function (err, stats) {
  207. datSize = stats.size;
  208. asyncCache.locationBuffer = new Buffer(datSize);
  209. cb2(err);
  210. });
  211. },
  212. function (cb2) {
  213. fs.read(datFile, asyncCache.locationBuffer, 0, datSize, 0, cb2);
  214. },
  215. function (cb2) {
  216. fs.close(datFile, cb2);
  217. },
  218. function (cb2) {
  219. fs.open(dataFiles.city, 'r', function (err, file) {
  220. datFile = file;
  221. cb2(err);
  222. });
  223. },
  224. function (cb2) {
  225. fs.fstat(datFile, function (err, stats) {
  226. datSize = stats.size;
  227. cb2(err);
  228. });
  229. }
  230. ], function (err) {
  231. if (err) {
  232. if (err.code !== 'ENOENT' && err.code !== 'EBADF') {
  233. throw err;
  234. }
  235. fs.open(dataFiles.country, 'r', function (err, file) {
  236. if (err) {
  237. cb(err);
  238. } else {
  239. datFile = file;
  240. fs.fstat(datFile, function (err, stats) {
  241. datSize = stats.size;
  242. asyncCache.recordSize = RECORD_SIZE;
  243. cb();
  244. });
  245. }
  246. });
  247. } else {
  248. cb();
  249. }
  250. });
  251. },
  252. function () {
  253. asyncCache.mainBuffer = new Buffer(datSize);
  254. async.series([
  255. function (cb2) {
  256. fs.read(datFile, asyncCache.mainBuffer, 0, datSize, 0, cb2);
  257. },
  258. function (cb2) {
  259. fs.close(datFile, cb2);
  260. }
  261. ], function (err) {
  262. if (err) {
  263. //keep old cache
  264. } else {
  265. asyncCache.lastLine = (datSize / asyncCache.recordSize) - 1;
  266. asyncCache.lastIP = asyncCache.mainBuffer.readUInt32BE((asyncCache.lastLine * asyncCache.recordSize) + 4);
  267. cache4 = asyncCache;
  268. }
  269. callback(err);
  270. });
  271. }
  272. ]);
  273. } else {
  274. try {
  275. datFile = fs.openSync(dataFiles.cityNames, 'r');
  276. datSize = fs.fstatSync(datFile).size;
  277. if (datSize === 0) {
  278. throw {
  279. code: 'EMPTY_FILE'
  280. };
  281. }
  282. cache4.locationBuffer = new Buffer(datSize);
  283. fs.readSync(datFile, cache4.locationBuffer, 0, datSize, 0);
  284. fs.closeSync(datFile);
  285. datFile = fs.openSync(dataFiles.city, 'r');
  286. datSize = fs.fstatSync(datFile).size;
  287. } catch(err) {
  288. if (err.code !== 'ENOENT' && err.code !== 'EBADF' && err.code !== 'EMPTY_FILE') {
  289. throw err;
  290. }
  291. datFile = fs.openSync(dataFiles.country, 'r');
  292. datSize = fs.fstatSync(datFile).size;
  293. cache4.recordSize = RECORD_SIZE;
  294. }
  295. cache4.mainBuffer = new Buffer(datSize);
  296. fs.readSync(datFile, cache4.mainBuffer, 0, datSize, 0);
  297. fs.closeSync(datFile);
  298. cache4.lastLine = (datSize / cache4.recordSize) - 1;
  299. cache4.lastIP = cache4.mainBuffer.readUInt32BE((cache4.lastLine * cache4.recordSize) + 4);
  300. cache4.firstIP = cache4.mainBuffer.readUInt32BE(0);
  301. }
  302. }
  303. function preload6(callback) {
  304. var datFile;
  305. var datSize;
  306. var asyncCache6 = {
  307. firstIP: null,
  308. lastIP: null,
  309. lastLine: 0,
  310. mainBuffer: null,
  311. recordSize: 58
  312. };
  313. //when the preload function receives a callback, do the task asynchronously
  314. if (typeof arguments[0] === 'function') {
  315. async.series([
  316. function (cb) {
  317. async.series([
  318. function (cb2) {
  319. fs.open(dataFiles.city6, 'r', function (err, file) {
  320. datFile = file;
  321. cb2(err);
  322. });
  323. },
  324. function (cb2) {
  325. fs.fstat(datFile, function (err, stats) {
  326. datSize = stats.size;
  327. cb2(err);
  328. });
  329. }
  330. ], function (err) {
  331. if (err) {
  332. if (err.code !== 'ENOENT' && err.code !== 'EBADF') {
  333. throw err;
  334. }
  335. fs.open(dataFiles.country6, 'r', function (err, file) {
  336. if (err) {
  337. cb(err);
  338. } else {
  339. datFile = file;
  340. fs.fstat(datFile, function (err, stats) {
  341. datSize = stats.size;
  342. asyncCache6.recordSize = RECORD_SIZE6;
  343. cb();
  344. });
  345. }
  346. });
  347. } else {
  348. cb();
  349. }
  350. });
  351. },
  352. function () {
  353. asyncCache6.mainBuffer = new Buffer(datSize);
  354. async.series([
  355. function (cb2) {
  356. fs.read(datFile, asyncCache6.mainBuffer, 0, datSize, 0, cb2);
  357. },
  358. function (cb2) {
  359. fs.close(datFile, cb2);
  360. }
  361. ], function (err) {
  362. if (err) {
  363. //keep old cache
  364. } else {
  365. asyncCache6.lastLine = (datSize / asyncCache6.recordSize) - 1;
  366. cache6 = asyncCache6;
  367. }
  368. callback(err);
  369. });
  370. }
  371. ]);
  372. } else {
  373. try {
  374. datFile = fs.openSync(dataFiles.city6, 'r');
  375. datSize = fs.fstatSync(datFile).size;
  376. if (datSize === 0) {
  377. throw {
  378. code: 'EMPTY_FILE'
  379. };
  380. }
  381. } catch(err) {
  382. if (err.code !== 'ENOENT' && err.code !== 'EBADF' && err.code !== 'EMPTY_FILE') {
  383. throw err;
  384. }
  385. datFile = fs.openSync(dataFiles.country6, 'r');
  386. datSize = fs.fstatSync(datFile).size;
  387. cache6.recordSize = RECORD_SIZE6;
  388. }
  389. cache6.mainBuffer = new Buffer(datSize);
  390. fs.readSync(datFile, cache6.mainBuffer, 0, datSize, 0);
  391. fs.closeSync(datFile);
  392. cache6.lastLine = (datSize / cache6.recordSize) - 1;
  393. }
  394. }
  395. module.exports = {
  396. cmp: utils.cmp,
  397. lookup: function(ip) {
  398. if (!ip) {
  399. return null;
  400. } else if (typeof ip === 'number') {
  401. return lookup4(ip);
  402. } else if (net.isIP(ip) === 4) {
  403. return lookup4(utils.aton4(ip));
  404. } else if (net.isIP(ip) === 6) {
  405. var ipv4 = get4mapped(ip);
  406. if (ipv4) {
  407. return lookup4(utils.aton4(ipv4));
  408. } else {
  409. return lookup6(utils.aton6(ip));
  410. }
  411. }
  412. return null;
  413. },
  414. pretty: function(n) {
  415. if (typeof n === 'string') {
  416. return n;
  417. } else if (typeof n === 'number') {
  418. return utils.ntoa4(n);
  419. } else if (n instanceof Array) {
  420. return utils.ntoa6(n);
  421. }
  422. return n;
  423. },
  424. // Start watching for data updates. The watcher waits one minute for file transfer to
  425. // completete before triggering the callback.
  426. startWatchingDataUpdate: function (callback) {
  427. fsWatcher.makeFsWatchFilter(watcherName, geodatadir, 60*1000, function () {
  428. //Reload data
  429. async.series([
  430. function (cb) {
  431. preload(cb);
  432. },
  433. function (cb) {
  434. preload6(cb);
  435. }
  436. ], callback);
  437. });
  438. },
  439. // Stop watching for data updates.
  440. stopWatchingDataUpdate: function () {
  441. fsWatcher.stopWatching(watcherName);
  442. }
  443. };
  444. preload();
  445. preload6();
  446. //lookup4 = gen_lookup('geoip-country.dat', 4);
  447. //lookup6 = gen_lookup('geoip-country6.dat', 16);