a
/
badguardhome
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

*(global): updated trackers database

This commit is contained in:
Andrey Meshkov 2020-01-30 12:28:01 +03:00
parent dcc575402b
commit 327030789d
5 changed files with 627 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
client/src/helpers/trackers/adguard.json
View File

@ -17,6 +17,21 @@
"101": "mobile_analytics"
},
"trackers": {
"akamai_technologies": {
"name": "Akamai Technologies",
"categoryId": 9,
"url": "https://www.akamai.com/"
},
"apple": {
"name": "Apple",
"categoryId": 8,
"url": "https://www.apple.com/"
},
"apple_ads": {
"name": "Apple Search Ads",
"categoryId": 4,
"url": "https://searchads.apple.com/"
},
"facebook_audience": {
"name": "Facebook Audience Network",
"categoryId": 4,
@ -62,13 +77,36 @@
"categoryId": 101,
"url": "https://branch.io/"
},
"adocean": {
"name": "Gemius Adocean",
"markmonitor": {
"name": "MarkMonitor",
"categoryId": 4,
"url": "https://adocean-global.com/"
"url": "https://www.markmonitor.com/"
},
"appcenter": {
"name": "Microsoft App Center",
"categoryId": 5,
"url": "https://appcenter.ms/"
},
"unity_ads": {
"name": "Unity Ads",
"categoryId": 4,
"url": "https://unity.com/solutions/mobile-business/monetize-your-game"
},
"azure": {
"name": "Microsoft Azure",
"categoryId": 10,
"url": "https://azure.microsoft.com/"
}
},
"trackerDomains": {
"akadns.net": "akamai_technologies",
"akamaiedge.net": "akamai_technologies",
"apple.com": "apple",
"apple-dns.net": "apple",
"aaplimg.com": "apple",
"icloud.com": "apple",
"mzstatic.com": "apple",
"iadsdk.apple.com": "apple_ads",
"graph.facebook.com": "facebook_audience",
"crashlytics.com": "crashlytics",
"flurry.com": "flurry",
@ -78,6 +116,9 @@
"appmetrica.yandex.com": "yandex_appmetrica",
"adjust.com": "adjust",
"mobileapptracking.com": "branch",
"adocean.cz": "adocean"
"edgecastcdn.net": "markmonitor",
"appcenter.ms": "appcenter",
"unityads.unity3d.com": "unity_ads",
"azure.com": "azure"
}
}

720
client/src/helpers/trackers/whotracksme.json
View File

File diff suppressed because it is too large Load Diff

135
scripts/whotracksme/index.js
View File

@ -1,76 +1,81 @@
const fs = require('fs');
const sqlite3 = require('sqlite3').verbose();
const downloadFileSync = require('download-file-sync');
const axios = require('axios');
const INPUT_SQL_URL = 'https://raw.githubusercontent.com/cliqz-oss/whotracks.me/master/whotracksme/data/assets/trackerdb.sql';
const OUTPUT_PATH = 'whotracksme.json';
console.log('Downloading ' + INPUT_SQL_URL);
let trackersDbSql = downloadFileSync(INPUT_SQL_URL).toString();
async function runScript() {
console.log('Downloading ' + INPUT_SQL_URL);
let response = await axios.get(INPUT_SQL_URL);
let trackersDbSql = response.data;
let transformToSqlite = function(sql) {
sql = sql.trim();
if (sql.indexOf("CREATE TABLE") >= 0) {
sql = sql.replace(/UNIQUE/g, '');
let transformToSqlite = function (sql) {
sql = sql.trim();
if (sql.indexOf("CREATE TABLE") >= 0) {
sql = sql.replace(/UNIQUE/g, '');
}
return sql;
}
return sql;
let whotracksme = {
timeUpdated: new Date().toISOString(),
categories: {},
trackers: {},
trackerDomains: {}
};
console.log('Initializing the in-memory trackers database');
let db = new sqlite3.Database(':memory:');
db.serialize(function () {
trackersDbSql.split(/;\s*$/gm).forEach(function (sql) {
sql = transformToSqlite(sql);
db.run(sql, function () { });
});
db.each("SELECT * FROM categories", function (err, row) {
if (err) {
console.error(err);
return;
}
whotracksme.categories[row.id] = row.name;
});
db.each("SELECT * FROM trackers", function (err, row) {
if (err) {
console.error(err);
return;
}
whotracksme.trackers[row.id] = {
"name": row.name,
"categoryId": row.category_id,
"url": row.website_url
};
});
db.each("SELECT * FROM tracker_domains", function (err, row) {
if (err) {
console.error(err);
return;
}
whotracksme.trackerDomains[row.domain] = row.tracker;
});
});
db.close(function (err) {
if (err) {
console.error(err);
return;
}
fs.writeFileSync(OUTPUT_PATH, JSON.stringify(whotracksme, 0, 4));
console.log('Trackers json file has been updated: ' + OUTPUT_PATH);
});
}
let whotracksme = {
timeUpdated: new Date().toISOString(),
categories: {},
trackers: {},
trackerDomains: {}
};
console.log('Initializing the in-memory trackers database');
let db = new sqlite3.Database(':memory:');
db.serialize(function() {
trackersDbSql.split(/;\s*$/gm).forEach(function(sql) {
sql = transformToSqlite(sql);
db.run(sql, function() {});
});
db.each("SELECT * FROM categories", function(err, row) {
if (err) {
console.error(err);
return;
}
whotracksme.categories[row.id] = row.name;
});
db.each("SELECT * FROM trackers", function(err, row) {
if (err) {
console.error(err);
return;
}
whotracksme.trackers[row.id] = {
"name": row.name,
"categoryId": row.category_id,
"url": row.website_url
};
});
db.each("SELECT * FROM tracker_domains", function(err, row) {
if (err) {
console.error(err);
return;
}
whotracksme.trackerDomains[row.domain] = row.tracker;
});
});
db.close(function(err) {
if (err) {
console.error(err);
return;
}
fs.writeFileSync(OUTPUT_PATH, JSON.stringify(whotracksme, 0, 4));
console.log('Trackers json file has been updated: ' + OUTPUT_PATH);
});
runScript();

2
scripts/whotracksme/package.json
View File

@ -9,7 +9,7 @@
"author": "",
"license": "ISC",
"dependencies": {
"download-file-sync": "^1.0.4",
"axios": "^0.19.2",
"sqlite3": "^4.1.0"
}
}

21
scripts/whotracksme/yarn.lock
View File

@ -56,6 +56,13 @@ aws4@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
axios@^0.19.2:
version "0.19.2"
resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27"
integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==
dependencies:
follow-redirects "1.5.10"
balanced-match@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@ -119,6 +126,13 @@ dashdash@^1.12.0:
dependencies:
assert-plus "^1.0.0"
debug@=3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
dependencies:
ms "2.0.0"
debug@^2.1.2:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@ -172,6 +186,13 @@ fast-json-stable-stringify@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2"
follow-redirects@1.5.10:
version "1.5.10"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a"
integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==
dependencies:
debug "=3.1.0"
forever-agent@~0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"