*(global): updated trackers database

client/src/helpers/trackers/adguard.json

@@ -17,6 +17,21 @@
         "101": "mobile_analytics"
     },
     "trackers": {
+        "akamai_technologies": {
+            "name": "Akamai Technologies",
+            "categoryId": 9,
+            "url": "https://www.akamai.com/"
+        },
+        "apple": {
+            "name": "Apple",
+            "categoryId": 8,
+            "url": "https://www.apple.com/"
+        },
+        "apple_ads": {
+            "name": "Apple Search Ads",
+            "categoryId": 4,
+            "url": "https://searchads.apple.com/"
+        },
         "facebook_audience": {
             "name": "Facebook Audience Network",
             "categoryId": 4,
@@ -62,13 +77,36 @@
             "categoryId": 101,
             "url": "https://branch.io/"
         },
-        "adocean": {
+        "markmonitor": {
-            "name": "Gemius Adocean",
+            "name": "MarkMonitor",
             "categoryId": 4,
-            "url": "https://adocean-global.com/"
+            "url": "https://www.markmonitor.com/"
+        },
+        "appcenter": {
+            "name": "Microsoft App Center",
+            "categoryId": 5,
+            "url": "https://appcenter.ms/"
+        },
+        "unity_ads": {
+            "name": "Unity Ads",
+            "categoryId": 4,
+            "url": "https://unity.com/solutions/mobile-business/monetize-your-game"
+        },
+        "azure": {
+            "name": "Microsoft Azure",
+            "categoryId": 10,
+            "url": "https://azure.microsoft.com/"
         }
     },
     "trackerDomains": {
+        "akadns.net": "akamai_technologies",
+        "akamaiedge.net": "akamai_technologies",
+        "apple.com": "apple",
+        "apple-dns.net": "apple",
+        "aaplimg.com": "apple",
+        "icloud.com": "apple",
+        "mzstatic.com": "apple",
+        "iadsdk.apple.com": "apple_ads",
         "graph.facebook.com": "facebook_audience",
         "crashlytics.com": "crashlytics",
         "flurry.com": "flurry",
@@ -78,6 +116,9 @@
         "appmetrica.yandex.com": "yandex_appmetrica",
         "adjust.com": "adjust",
         "mobileapptracking.com": "branch",
-        "adocean.cz": "adocean"
+        "edgecastcdn.net": "markmonitor",
+        "appcenter.ms": "appcenter",
+        "unityads.unity3d.com": "unity_ads",
+        "azure.com": "azure"
     }
 }

scripts/whotracksme/index.js

@@ -1,76 +1,81 @@
 const fs = require('fs');
 const sqlite3 = require('sqlite3').verbose();
-const downloadFileSync = require('download-file-sync');
+const axios = require('axios');
 
 const INPUT_SQL_URL = 'https://raw.githubusercontent.com/cliqz-oss/whotracks.me/master/whotracksme/data/assets/trackerdb.sql';
 const OUTPUT_PATH = 'whotracksme.json';
 
-console.log('Downloading ' + INPUT_SQL_URL);
+async function runScript() {
-let trackersDbSql = downloadFileSync(INPUT_SQL_URL).toString();
+    console.log('Downloading ' + INPUT_SQL_URL);
+    let response = await axios.get(INPUT_SQL_URL);
+    let trackersDbSql = response.data;
 
-let transformToSqlite = function(sql) {
+    let transformToSqlite = function (sql) {
-    sql = sql.trim();
+        sql = sql.trim();
-    
+
-    if (sql.indexOf("CREATE TABLE") >= 0) {
+        if (sql.indexOf("CREATE TABLE") >= 0) {
-        sql = sql.replace(/UNIQUE/g, '');
+            sql = sql.replace(/UNIQUE/g, '');
+        }
+
+        return sql;
     }
 
-    return sql;
+    let whotracksme = {
+        timeUpdated: new Date().toISOString(),
+        categories: {},
+        trackers: {},
+        trackerDomains: {}
+    };
+
+    console.log('Initializing the in-memory trackers database');
+    let db = new sqlite3.Database(':memory:');
+    db.serialize(function () {
+        trackersDbSql.split(/;\s*$/gm).forEach(function (sql) {
+            sql = transformToSqlite(sql);
+            db.run(sql, function () { });
+        });
+
+        db.each("SELECT * FROM categories", function (err, row) {
+            if (err) {
+                console.error(err);
+                return;
+            }
+
+            whotracksme.categories[row.id] = row.name;
+        });
+
+        db.each("SELECT * FROM trackers", function (err, row) {
+            if (err) {
+                console.error(err);
+                return;
+            }
+
+            whotracksme.trackers[row.id] = {
+                "name": row.name,
+                "categoryId": row.category_id,
+                "url": row.website_url
+            };
+        });
+
+        db.each("SELECT * FROM tracker_domains", function (err, row) {
+            if (err) {
+                console.error(err);
+                return;
+            }
+
+            whotracksme.trackerDomains[row.domain] = row.tracker;
+        });
+    });
+
+    db.close(function (err) {
+        if (err) {
+            console.error(err);
+            return;
+        }
+
+        fs.writeFileSync(OUTPUT_PATH, JSON.stringify(whotracksme, 0, 4));
+        console.log('Trackers json file has been updated: ' + OUTPUT_PATH);
+    });
 }
 
-let whotracksme = {
+runScript();
-    timeUpdated: new Date().toISOString(),
-    categories: {},
-    trackers: {},
-    trackerDomains: {}
-};
-
-console.log('Initializing the in-memory trackers database');
-let db = new sqlite3.Database(':memory:');
-db.serialize(function() {
-    trackersDbSql.split(/;\s*$/gm).forEach(function(sql) { 
-        sql = transformToSqlite(sql);
-        db.run(sql, function() {});
-    });
-
-    db.each("SELECT * FROM categories", function(err, row) {
-        if (err) {
-            console.error(err);
-            return;
-        }
-
-        whotracksme.categories[row.id] = row.name;
-    });    
-
-    db.each("SELECT * FROM trackers", function(err, row) {
-        if (err) {
-            console.error(err);
-            return;
-        }
-
-        whotracksme.trackers[row.id] = {
-            "name": row.name,
-            "categoryId": row.category_id,
-            "url": row.website_url
-        };
-    });
-
-    db.each("SELECT * FROM tracker_domains", function(err, row) {
-        if (err) {
-            console.error(err);
-            return;
-        }
-
-        whotracksme.trackerDomains[row.domain] = row.tracker;
-    });
-});
-
-db.close(function(err) {
-    if (err) {
-        console.error(err);
-        return;
-    }
-
-    fs.writeFileSync(OUTPUT_PATH, JSON.stringify(whotracksme, 0, 4));
-    console.log('Trackers json file has been updated: ' + OUTPUT_PATH);
-});

scripts/whotracksme/package.json

@@ -9,7 +9,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "download-file-sync": "^1.0.4",
+    "axios": "^0.19.2",
     "sqlite3": "^4.1.0"
   }
 }

scripts/whotracksme/yarn.lock

@@ -56,6 +56,13 @@ aws4@^1.8.0:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
 
+axios@^0.19.2:
+  version "0.19.2"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27"
+  integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==
+  dependencies:
+    follow-redirects "1.5.10"
+
 balanced-match@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@@ -119,6 +126,13 @@ dashdash@^1.12.0:
   dependencies:
     assert-plus "^1.0.0"
 
+debug@=3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
+  integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
+  dependencies:
+    ms "2.0.0"
+
 debug@^2.1.2:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -172,6 +186,13 @@ fast-json-stable-stringify@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2"
 
+follow-redirects@1.5.10:
+  version "1.5.10"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a"
+  integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==
+  dependencies:
+    debug "=3.1.0"
+
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"