Add Google Workspace detection, add DNS resolution to NPM driver

main
Elbert Alias 4 years ago
parent 13da9aeefa
commit 3216f41a03

@ -1,5 +1,6 @@
const { URL } = require('url')
const fs = require('fs')
const dns = require('dns').promises
const path = require('path')
const http = require('http')
const https = require('https')
@ -227,6 +228,8 @@ class Site {
this.listeners = {}
this.pages = []
this.dns = []
}
log(message, source = 'driver', type = 'log') {
@ -629,6 +632,44 @@ class Site {
html = batches.join('\n')
}
// DNS
if (!this.dns.length) {
try {
const records = {}
;[
records.cname,
records.ns,
records.mx,
records.txt,
] = await this.promiseTimeout(
Promise.all([
dns.resolveCname(url.hostname),
dns.resolveNs(url.hostname),
dns.resolveMx(url.hostname.replace(/^www\./, '')),
dns.resolveTxt(url.hostname.replace(/^www\./, '')),
])
)
this.dns = Object.keys(records).reduce((dns, type) => {
dns[type] = dns[type] || []
Array.prototype.push.apply(
dns[type],
records[type]
.map((value) => (value.exchange ? value.exchange : value))
.flat()
)
return dns
}, {})
this.onDetect(analyze({ dns: this.dns }))
} catch (error) {
// Continue
}
}
// Validate response
if (url.protocol !== 'file:' && !this.analyzedUrls[url.href].status) {
await page.close()
@ -640,7 +681,6 @@ class Site {
this.onDetect(analyzeDom(dom))
this.onDetect(analyzeJs(js))
this.onDetect(
analyze({
url,

@ -19,6 +19,7 @@
"termsContent": { "message": "Aquesta extensió envia informació anònima sobre els llocs web que visiteu, inclosos el nom de domini i les tecnologies identificades a <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Això pot desactivar-se a Opcions." },
"privacyPolicy": { "message": "Política de privadesa" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Taulers de missatgeria" },
"categoryName3": { "message": "Gestor de bases de dades" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Diese Erweiterung sendet anonyme Informationen über Websites, die Sie besuchen, einschließlich der Domain und der identifizierten Technologien, an <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Dies kann in den Einstellungen deaktiviert werden." },
"privacyPolicy": { "message": "Datenschutzerklärung" },
"createAlert": { "message": "Alarm für diese Website erstellen" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Nachrichten Board" },
"categoryName3": { "message": "Datenbankverwaltung" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Διαδικτυακό Φόρουμ" },
"categoryName3": { "message": "Διαχειριστής Βάσης Δεδομένων" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Message boards" },
"categoryName3": { "message": "Database managers" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "Gestor de Contenido" },
"categoryName2": { "message": "Foro" },
"categoryName3": { "message": "Gestor de Bases de Datos" },

@ -19,6 +19,7 @@
"termsContent": { "message": "این افزونه اطلاعات وب‌سایت‌های بازدید شده توسط شما را به صورت ناشناس ارسال می‌کند، مانند آدرس سایت و تکنولوژی‌های استفاده شده در آن سایت را ارسال می‌کند. اطلاعات بیشتر در <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. شما می‌توانید این افزونه را غیرفعال کنید." },
"privacyPolicy": { "message": "سیاست حفظ حریم خصوصی" },
"createAlert": { "message": "ساخت یک هشدار برای این وبسایت" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "سیستم مدیریت محتوا" },
"categoryName2": { "message": "انجمن پیام" },
"categoryName3": { "message": "مدیریت پایگاه داده" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Forum" },
"categoryName3": { "message": "Gestionnaire de base de données" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Esta extensión envía anonimamente información acerca das webs que visitas, incluindo dominio e aplicativos identificados, a <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Isto pode ser desactivado nas preferencias." },
"privacyPolicy": { "message": "Política de privacidade" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Taboleiro de mensaxes" },
"categoryName3": { "message": "Xestor de base de datos" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Διαδικτυακό Φόρουμ" },
"categoryName3": { "message": "Διαχειριστής Βάσης Δεδομένων" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "Sistem Pengelola Konten" },
"categoryName2": { "message": "Papan Pesan" },
"categoryName3": { "message": "Pengelola Basis Data" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Forum" },
"categoryName3": { "message": "Gestore di Database" },

@ -19,6 +19,7 @@
"termsContent": { "message": "この拡張機能は、ドメイン名や特定された技術など、アクセスしたWebサイトに関する匿名情報を<a href='https://www.wappalyzer.com'>wappalyzer.com</a>に送信します。これは設定で無効にできます。" },
"privacyPolicy": { "message": "プライバシーポリシー" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "メッセージボード" },
"categoryName3": { "message": "データベースマネージャー" },

@ -19,6 +19,7 @@
"termsContent": { "message": "이 확장 기능은 사이트의 도메인과 식별된 기술을 포함한 익명 정보를 <a href='https://www.wappalyzer.com'>wappalyzer.com</a>에 전송합니다. 이 기능은 설정에서 비활성화 할 수 있습니다." },
"privacyPolicy": { "message": "개인정보처리방침" },
"createAlert": { "message": "이 웹 사이트에 대한 알림 받기" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "포럼 소프트웨어" },
"categoryName3": { "message": "데이터베이스 관리 도구" },

@ -19,6 +19,7 @@
"termsContent": { "message": "To rozszerzenie wysyła anonimowe informacje o stronach, które odwiedzasz, uwzględniając nazwy domen i zidentyfikowane technologie do <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Opcja może zostać wyłączona w ustawieniach." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "System zarządzania treścią" },
"categoryName2": { "message": "Forum" },
"categoryName3": { "message": "Menedżer baz danych" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Esta extensão envia informações anónimas sobre os sites que visitas, incluindo o nome de domínio e as tecnologias identificadas, para o <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Isso pode ser desativado nas configurações." },
"privacyPolicy": { "message": "Políticas de Privacidade" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Fórum" },
"categoryName3": { "message": "Gestor de Base de Dados" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Esta extensão envia informações anônimas sobre os sites que você visita, incluindo domínio e tecnologias identificadas para <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Este comportamento pode ser desativado nas configurações." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Fórum" },
"categoryName3": { "message": "Gestão de Banco de Dados" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Forum de discuții" },
"categoryName3": { "message": "Manager baze de date" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Расширение отправляет обезличенную статистику посещенных сайтов, включая доменное имя и распознанные технологии на <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. Это можно отключить в настройках." },
"privacyPolicy": { "message": "Политика конфиденциальности" },
"createAlert": { "message": "Отправить жалобу на этот сайт" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Форум" },
"categoryName3": { "message": "Менеджер БД" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Message Board" },
"categoryName3": { "message": "Správca databáz" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Bu eklenti, ziyaret ettiğiniz web site bilgilerini, alan adları ve tespit edilen teknolojiler ile beraber anonim olarak <a href='https://www.wappalyzer.com'>wappalyzer.com</a>'a gönderir. Bunu, eklenti ayarlarından değiştirebilirsiniz." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "İçerik Yönetim Sistemi" },
"categoryName2": { "message": "Mesaj Tahtası" },
"categoryName3": { "message": "Veritabanı Yöneticisi" },

@ -19,6 +19,7 @@
"termsContent": { "message": "Це розширення надсилає на <a href=\"https://wappalyzer.com\">Wapplayzer.com</a> анонімну інформацію про відвідувані вами веб-сайти, включаючи доменні імена та визначені технології. Це можна відключити в налаштуваннях." },
"privacyPolicy": { "message": "Політика приватності" },
"createAlert": { "message": "Поскаржитись на цей сайт" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS" },
"categoryName2": { "message": "Форум" },
"categoryName3": { "message": "Менеджер БД" },

@ -19,6 +19,7 @@
"termsContent": { "message": "This extension sends anonymous information about websites you visit, including domain name and identified technologies, to <a href='https://www.wappalyzer.com'>wappalyzer.com</a>. This can be disabled in the settings." },
"privacyPolicy": { "message": "Privacy policy" },
"createAlert": { "message": "Create an alert for this website" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "CMS (KBT)" },
"categoryName2": { "message": "Forum" },
"categoryName3": { "message": "MB boshqaruvi" },

@ -19,6 +19,7 @@
"termsContent": { "message": "此扩展程序会匿名发送您访问的网站信息至 <a href='https://www.wappalyzer.com'>wappalyzer.com</a>,包含域名和检测到的技术。这可以在设置中禁用。" },
"privacyPolicy": { "message": "隐私政策" },
"createAlert": { "message": "为该网站创建提醒" },
"leadLists": { "message": "Website & contact lists" },
"categoryName1": { "message": "内容管理系统CMS" },
"categoryName2": { "message": "信息板" },
"categoryName3": { "message": "数据库管理器" },

@ -44,6 +44,7 @@ a:hover {
border-bottom: 1px solid var(--color-secondary);
display: flex;
height: 4.5rem;
justify-content: space-between;
}
.header__logo {
@ -55,6 +56,20 @@ a:hover {
display: none;
}
.header__cta {
text-align: right;
margin-right: 1.5rem;
white-space: nowrap;
}
.header__open-in-new {
color: var(--color-primary);
height: 1.1rem;
margin-left: .1rem;
vertical-align: middle;
width: 1.1rem;
}
.footer {
align-items: center;
background: #fff;

@ -13,10 +13,18 @@
</head>
<body>
<div class="header">
<a href="https://www.wappalyzer.com/" class="header__link">
<a href="https://www.wappalyzer.com/?utm_source=popup&utm_medium=extension&utm_campaign=wappalyzer" class="header__link">
<img alt="" class="header__logo header__logo--light" src="../images/logo-purple.svg">
<img alt="" class="header__logo header__logo--dark" src="../images/logo-white.svg">
</a>
<div class="header__cta">
<a href="https://www.wappalyzer.com/lists/?utm_source=popup&utm_medium=extension&utm_campaign=wappalyzer" data-i18n="leadLists"></a>
<svg class="header__open-in-new" viewBox="0 0 24 24">
<path fill="currentColor" d="M4,11V13H16L10.5,18.5L11.92,19.92L19.84,12L11.92,4.08L10.5,5.5L16,11H4Z" />
</svg>
</div>
</div>
<div class="empty" data-i18n="noAppsDetected"></div>
@ -28,7 +36,7 @@
<button class="terms__accept" data-i18n="termsAccept">&nbsp;</button>
<a class="terms__privacy" href="https://www.wappalyzer.com/privacy" data-i18n="privacyPolicy"></a>
<a class="terms__privacy" href="https://www.wappalyzer.com/privacy/?utm_source=popup&utm_medium=extension&utm_campaign=wappalyzer" data-i18n="privacyPolicy"></a>
</div>
<div data-template="category" class="category">
@ -75,7 +83,7 @@
<path fill="currentColor" d="M12,8H4A2,2 0 0,0 2,10V14A2,2 0 0,0 4,16H5V20A1,1 0 0,0 6,21H8A1,1 0 0,0 9,20V16H12L17,20V4L12,8M21.5,12C21.5,13.71 20.54,15.26 19,16V8C20.53,8.75 21.5,10.3 21.5,12Z" />
</svg>
<a class="alerts__link" href="https://www.wappalyzer.com/alerts/" data-i18n="createAlert"></a>
<a class="alerts__link" href="https://www.wappalyzer.com/alerts/?utm_source=popup&utm_medium=extension&utm_campaign=wappalyzer" data-i18n="createAlert"></a>
</div>
<div class="spacer"></div>

@ -0,0 +1,3 @@
<svg width="75" height="75" viewBox="0 0 75 75" fill="none" xmlns="http://www.w3.org/2000/svg">
<path opacity="0.54" d="M55.9639 20.9886C53.7919 18.7385 51.2393 16.8901 48.4239 15.5286C45.2666 14.1841 41.8537 13.5459 38.4239 13.6586C35.3811 13.6423 32.3634 14.2092 29.5339 15.3286C26.8161 16.3959 24.3379 17.9937 22.2439 20.0286C20.0836 22.1153 18.3808 24.6285 17.2439 27.4086C14.8171 33.5854 14.8171 40.4517 17.2439 46.6286C18.3841 49.3923 20.0868 51.8885 22.2439 53.9586C24.3828 55.9973 26.891 57.6094 29.6339 58.7086C32.5001 59.8336 35.5549 60.4005 38.6339 60.3786C41.6587 60.446 44.6706 59.9649 47.5239 58.9586C49.9346 58.0771 52.1619 56.7584 54.0939 55.0686C55.3817 53.8696 56.4868 52.4882 57.3739 50.9686C58.3653 49.2364 59.1315 47.3848 59.6539 45.4586H38.2639V33.3286H73.1539C73.3787 34.318 73.5456 35.3197 73.6539 36.3286C73.7935 37.4866 73.8603 38.6522 73.8539 39.8186C73.9081 44.2912 73.1909 48.7396 71.7339 52.9686C70.3367 56.8897 68.1519 60.4834 65.3139 63.5286C62.1032 66.9232 58.207 69.5957 53.8839 71.3686C49.0336 73.304 43.8451 74.2492 38.6239 74.1486C28.7427 74.2273 19.2195 70.4539 12.0739 63.6286C8.65681 60.287 5.91076 56.3223 3.98389 51.9486C1.94455 47.2567 0.928647 42.184 1.00389 37.0686C0.967549 31.9613 1.98899 26.9017 4.00389 22.2086C5.87762 17.7906 8.63004 13.7998 12.0939 10.4786C15.5558 7.15935 19.6319 4.54724 24.0939 2.78856C28.7234 0.922558 33.6726 -0.0217674 38.6639 0.00856564C44.0078 -0.105303 49.3149 0.917263 54.2339 3.00857C58.6494 4.97774 62.622 7.81776 65.9139 11.3586L55.9639 20.9886Z" fill="black"/>
</svg>

After

Width:  |  Height:  |  Size: 1.5 KiB

@ -63,12 +63,6 @@ const Driver = {
['responseHeaders', 'blocking']
)
// chrome.webRequest.onCompleted.addListener(
// Driver.onWebRequestComplete,
// { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
// ['responseHeaders']
// )
chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null))
// Enable messaging between scripts

@ -65,8 +65,9 @@
"webRequest",
"webNavigation",
"webRequestBlocking",
"dns",
"http://*/*",
"https://*/*"
],
"content_security_policy": "script-src 'self'; object-src 'self'"
}
}

@ -1064,6 +1064,21 @@
"icon": "Amazon.svg",
"website": "https://affiliate-program.amazon.com.au/"
},
"Google Workspace": {
"cats": [
30,
61
],
"description": "Google Workspace, formerly G Suite, is a collection of cloud computing, productivity and collaboration tools.",
"dns": {
"MX": [
"aspmx\\.l\\.google\\.com",
"googlemail\\.com"
]
},
"icon": "Google Workspace.svg",
"website": "https://workspace.google.com/"
},
"Amazon Cloudfront": {
"cats": [
31
@ -1073,6 +1088,9 @@
"Via": "\\(CloudFront\\)$",
"X-Amz-Cf-Id": ""
},
"dns": {
"CNAME": "^[a-z0-9]+\\.cloudfront.net$"
},
"icon": "Amazon-Cloudfront.svg",
"implies": "Amazon Web Services",
"website": "http://aws.amazon.com/cloudfront/"
@ -1145,6 +1163,9 @@
"cats": [
62
],
"dns": {
"NS": "\\.awsdns-"
},
"description": "Amazon Web Services (AWS) is a comprehensive cloud services platform offering compute power, database storage, content delivery and other functionality.",
"icon": "aws.svg",
"website": "https://aws.amazon.com/"
@ -12724,6 +12745,9 @@
"__stripe_mid": "",
"__stripe_sid": ""
},
"dns": {
"TXT": "stripe-verification="
},
"description": "Stripe offers online payment processing for internet businesses as well as fraud prevention, invoicing and subscription management.",
"html": "<input[^>]+data-stripe",
"icon": "Stripe.svg",
@ -16707,4 +16731,4 @@
"website": "https://www.xt-commerce.com"
}
}
}
}

@ -187,6 +187,7 @@ const Wappalyzer = {
robots,
meta,
headers,
dns,
certIssuer,
cookies,
scripts,
@ -210,6 +211,7 @@ const Wappalyzer = {
mm(technology, 'cookies', cookies),
mm(technology, 'meta', meta),
mm(technology, 'headers', headers),
mm(technology, 'dns', dns),
])
)
).filter((technology) => technology)
@ -237,6 +239,7 @@ const Wappalyzer = {
robots,
meta,
headers,
dns,
certIssuer,
cookies,
scripts,
@ -254,6 +257,7 @@ const Wappalyzer = {
slug: Wappalyzer.slugify(name),
url: transform(url),
headers: transform(headers),
dns: transform(dns),
cookies: transform(cookies),
dom: transform(dom, true),
html: transform(html),