diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 4d1ce77d7..38cf01994 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -1,4 +1,4 @@ -const os = require('os') +// const os = require('os') const fs = require('fs') const dns = require('dns').promises const path = require('path') @@ -22,10 +22,6 @@ const chromiumArgs = [ `--user-data-dir=${CHROMIUM_DATA_DIR || '/tmp/chromium'}`, ] -if (os.arch() === 'arm64') { - chromiumArgs.push('--single-process') -} - const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/ const categories = JSON.parse( @@ -400,6 +396,8 @@ class Site { this.cache = {} this.probed = false + + this.destroyed = false } log(message, source = 'driver', type = 'log') { @@ -425,7 +423,7 @@ class Site { emit(event, params) { if (this.listeners[event]) { - return Promise.all( + return Promise.allSettled( this.listeners[event].map((listener) => listener(params)) ) } @@ -435,7 +433,7 @@ class Site { promise, fallback, errorMessage = 'Operation took too long to complete', - maxWait = this.options.maxWait + maxWait = Math.min(this.options.maxWait, 1000) ) { let timeout = null @@ -470,12 +468,16 @@ class Site { } async goto(url) { + if (this.destroyed) { + return + } + // Return when the URL is a duplicate or maxUrls has been reached if (this.analyzedUrls[url.href]) { return [] } - this.log(`Navigate to ${url}`, 'page') + this.log(`Navigate to ${url}`) this.analyzedUrls[url.href] = { status: 0, @@ -493,7 +495,13 @@ class Site { try { page = await this.browser.newPage() + + if (!page || page.isClosed()) { + throw new Error('Page did not open') + } } catch (error) { + error.message += ` (${url})` + this.error(error) await this.initDriver() @@ -509,9 +517,15 @@ class Site { await page.setRequestInterception(true) + await page.setUserAgent(this.options.userAgent) + page.on('dialog', (dialog) => dialog.dismiss()) - page.on('error', (error) => this.error(error)) + page.on('error', (error) => { + error.message += ` (${url})` + + this.error(error) + }) let responseReceived = false @@ -523,6 +537,8 @@ class Site { try { ;({ hostname } = new URL(request.url())) } catch (error) { + request.abort('blockedbyclient') + return } @@ -563,11 +579,17 @@ class Site { request.continue({ headers }) } } catch (error) { + error.message += ` (${url})` + this.error(error) } }) page.on('response', async (response) => { + if (this.destroyed || !page || page.__closed || page.isClosed()) { + return + } + try { if ( response.status() < 300 && @@ -578,7 +600,15 @@ class Site { await this.onDetect(response.url(), analyze({ scripts })) } + } catch (error) { + if (error.constructor.name !== 'ProtocolError') { + error.message += ` (${url})` + + this.error(error) + } + } + try { if (response.url() === url.href) { this.analyzedUrls[url.href] = { status: response.status(), @@ -625,26 +655,21 @@ class Site { await this.emit('response', { page, response, headers, certIssuer }) } } catch (error) { + error.message += ` (${url})` + this.error(error) } }) - await page.setUserAgent(this.options.userAgent) - try { - try { - await this.promiseTimeout(page.goto(url.href)) - } catch (error) { - if ( - error.constructor.name !== 'TimeoutError' && - error.code !== 'PROMISE_TIMEOUT_ERROR' - ) { - throw error - } - } + await page.goto(url.href) if (page.url() === 'about:blank') { - throw new Error('The website failed to load') + const error = new Error(`The page failed to load (${url})`) + + error.code = 'WAPPALYZER_PAGE_EMPTY' + + throw error } if (!this.options.noScripts) { @@ -665,6 +690,8 @@ class Site { {} ) } catch (error) { + error.message += ` (${url})` + this.error(error) } @@ -906,65 +933,57 @@ class Site { ...this.cache[url.href], }) - await page.close() - - this.log(`Page closed (${url})`) + page.__closed = true - return reducedLinks - } catch (error) { try { await page.close() this.log(`Page closed (${url})`) } catch (error) { - this.log(error) + // Continue } - let hostname = url + return reducedLinks + } catch (error) { + page.__closed = true try { - ;({ hostname } = new URL(url)) + await page.close() + + this.log(`Page closed (${url})`) } catch (error) { // Continue } - if ( - error.constructor.name === 'TimeoutError' || - error.code === 'PROMISE_TIMEOUT_ERROR' - ) { - const newError = new Error( - `The website took too long to respond: ${ - error.message || error - } at ${hostname}` - ) - - newError.code = 'WAPPALYZER_TIMEOUT_ERROR' - - throw newError - } - if (error.message.includes('net::ERR_NAME_NOT_RESOLVED')) { - const newError = new Error( - `Hostname could not be resolved at ${hostname}` - ) + const newError = new Error(`Hostname could not be resolved (${url})`) newError.code = 'WAPPALYZER_DNS_ERROR' throw newError } + if ( + error.constructor.name === 'TimeoutError' || + error.code === 'PROMISE_TIMEOUT_ERROR' + ) { + error.code = 'WAPPALYZER_TIMEOUT_ERROR' + } + + error.message += ` (${url})` + throw error } } async analyze(url = this.originalUrl, index = 1, depth = 1) { - try { - if (this.options.recursive) { - await sleep(this.options.delay * index) - } + if (this.options.recursive) { + await sleep(this.options.delay * index) + } - await Promise.all([ - (async () => { + await Promise.allSettled([ + (async () => { + try { const links = ((await this.goto(url)) || []).filter( ({ href }) => !this.analyzedUrls[href] ) @@ -983,23 +1002,25 @@ class Site { depth + 1 ) } - })(), - (async () => { - if (this.options.probe && !this.probed) { - this.probed = true - - await this.probe(url) + } catch (error) { + this.analyzedUrls[url.href] = { + status: this.analyzedUrls[url.href]?.status || 0, + error: error.message || error.toString(), } - })(), - ]) - } catch (error) { - this.analyzedUrls[url.href] = { - status: this.analyzedUrls[url.href]?.status || 0, - error: error.message || error.toString(), - } - this.error(error) - } + error.message += ` (${url})` + + this.error(error) + } + })(), + (async () => { + if (this.options.probe && !this.probed) { + this.probed = true + + await this.probe(url) + } + })(), + ]) const patterns = this.options.extended ? this.detections.reduce( @@ -1076,6 +1097,8 @@ class Site { return this.promiseTimeout( func(hostname).catch((error) => { if (error.code !== 'ENODATA') { + error.message += ` (${url})` + this.error(error) } @@ -1089,7 +1112,7 @@ class Site { const domain = url.hostname.replace(/^www\./, '') - await Promise.all([ + await Promise.allSettled([ // Static files ...Object.keys(files).map(async (file, index) => { const path = files[file] @@ -1099,7 +1122,7 @@ class Site { const body = await get(new URL(path, url.href), { userAgent: this.options.userAgent, - timeout: Math.min(this.options.maxWait, 3000), + timeout: Math.min(this.options.maxWait, 1000), }) this.log(`Probe ok (${path})`) @@ -1156,7 +1179,7 @@ class Site { const batched = links.splice(0, this.options.batchSize) - await Promise.all( + await Promise.allSettled( batched.map((link, index) => this.analyze(link, index, depth)) ) @@ -1189,7 +1212,7 @@ class Site { ), ] - await Promise.all( + await Promise.allSettled( requires.map(async ({ name, categoryId, technologies }) => { const id = categoryId ? `category:${categoryId}` @@ -1242,9 +1265,11 @@ class Site { } async destroy() { - await Promise.all( + await Promise.allSettled( this.pages.map(async (page) => { if (page) { + page.__closed = true + try { await page.close() } catch (error) { @@ -1254,6 +1279,8 @@ class Site { }) ) + this.destroyed = true + this.log('Site closed') } } diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index a2cb12dcb..be190cf96 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.10.27", + "version": "6.10.35", "author": "Wappalyzer", "license": "MIT", "repository": { @@ -38,7 +38,7 @@ "wappalyzer": "./cli.js" }, "dependencies": { - "puppeteer": "^13.5.2" + "puppeteer": "~14.1.0" }, "engines": { "node": ">=14" diff --git a/src/drivers/webextension/images/icons/ContentSquare.png b/src/drivers/webextension/images/icons/ContentSquare.png deleted file mode 100644 index 22b324e50..000000000 Binary files a/src/drivers/webextension/images/icons/ContentSquare.png and /dev/null differ diff --git a/src/drivers/webextension/images/icons/Contentsquare.svg b/src/drivers/webextension/images/icons/Contentsquare.svg new file mode 100644 index 000000000..0fc093b9f --- /dev/null +++ b/src/drivers/webextension/images/icons/Contentsquare.svg @@ -0,0 +1,3 @@ + + + diff --git a/src/drivers/webextension/images/icons/LiveSession.svg b/src/drivers/webextension/images/icons/LiveSession.svg new file mode 100644 index 000000000..74e3995f8 --- /dev/null +++ b/src/drivers/webextension/images/icons/LiveSession.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/drivers/webextension/images/icons/MageWorx.svg b/src/drivers/webextension/images/icons/MageWorx.svg new file mode 100644 index 000000000..538684e3f --- /dev/null +++ b/src/drivers/webextension/images/icons/MageWorx.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/drivers/webextension/images/icons/SabaVision.png b/src/drivers/webextension/images/icons/SabaVision.png new file mode 100644 index 000000000..09b9ab94e Binary files /dev/null and b/src/drivers/webextension/images/icons/SabaVision.png differ diff --git a/src/drivers/webextension/images/icons/Smartlook.svg b/src/drivers/webextension/images/icons/Smartlook.svg new file mode 100644 index 000000000..224de6f86 --- /dev/null +++ b/src/drivers/webextension/images/icons/Smartlook.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/drivers/webextension/images/icons/Yektanet.png b/src/drivers/webextension/images/icons/Yektanet.png new file mode 100644 index 000000000..d0e50e736 Binary files /dev/null and b/src/drivers/webextension/images/icons/Yektanet.png differ diff --git a/src/drivers/webextension/images/icons/atws.png b/src/drivers/webextension/images/icons/atws.png new file mode 100644 index 000000000..340ef1005 Binary files /dev/null and b/src/drivers/webextension/images/icons/atws.png differ diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index 41c3095a3..5c2cb2850 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.10.27", + "version": "6.10.35", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/package.json b/src/package.json index 0254441fc..82a980796 100644 --- a/src/package.json +++ b/src/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.10.27", + "version": "6.10.35", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/technologies/c.json b/src/technologies/c.json index b70343cc5..6bc1bab1f 100644 --- a/src/technologies/c.json +++ b/src/technologies/c.json @@ -2292,13 +2292,13 @@ }, "website": "http://www.gocontentbox.org" }, - "ContentSquare": { + "Contentsquare": { "cats": [ 10, 74 ], - "description": "ContentSquare is an enterprise-level UX optimisation platform.", - "icon": "ContentSquare.png", + "description": "Contentsquare is an enterprise-level UX optimisation platform.", + "icon": "Contentsquare.svg", "js": { "CS_CONF.trackerDomain": "" }, diff --git a/src/technologies/g.json b/src/technologies/g.json index 47c0eb977..16b2bf287 100644 --- a/src/technologies/g.json +++ b/src/technologies/g.json @@ -1195,7 +1195,8 @@ "Goog_AdSense_": "", "Goog_AdSense_OsdAdapter": "", "__google_ad_urls": "", - "google_ad_": "" + "google_ad_": "", + "adsbygoogle": "" }, "saas": true, "scriptSrc": [ @@ -1204,7 +1205,7 @@ "2mdn\\.net", "ad\\.ca\\.doubleclick\\.net" ], - "website": "https://www.google.fr/adsense/start/" + "website": "https://www.google.com/adsense/start/" }, "Google Ads": { "cats": [ diff --git a/src/technologies/l.json b/src/technologies/l.json index 0bcb01341..40a16583f 100644 --- a/src/technologies/l.json +++ b/src/technologies/l.json @@ -1647,5 +1647,22 @@ "implies": "YouTube", "oss": true, "website": "https://github.com/paulirish/lite-youtube-embed" + }, + "LiveSession": { + "cats": [ + 10 + ], + "description": "LiveSession helps increase conversion rates using session replays, and event-based product analytics.", + "icon": "LiveSession.svg", + "scriptSrc": [ + "cdn\\.livesession\\.io" + ], + "website": "https://livesession.io/", + "saas": true, + "pricing": [ + "low", + "freemium", + "recurring" + ] } -} \ No newline at end of file +} diff --git a/src/technologies/m.json b/src/technologies/m.json index b34e62aca..4ea0df965 100644 --- a/src/technologies/m.json +++ b/src/technologies/m.json @@ -268,6 +268,18 @@ ], "website": "https://magento.com" }, + "MageWorx Search Autocomplete": { + "cats": [ + 29 + ], + "description": "MageWorx Search Autocomplete extension offers an AJAX-based popup window that displays and updates relevant search results while a customer forms his or her query.", + "icon": "MageWorx.svg", + "requires": "Magento", + "dom":"link[href*='MageWorx_SearchSuiteAutocomplete']", + "oss": true, + "scriptSrc": "MageWorx_SearchSuiteAutocomplete", + "website": "https://github.com/mageworx/search-suite-autocomplete" + }, "Magisto": { "cats": [ 14 @@ -1298,7 +1310,8 @@ "description": "ASP.NET is an open-source, server-side web-application framework designed for web development to produce dynamic web pages.", "headers": { "X-AspNet-Version": "(.+)\\;version:\\1", - "X-Powered-By": "^ASP\\.NET" + "X-Powered-By": "^ASP\\.NET", + "set-cookie": "\\.AspNetCore" }, "html": "]+name=\"__VIEWSTATE", "icon": "Microsoft ASP.NET.svg", diff --git a/src/technologies/s.json b/src/technologies/s.json index d32baa9e5..e6ee209e0 100644 --- a/src/technologies/s.json +++ b/src/technologies/s.json @@ -344,6 +344,23 @@ ], "website": "https://saba.host" }, + "SabaVision": { + "cats": [ + 36 + ], + "description": "SabaVision, one of the core products of SabaIdea, is Iran's largest online advertising agency.", + "icon": "SabaVision.png", + "meta": { + "sabavision_zone": "" + }, + "js": { + "sabaVisionWebsitePage": "", + "sabaVisionWebsiteID": "", + "SabavisionElement": "", + "__SABAVISION_GET_ADD_TIMEOUT": "" + }, + "website": "https://www.sabavision.com" + }, "Saber": { "cats": [ 57 @@ -3122,6 +3139,25 @@ "scriptSrc": "\\.smartling\\.com/", "website": "https://www.smartling.com" }, + "Smartlook": { + "cats": [ + 10 + ], + "description": "Smartlook is a qualitative analytics solution for websites and mobile apps.", + "icon": "Smartlook.svg", + "js": { + "smartlook": "\\;confidence:50", + "smartlook_key": "\\;confidence:50" + }, + "pricing": [ + "freemium", + "low", + "recurring" + ], + "saas": true, + "scriptSrc": "\\.smartlook\\.com/", + "website": "https://www.smartlook.com" + }, "Smartstore": { "cats": [ 6 diff --git a/src/technologies/t.json b/src/technologies/t.json index 2dd96c310..562f3ef6d 100644 --- a/src/technologies/t.json +++ b/src/technologies/t.json @@ -407,6 +407,25 @@ "scriptSrc": "cdn\\.tamara\\.co", "website": "https://tamara.co/" }, + "Tangled Network": { + "cats": [ + 88 + ], + "description": "Tangled Network provides a managed services in website devleopment, web and database hosting and domain registration, with a focus on everything managed for small and medium sized businesses.", + "dns": { + "NS": "\\.tanglednetwork\\.com", + "SOA": "\\.tanglednetwork\\.com" + }, + "headers": { + "X-Hosting-Provider": "Tangled Network" + }, + "icon": "atws.png", + "pricing": [ + "low", + "recurring" + ], + "website": "https://tanglednetwork.com" + }, "Tapad": { "cats": [ 36 @@ -2757,4 +2776,4 @@ "implies": "Node.js", "website": "https://totaljs.com" } -} \ No newline at end of file +} diff --git a/src/technologies/y.json b/src/technologies/y.json index 7fdc0f2b7..a24de2ff6 100644 --- a/src/technologies/y.json +++ b/src/technologies/y.json @@ -209,6 +209,20 @@ "icon": "Yaws.png", "website": "http://yaws.hyber.org" }, + "Yektanet": { + "cats": [ + 36 + ], + "description": "Yektanet is the biggest and most advanced native advertising network in Iran.", + "icon": "Yektanet.png", + "meta": { + "yektanet_session_last_activity": "" + }, + "js": { + "yektanet": "" + }, + "website": "https://www.yektanet.com" + }, "Yelp Reservations": { "cats": [ 93