README.md

@@ -105,6 +105,8 @@ quit()
 - `INVALID_TARGET_LANGUAGE`
 - `UNSUPPORTED_SOURCE_LANGUAGE`
 - `UNSUPPORTED_TARGET_LANGUAGE`
+- `LANGUAGE_SWITCH_FAILED:source` / `LANGUAGE_SWITCH_FAILED:target`
+- `TRANSLATION_RESULT_UNAVAILABLE`
 
 ### Examples
 
@@ -238,4 +240,9 @@ translate('hey, what\'s up ?', 'auto', 'fr-FR').then(console.log);
 * `1.0.10` (2020-08-21) • Actually remove jsdom NPM dependency
 * `1.0.11` (2020-08-24)
     - Fixed & improved supported languages parsing
-    - Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors
+    - Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors
+* `1.0.12`
+    - Fixed random translation errors caused by transient UI selectors not being ready (#2)
+    - Verified language switching actually happened instead of silently keeping the previous language
+    - Retry once on a fresh page when the failure looks transient
+    - Implemented `LANGUAGE_SWITCH_FAILED` & `TRANSLATION_RESULT_UNAVAILABLE` errors

index.js

@@ -1,19 +1,152 @@
 const
 	puppeteer = require('puppeteer'),
 	axios = require('axios'),
-	homepage = 'https://www.deepl.com/translator';
+	homepage = process.env.DEEPL_TRANSLATOR_HOMEPAGE || 'https://www.deepl.com/translator';
 
 let browser;
 
 const getBrowser = async () => {
 	if(browser) return browser;
-	browser = await puppeteer.launch({ args: ['--window-size=1920,1080'] });
+	browser = await puppeteer.launch({
+		executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
+		args: ['--window-size=1920,1080']
+	});
 	browser.on('disconnected', () => browser = null);
 	return browser;
 };
 
 const getNewPage = async () => await (await getBrowser()).newPage();
 
+// Puppeteer TimeoutError on old versions is just an Error with this name;
+// it is indistinguishable from our errors and is transient.
+const isTransient = error => {
+	if(!error) return false;
+	if(error.name === 'TimeoutError') return true;
+	return /Waiting for|No node found|Navigation timeout|TimeoutError/i.test(error.message || '');
+};
+
+// Opens a language dropdown, picks an option, then waits for the menu to close.
+// Several retries are necessary because the dropdown may open while still
+// swapping panels, so a click at the wrong moment is silently ignored.
+const chooseLanguage = async (page, { select, menu, option, name, expected }) => {
+	const maxAttempts = 3;
+	for(let attempt = 1; attempt <= maxAttempts; attempt ++){
+		await page.click(select);
+		await page.waitForSelector(menu, { visible: true });
+		try {
+			await page.click(option);
+		}
+		catch(error){
+			await page.keyboard.press('Escape').catch(() => {});
+			throw error;
+		}
+		await page.waitForSelector(menu, { hidden: true });
+
+		// The click might have targeted the wrong panel / been ignored :
+		// DeepL would then keep the previously selected language.
+		const label = await page.$eval(select, node => node.textContent.trim()).catch(() => '');
+		if(!expected || new RegExp(expected).test(label)) return;
+
+		if(attempt === maxAttempts)
+			throw new Error(`LANGUAGE_SWITCH_FAILED:${name}`);
+	}
+};
+
+const runTranslation = async (page, sentence, sourceLanguage, targetLanguage) => {
+	const
+		sourceLangSelect = '.lmt__language_select--source button',
+		targetLangSelect = '.lmt__language_select--target button',
+		sourceLangMenu = '.lmt__language_select__menu_source',
+		targetLangMenu = '.lmt__language_select__menu_target',
+		sourceLangOption = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`,
+		targetLangOption = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`,
+		originalSentenceField = '.lmt__source_textarea',
+		targetSentenceField = '.lmt__target_textarea',
+		sourceLangLabel = { // DeepL language menu labels
+			auto: 'Detect language',
+			en: 'English',
+			de: 'German',
+			fr: 'French',
+			es: 'Spanish',
+			pt: 'Portuguese',
+			it: 'Italian',
+			nl: 'Dutch',
+			pl: 'Polish',
+			ru: 'Russian',
+			ja: 'Japanese',
+			zh: 'Chinese'
+		},
+		targetLangLabel = {
+			'en-US': 'English ( American )',
+			'en-GB': 'English ( British )',
+			'de-DE': 'German',
+			'fr-FR': 'French',
+			'es-ES': 'Spanish',
+			'pt-BR': 'Portuguese ( Brazilian )',
+			'it-IT': 'Italian',
+			'nl-NL': 'Dutch',
+			'pl-PL': 'Polish',
+			'ru-RU': 'Russian',
+			'ja-JA': 'Japanese',
+			'zh-ZH': 'Chinese'
+		};
+
+	await page.goto(homepage);
+
+	await chooseLanguage(page, {
+		select: sourceLangSelect,
+		menu: sourceLangMenu,
+		option: sourceLangOption,
+		name: 'source',
+		expected: sourceLangLabel[sourceLanguage]
+	});
+
+	await chooseLanguage(page, {
+		select: targetLangSelect,
+		menu: targetLangMenu,
+		option: targetLangOption,
+		name: 'target',
+		expected: targetLangLabel[targetLanguage]
+	});
+
+	await page.waitForSelector(originalSentenceField, { visible: true });
+
+	// Register before typing : the translation request may fire right after the
+	// last keystroke, and attaching afterwards could miss it entirely.
+	const sentences = [];
+	let _res;
+	page.on('requestfinished', request => request.response().json().then(res => {
+		if(!res['result']) return;
+		sentences.push(...res['result']['translations'][0]['beams'].map(item => ({
+			value: item['postprocessed_sentence'],
+			confidence: item['totalLogProb']
+		})));
+		_res = {
+			source: {
+				lang: res['result']['source_lang'].toLowerCase(),
+				...(sourceLanguage === 'auto' ? {
+					confident: !!res['result']['source_lang_is_confident']
+				} : {}),
+				sentence
+			},
+			target: {
+				lang: res['result']['target_lang'].toLowerCase(),
+				sentences: sentences.sort((a, b) => a.confidence - b.confidence).map(item => item.value)
+			}
+		};
+	}).catch(() => {}));
+
+	await page.type(originalSentenceField, sentence);
+
+	await page.waitForSelector('.lmt--active_translation_request');
+	await page.waitForSelector('.lmt--active_translation_request', { hidden: true });
+	if(!_res) throw new Error('TRANSLATION_RESULT_UNAVAILABLE');
+
+	_res.target.translation = await page.$eval(targetSentenceField, node => node.value);
+	if(!_res.target.translation) throw new Error('TRANSLATION_RESULT_UNAVAILABLE');
+	return _res;
+};
+
 module.exports = {
 	translate: async (sentence, sourceLanguage = 'auto', targetLanguage) => {
 		if(!/^(auto|[a-z]{2})$/.test(sourceLanguage))
@@ -21,70 +154,25 @@ module.exports = {
 		if(!/^[a-z]{2}-[A-Z]{2}$/.test(targetLanguage))
 			throw new Error('INVALID_TARGET_LANGUAGE');
 
-		const
+		// Transient failures (missing node, exceeded timeout) get a fresh
-			sourceLangSelect = '.lmt__language_select--source button',
+		// page and another try instead of failing the whole translation.
-			targetLangSelect = '.lmt__language_select--target button',
+		const maxAttempts = 2;
-			sourceLangMenu = '.lmt__language_select__menu_source',
+		let lastError;
-			targetLangMenu = '.lmt__language_select__menu_target',
+		for(let attempt = 1; attempt <= maxAttempts; attempt ++){
-			sourceLangButton = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`,
+			let page;
-			targetLangButton = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`,
+			try {
-			originalSentenceField = '.lmt__source_textarea',
+				page = await getNewPage();
-			targetSentenceField = '.lmt__target_textarea'/*,
+				const result = await runTranslation(page, sentence, sourceLanguage, targetLanguage);
-			 targetSentencesContainer = '.lmt__translations_as_text'*/;
+				await page.close().catch(() => {});
-
+				return result;
-		const page = await getNewPage();
+			}
-
+			catch(error){
-		await page.goto(homepage);
+				if(page) await page.close().catch(() => {});
-		await page.click(sourceLangSelect);
+				if(!isTransient(error) || attempt === maxAttempts) throw error;
-		await page.waitForSelector(sourceLangMenu, { visible: true });
+				lastError = error;
-		try {
+			}
-			await page.click(sourceLangButton);
 		}
-		catch(_){
+		throw lastError;
-			throw new Error('UNSUPPORTED_SOURCE_LANGUAGE');
-		}
-		await page.waitForSelector(sourceLangMenu, { hidden: true });
-
-		await page.click(targetLangSelect);
-		await page.waitForSelector(targetLangMenu, { visible: true });
-		try {
-			await page.click(targetLangButton);
-		}
-		catch(_){
-			throw new Error('UNSUPPORTED_TARGET_LANGUAGE');
-		}
-		await page.waitForSelector(targetLangMenu, { hidden: true });
-
-		await page.waitForSelector(originalSentenceField);
-		await page.type(originalSentenceField, sentence);
-
-		let sentences = [];
-		let _res = {};
-		page.on('requestfinished', request => request.response().json().then(res => {
-			if(!res['result']) return;
-			sentences.push(...res['result']['translations'][0]['beams'].map(item => ({
-				value: item['postprocessed_sentence'],
-				confidence: item['totalLogProb']
-			})));
-			_res = {
-				source: {
-					lang: res['result']['source_lang'].toLowerCase(),
-					...(sourceLanguage === 'auto' ? {
-						confident: !!res['result']['source_lang_is_confident']
-					} : {}),
-					sentence
-				},
-				target: {
-					lang: res['result']['target_lang'].toLowerCase(),
-					sentences: sentences.sort((a, b) => a.confidence - b.confidence).map(item => item.value)
-				}
-			};
-		}).catch(() => {}));
-		await page.waitForSelector('.lmt--active_translation_request');
-		await page.waitForSelector('.lmt--active_translation_request', { hidden: true });
-		_res.target.translation = await page.$eval(targetSentenceField, el => el.value);
-		page.close().catch(() => {});
-		return _res;
 	},
 	quit: async () => {
 		if(browser)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepl-scraper",
-  "version": "1.0.11",
+  "version": "1.0.12",
   "description": "Scrape data from DeepL translator without applying for the paid authenticated API",
   "keywords": [
     "deepl",