README.md

@@ -105,6 +105,8 @@ quit()
 - `INVALID_TARGET_LANGUAGE`
 - `UNSUPPORTED_SOURCE_LANGUAGE`
 - `UNSUPPORTED_TARGET_LANGUAGE`
+- `LANGUAGE_SWITCH_FAILED:source` / `LANGUAGE_SWITCH_FAILED:target`
+- `TRANSLATION_RESULT_UNAVAILABLE`
 
 ### Examples
 
@@ -239,3 +241,8 @@ translate('hey, what\'s up ?', 'auto', 'fr-FR').then(console.log);
 * `1.0.11` (2020-08-24)
     - Fixed & improved supported languages parsing
     - Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors
+* `1.0.12`
+    - Fixed random translation errors caused by transient UI selectors not being ready (#2)
+    - Verified language switching actually happened instead of silently keeping the previous language
+    - Retry once on a fresh page when the failure looks transient
+    - Implemented `LANGUAGE_SWITCH_FAILED` & `TRANSLATION_RESULT_UNAVAILABLE` errors

index.js

@@ -1,65 +1,120 @@
 const
 	puppeteer = require('puppeteer'),
 	axios = require('axios'),
-	homepage = 'https://www.deepl.com/translator';
+	homepage = process.env.DEEPL_TRANSLATOR_HOMEPAGE || 'https://www.deepl.com/translator';
 
 let browser;
 
 const getBrowser = async () => {
 	if(browser) return browser;
-	browser = await puppeteer.launch({ args: ['--window-size=1920,1080'] });
+	browser = await puppeteer.launch({
+		executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
+		args: ['--window-size=1920,1080']
+	});
 	browser.on('disconnected', () => browser = null);
 	return browser;
 };
 
 const getNewPage = async () => await (await getBrowser()).newPage();
 
-module.exports = {
+// Puppeteer TimeoutError on old versions is just an Error with this name;
-	translate: async (sentence, sourceLanguage = 'auto', targetLanguage) => {
+// it is indistinguishable from our errors and is transient.
-		if(!/^(auto|[a-z]{2})$/.test(sourceLanguage))
+const isTransient = error => {
-			throw new Error('INVALID_SOURCE_LANGUAGE');
+	if(!error) return false;
-		if(!/^[a-z]{2}-[A-Z]{2}$/.test(targetLanguage))
+	if(error.name === 'TimeoutError') return true;
-			throw new Error('INVALID_TARGET_LANGUAGE');
+	return /Waiting for|No node found|Navigation timeout|TimeoutError/i.test(error.message || '');
+};
 
+// Opens a language dropdown, picks an option, then waits for the menu to close.
+// Several retries are necessary because the dropdown may open while still
+// swapping panels, so a click at the wrong moment is silently ignored.
+const chooseLanguage = async (page, { select, menu, option, name, expected }) => {
+	const maxAttempts = 3;
+	for(let attempt = 1; attempt <= maxAttempts; attempt ++){
+		await page.click(select);
+		await page.waitForSelector(menu, { visible: true });
+		try {
+			await page.click(option);
+		}
+		catch(error){
+			await page.keyboard.press('Escape').catch(() => {});
+			throw error;
+		}
+		await page.waitForSelector(menu, { hidden: true });
+
+		// The click might have targeted the wrong panel / been ignored :
+		// DeepL would then keep the previously selected language.
+		const label = await page.$eval(select, node => node.textContent.trim()).catch(() => '');
+		if(!expected || new RegExp(expected).test(label)) return;
+
+		if(attempt === maxAttempts)
+			throw new Error(`LANGUAGE_SWITCH_FAILED:${name}`);
+	}
+};
+
+const runTranslation = async (page, sentence, sourceLanguage, targetLanguage) => {
 	const
 		sourceLangSelect = '.lmt__language_select--source button',
 		targetLangSelect = '.lmt__language_select--target button',
 		sourceLangMenu = '.lmt__language_select__menu_source',
 		targetLangMenu = '.lmt__language_select__menu_target',
-			sourceLangButton = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`,
+		sourceLangOption = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`,
-			targetLangButton = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`,
+		targetLangOption = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`,
 		originalSentenceField = '.lmt__source_textarea',
-			targetSentenceField = '.lmt__target_textarea'/*,
+		targetSentenceField = '.lmt__target_textarea',
-			 targetSentencesContainer = '.lmt__translations_as_text'*/;
+		sourceLangLabel = { // DeepL language menu labels
-
+			auto: 'Detect language',
-		const page = await getNewPage();
+			en: 'English',
+			de: 'German',
+			fr: 'French',
+			es: 'Spanish',
+			pt: 'Portuguese',
+			it: 'Italian',
+			nl: 'Dutch',
+			pl: 'Polish',
+			ru: 'Russian',
+			ja: 'Japanese',
+			zh: 'Chinese'
+		},
+		targetLangLabel = {
+			'en-US': 'English ( American )',
+			'en-GB': 'English ( British )',
+			'de-DE': 'German',
+			'fr-FR': 'French',
+			'es-ES': 'Spanish',
+			'pt-BR': 'Portuguese ( Brazilian )',
+			'it-IT': 'Italian',
+			'nl-NL': 'Dutch',
+			'pl-PL': 'Polish',
+			'ru-RU': 'Russian',
+			'ja-JA': 'Japanese',
+			'zh-ZH': 'Chinese'
+		};
 
 	await page.goto(homepage);
-		await page.click(sourceLangSelect);
-		await page.waitForSelector(sourceLangMenu, { visible: true });
-		try {
-			await page.click(sourceLangButton);
-		}
-		catch(_){
-			throw new Error('UNSUPPORTED_SOURCE_LANGUAGE');
-		}
-		await page.waitForSelector(sourceLangMenu, { hidden: true });
 
-		await page.click(targetLangSelect);
+	await chooseLanguage(page, {
-		await page.waitForSelector(targetLangMenu, { visible: true });
+		select: sourceLangSelect,
-		try {
+		menu: sourceLangMenu,
-			await page.click(targetLangButton);
+		option: sourceLangOption,
-		}
+		name: 'source',
-		catch(_){
+		expected: sourceLangLabel[sourceLanguage]
-			throw new Error('UNSUPPORTED_TARGET_LANGUAGE');
+	});
-		}
-		await page.waitForSelector(targetLangMenu, { hidden: true });
 
-		await page.waitForSelector(originalSentenceField);
+	await chooseLanguage(page, {
-		await page.type(originalSentenceField, sentence);
+		select: targetLangSelect,
+		menu: targetLangMenu,
+		option: targetLangOption,
+		name: 'target',
+		expected: targetLangLabel[targetLanguage]
+	});
 
-		let sentences = [];
+	await page.waitForSelector(originalSentenceField, { visible: true });
-		let _res = {};
+
+	// Register before typing : the translation request may fire right after the
+	// last keystroke, and attaching afterwards could miss it entirely.
+	const sentences = [];
+	let _res;
 	page.on('requestfinished', request => request.response().json().then(res => {
 		if(!res['result']) return;
 		sentences.push(...res['result']['translations'][0]['beams'].map(item => ({
@@ -80,11 +135,44 @@ module.exports = {
 			}
 		};
 	}).catch(() => {}));
+
+	await page.type(originalSentenceField, sentence);
+
 	await page.waitForSelector('.lmt--active_translation_request');
 	await page.waitForSelector('.lmt--active_translation_request', { hidden: true });
-		_res.target.translation = await page.$eval(targetSentenceField, el => el.value);
+	if(!_res) throw new Error('TRANSLATION_RESULT_UNAVAILABLE');
-		page.close().catch(() => {});
+
+	_res.target.translation = await page.$eval(targetSentenceField, node => node.value);
+	if(!_res.target.translation) throw new Error('TRANSLATION_RESULT_UNAVAILABLE');
 	return _res;
+};
+
+module.exports = {
+	translate: async (sentence, sourceLanguage = 'auto', targetLanguage) => {
+		if(!/^(auto|[a-z]{2})$/.test(sourceLanguage))
+			throw new Error('INVALID_SOURCE_LANGUAGE');
+		if(!/^[a-z]{2}-[A-Z]{2}$/.test(targetLanguage))
+			throw new Error('INVALID_TARGET_LANGUAGE');
+
+		// Transient failures (missing node, exceeded timeout) get a fresh
+		// page and another try instead of failing the whole translation.
+		const maxAttempts = 2;
+		let lastError;
+		for(let attempt = 1; attempt <= maxAttempts; attempt ++){
+			let page;
+			try {
+				page = await getNewPage();
+				const result = await runTranslation(page, sentence, sourceLanguage, targetLanguage);
+				await page.close().catch(() => {});
+				return result;
+			}
+			catch(error){
+				if(page) await page.close().catch(() => {});
+				if(!isTransient(error) || attempt === maxAttempts) throw error;
+				lastError = error;
+			}
+		}
+		throw lastError;
 	},
 	quit: async () => {
 		if(browser)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepl-scraper",
-  "version": "1.0.11",
+  "version": "1.0.12",
   "description": "Scrape data from DeepL translator without applying for the paid authenticated API",
   "keywords": [
     "deepl",