diff --git a/README.md b/README.md index 739c029..ecf4977 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ quit() - `INVALID_TARGET_LANGUAGE` - `UNSUPPORTED_SOURCE_LANGUAGE` - `UNSUPPORTED_TARGET_LANGUAGE` +- `LANGUAGE_SWITCH_FAILED:source` / `LANGUAGE_SWITCH_FAILED:target` +- `TRANSLATION_RESULT_UNAVAILABLE` ### Examples @@ -238,4 +240,9 @@ translate('hey, what\'s up ?', 'auto', 'fr-FR').then(console.log); * `1.0.10` (2020-08-21) • Actually remove jsdom NPM dependency * `1.0.11` (2020-08-24) - Fixed & improved supported languages parsing - - Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors \ No newline at end of file + - Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors +* `1.0.12` + - Fixed random translation errors caused by transient UI selectors not being ready (#2) + - Verified language switching actually happened instead of silently keeping the previous language + - Retry once on a fresh page when the failure looks transient + - Implemented `LANGUAGE_SWITCH_FAILED` & `TRANSLATION_RESULT_UNAVAILABLE` errors \ No newline at end of file diff --git a/index.js b/index.js index 9802d19..03a40d4 100644 --- a/index.js +++ b/index.js @@ -1,19 +1,152 @@ const puppeteer = require('puppeteer'), axios = require('axios'), - homepage = 'https://www.deepl.com/translator'; + homepage = process.env.DEEPL_TRANSLATOR_HOMEPAGE || 'https://www.deepl.com/translator'; let browser; const getBrowser = async () => { if(browser) return browser; - browser = await puppeteer.launch({ args: ['--window-size=1920,1080'] }); + browser = await puppeteer.launch({ + executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined, + args: ['--window-size=1920,1080'] + }); browser.on('disconnected', () => browser = null); return browser; }; const getNewPage = async () => await (await getBrowser()).newPage(); +// Puppeteer TimeoutError on old versions is just an Error with this name; +// it is indistinguishable from our errors and is transient. +const isTransient = error => { + if(!error) return false; + if(error.name === 'TimeoutError') return true; + return /Waiting for|No node found|Navigation timeout|TimeoutError/i.test(error.message || ''); +}; + +// Opens a language dropdown, picks an option, then waits for the menu to close. +// Several retries are necessary because the dropdown may open while still +// swapping panels, so a click at the wrong moment is silently ignored. +const chooseLanguage = async (page, { select, menu, option, name, expected }) => { + const maxAttempts = 3; + for(let attempt = 1; attempt <= maxAttempts; attempt ++){ + await page.click(select); + await page.waitForSelector(menu, { visible: true }); + try { + await page.click(option); + } + catch(error){ + await page.keyboard.press('Escape').catch(() => {}); + throw error; + } + await page.waitForSelector(menu, { hidden: true }); + + // The click might have targeted the wrong panel / been ignored : + // DeepL would then keep the previously selected language. + const label = await page.$eval(select, node => node.textContent.trim()).catch(() => ''); + if(!expected || new RegExp(expected).test(label)) return; + + if(attempt === maxAttempts) + throw new Error(`LANGUAGE_SWITCH_FAILED:${name}`); + } +}; + +const runTranslation = async (page, sentence, sourceLanguage, targetLanguage) => { + const + sourceLangSelect = '.lmt__language_select--source button', + targetLangSelect = '.lmt__language_select--target button', + sourceLangMenu = '.lmt__language_select__menu_source', + targetLangMenu = '.lmt__language_select__menu_target', + sourceLangOption = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`, + targetLangOption = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`, + originalSentenceField = '.lmt__source_textarea', + targetSentenceField = '.lmt__target_textarea', + sourceLangLabel = { // DeepL language menu labels + auto: 'Detect language', + en: 'English', + de: 'German', + fr: 'French', + es: 'Spanish', + pt: 'Portuguese', + it: 'Italian', + nl: 'Dutch', + pl: 'Polish', + ru: 'Russian', + ja: 'Japanese', + zh: 'Chinese' + }, + targetLangLabel = { + 'en-US': 'English ( American )', + 'en-GB': 'English ( British )', + 'de-DE': 'German', + 'fr-FR': 'French', + 'es-ES': 'Spanish', + 'pt-BR': 'Portuguese ( Brazilian )', + 'it-IT': 'Italian', + 'nl-NL': 'Dutch', + 'pl-PL': 'Polish', + 'ru-RU': 'Russian', + 'ja-JA': 'Japanese', + 'zh-ZH': 'Chinese' + }; + + await page.goto(homepage); + + await chooseLanguage(page, { + select: sourceLangSelect, + menu: sourceLangMenu, + option: sourceLangOption, + name: 'source', + expected: sourceLangLabel[sourceLanguage] + }); + + await chooseLanguage(page, { + select: targetLangSelect, + menu: targetLangMenu, + option: targetLangOption, + name: 'target', + expected: targetLangLabel[targetLanguage] + }); + + await page.waitForSelector(originalSentenceField, { visible: true }); + + // Register before typing : the translation request may fire right after the + // last keystroke, and attaching afterwards could miss it entirely. + const sentences = []; + let _res; + page.on('requestfinished', request => request.response().json().then(res => { + if(!res['result']) return; + sentences.push(...res['result']['translations'][0]['beams'].map(item => ({ + value: item['postprocessed_sentence'], + confidence: item['totalLogProb'] + }))); + _res = { + source: { + lang: res['result']['source_lang'].toLowerCase(), + ...(sourceLanguage === 'auto' ? { + confident: !!res['result']['source_lang_is_confident'] + } : {}), + sentence + }, + target: { + lang: res['result']['target_lang'].toLowerCase(), + sentences: sentences.sort((a, b) => a.confidence - b.confidence).map(item => item.value) + } + }; + }).catch(() => {})); + + await page.type(originalSentenceField, sentence); + + await page.waitForSelector('.lmt--active_translation_request'); + await page.waitForSelector('.lmt--active_translation_request', { hidden: true }); + if(!_res) throw new Error('TRANSLATION_RESULT_UNAVAILABLE'); + + _res.target.translation = await page.$eval(targetSentenceField, node => node.value); + if(!_res.target.translation) throw new Error('TRANSLATION_RESULT_UNAVAILABLE'); + return _res; +}; + module.exports = { translate: async (sentence, sourceLanguage = 'auto', targetLanguage) => { if(!/^(auto|[a-z]{2})$/.test(sourceLanguage)) @@ -21,70 +154,25 @@ module.exports = { if(!/^[a-z]{2}-[A-Z]{2}$/.test(targetLanguage)) throw new Error('INVALID_TARGET_LANGUAGE'); - const - sourceLangSelect = '.lmt__language_select--source button', - targetLangSelect = '.lmt__language_select--target button', - sourceLangMenu = '.lmt__language_select__menu_source', - targetLangMenu = '.lmt__language_select__menu_target', - sourceLangButton = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`, - targetLangButton = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`, - originalSentenceField = '.lmt__source_textarea', - targetSentenceField = '.lmt__target_textarea'/*, - targetSentencesContainer = '.lmt__translations_as_text'*/; - - const page = await getNewPage(); - - await page.goto(homepage); - await page.click(sourceLangSelect); - await page.waitForSelector(sourceLangMenu, { visible: true }); - try { - await page.click(sourceLangButton); + // Transient failures (missing node, exceeded timeout) get a fresh + // page and another try instead of failing the whole translation. + const maxAttempts = 2; + let lastError; + for(let attempt = 1; attempt <= maxAttempts; attempt ++){ + let page; + try { + page = await getNewPage(); + const result = await runTranslation(page, sentence, sourceLanguage, targetLanguage); + await page.close().catch(() => {}); + return result; + } + catch(error){ + if(page) await page.close().catch(() => {}); + if(!isTransient(error) || attempt === maxAttempts) throw error; + lastError = error; + } } - catch(_){ - throw new Error('UNSUPPORTED_SOURCE_LANGUAGE'); - } - await page.waitForSelector(sourceLangMenu, { hidden: true }); - - await page.click(targetLangSelect); - await page.waitForSelector(targetLangMenu, { visible: true }); - try { - await page.click(targetLangButton); - } - catch(_){ - throw new Error('UNSUPPORTED_TARGET_LANGUAGE'); - } - await page.waitForSelector(targetLangMenu, { hidden: true }); - - await page.waitForSelector(originalSentenceField); - await page.type(originalSentenceField, sentence); - - let sentences = []; - let _res = {}; - page.on('requestfinished', request => request.response().json().then(res => { - if(!res['result']) return; - sentences.push(...res['result']['translations'][0]['beams'].map(item => ({ - value: item['postprocessed_sentence'], - confidence: item['totalLogProb'] - }))); - _res = { - source: { - lang: res['result']['source_lang'].toLowerCase(), - ...(sourceLanguage === 'auto' ? { - confident: !!res['result']['source_lang_is_confident'] - } : {}), - sentence - }, - target: { - lang: res['result']['target_lang'].toLowerCase(), - sentences: sentences.sort((a, b) => a.confidence - b.confidence).map(item => item.value) - } - }; - }).catch(() => {})); - await page.waitForSelector('.lmt--active_translation_request'); - await page.waitForSelector('.lmt--active_translation_request', { hidden: true }); - _res.target.translation = await page.$eval(targetSentenceField, el => el.value); - page.close().catch(() => {}); - return _res; + throw lastError; }, quit: async () => { if(browser) diff --git a/package.json b/package.json index 6f8e588..88d05eb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "deepl-scraper", - "version": "1.0.11", + "version": "1.0.12", "description": "Scrape data from DeepL translator without applying for the paid authenticated API", "keywords": [ "deepl",