🐛 Fix random translation failures and stuck target language (closes #2) #8
3 changed files with 162 additions and 67 deletions
|
|
@ -105,6 +105,8 @@ quit()
|
|||
- `INVALID_TARGET_LANGUAGE`
|
||||
- `UNSUPPORTED_SOURCE_LANGUAGE`
|
||||
- `UNSUPPORTED_TARGET_LANGUAGE`
|
||||
- `LANGUAGE_SWITCH_FAILED:source` / `LANGUAGE_SWITCH_FAILED:target`
|
||||
- `TRANSLATION_RESULT_UNAVAILABLE`
|
||||
|
||||
### Examples
|
||||
|
||||
|
|
@ -238,4 +240,9 @@ translate('hey, what\'s up ?', 'auto', 'fr-FR').then(console.log);
|
|||
* `1.0.10` (2020-08-21) • Actually remove jsdom NPM dependency
|
||||
* `1.0.11` (2020-08-24)
|
||||
- Fixed & improved supported languages parsing
|
||||
- Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors
|
||||
- Implemented `UNSUPPORTED_SOURCE_LANGUAGE` & `UNSUPPORTED_TARGET_LANGUAGE` errors
|
||||
* `1.0.12`
|
||||
- Fixed random translation errors caused by transient UI selectors not being ready (#2)
|
||||
- Verified language switching actually happened instead of silently keeping the previous language
|
||||
- Retry once on a fresh page when the failure looks transient
|
||||
- Implemented `LANGUAGE_SWITCH_FAILED` & `TRANSLATION_RESULT_UNAVAILABLE` errors
|
||||
218
index.js
218
index.js
|
|
@ -1,19 +1,152 @@
|
|||
const
|
||||
puppeteer = require('puppeteer'),
|
||||
axios = require('axios'),
|
||||
homepage = 'https://www.deepl.com/translator';
|
||||
homepage = process.env.DEEPL_TRANSLATOR_HOMEPAGE || 'https://www.deepl.com/translator';
|
||||
|
||||
let browser;
|
||||
|
||||
const getBrowser = async () => {
|
||||
if(browser) return browser;
|
||||
browser = await puppeteer.launch({ args: ['--window-size=1920,1080'] });
|
||||
browser = await puppeteer.launch({
|
||||
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
|
||||
args: ['--window-size=1920,1080']
|
||||
});
|
||||
browser.on('disconnected', () => browser = null);
|
||||
return browser;
|
||||
};
|
||||
|
||||
const getNewPage = async () => await (await getBrowser()).newPage();
|
||||
|
||||
// Puppeteer TimeoutError on old versions is just an Error with this name;
|
||||
// it is indistinguishable from our errors and is transient.
|
||||
const isTransient = error => {
|
||||
if(!error) return false;
|
||||
if(error.name === 'TimeoutError') return true;
|
||||
return /Waiting for|No node found|Navigation timeout|TimeoutError/i.test(error.message || '');
|
||||
};
|
||||
|
||||
// Opens a language dropdown, picks an option, then waits for the menu to close.
|
||||
// Several retries are necessary because the dropdown may open while still
|
||||
// swapping panels, so a click at the wrong moment is silently ignored.
|
||||
const chooseLanguage = async (page, { select, menu, option, name, expected }) => {
|
||||
const maxAttempts = 3;
|
||||
for(let attempt = 1; attempt <= maxAttempts; attempt ++){
|
||||
await page.click(select);
|
||||
await page.waitForSelector(menu, { visible: true });
|
||||
try {
|
||||
await page.click(option);
|
||||
}
|
||||
catch(error){
|
||||
await page.keyboard.press('Escape').catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
await page.waitForSelector(menu, { hidden: true });
|
||||
|
||||
// The click might have targeted the wrong panel / been ignored :
|
||||
// DeepL would then keep the previously selected language.
|
||||
const label = await page.$eval(select, node => node.textContent.trim()).catch(() => '');
|
||||
if(!expected || new RegExp(expected).test(label)) return;
|
||||
|
||||
if(attempt === maxAttempts)
|
||||
throw new Error(`LANGUAGE_SWITCH_FAILED:${name}`);
|
||||
}
|
||||
};
|
||||
|
||||
const runTranslation = async (page, sentence, sourceLanguage, targetLanguage) => {
|
||||
const
|
||||
sourceLangSelect = '.lmt__language_select--source button',
|
||||
targetLangSelect = '.lmt__language_select--target button',
|
||||
sourceLangMenu = '.lmt__language_select__menu_source',
|
||||
targetLangMenu = '.lmt__language_select__menu_target',
|
||||
sourceLangOption = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`,
|
||||
targetLangOption = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`,
|
||||
originalSentenceField = '.lmt__source_textarea',
|
||||
targetSentenceField = '.lmt__target_textarea',
|
||||
sourceLangLabel = { // DeepL language menu labels
|
||||
auto: 'Detect language',
|
||||
en: 'English',
|
||||
de: 'German',
|
||||
fr: 'French',
|
||||
es: 'Spanish',
|
||||
pt: 'Portuguese',
|
||||
it: 'Italian',
|
||||
nl: 'Dutch',
|
||||
pl: 'Polish',
|
||||
ru: 'Russian',
|
||||
ja: 'Japanese',
|
||||
zh: 'Chinese'
|
||||
},
|
||||
targetLangLabel = {
|
||||
'en-US': 'English ( American )',
|
||||
'en-GB': 'English ( British )',
|
||||
'de-DE': 'German',
|
||||
'fr-FR': 'French',
|
||||
'es-ES': 'Spanish',
|
||||
'pt-BR': 'Portuguese ( Brazilian )',
|
||||
'it-IT': 'Italian',
|
||||
'nl-NL': 'Dutch',
|
||||
'pl-PL': 'Polish',
|
||||
'ru-RU': 'Russian',
|
||||
'ja-JA': 'Japanese',
|
||||
'zh-ZH': 'Chinese'
|
||||
};
|
||||
|
||||
await page.goto(homepage);
|
||||
|
||||
await chooseLanguage(page, {
|
||||
select: sourceLangSelect,
|
||||
menu: sourceLangMenu,
|
||||
option: sourceLangOption,
|
||||
name: 'source',
|
||||
expected: sourceLangLabel[sourceLanguage]
|
||||
});
|
||||
|
||||
await chooseLanguage(page, {
|
||||
select: targetLangSelect,
|
||||
menu: targetLangMenu,
|
||||
option: targetLangOption,
|
||||
name: 'target',
|
||||
expected: targetLangLabel[targetLanguage]
|
||||
});
|
||||
|
||||
await page.waitForSelector(originalSentenceField, { visible: true });
|
||||
|
||||
// Register before typing : the translation request may fire right after the
|
||||
// last keystroke, and attaching afterwards could miss it entirely.
|
||||
const sentences = [];
|
||||
let _res;
|
||||
page.on('requestfinished', request => request.response().json().then(res => {
|
||||
if(!res['result']) return;
|
||||
sentences.push(...res['result']['translations'][0]['beams'].map(item => ({
|
||||
value: item['postprocessed_sentence'],
|
||||
confidence: item['totalLogProb']
|
||||
})));
|
||||
_res = {
|
||||
source: {
|
||||
lang: res['result']['source_lang'].toLowerCase(),
|
||||
...(sourceLanguage === 'auto' ? {
|
||||
confident: !!res['result']['source_lang_is_confident']
|
||||
} : {}),
|
||||
sentence
|
||||
},
|
||||
target: {
|
||||
lang: res['result']['target_lang'].toLowerCase(),
|
||||
sentences: sentences.sort((a, b) => a.confidence - b.confidence).map(item => item.value)
|
||||
}
|
||||
};
|
||||
}).catch(() => {}));
|
||||
|
||||
await page.type(originalSentenceField, sentence);
|
||||
|
||||
await page.waitForSelector('.lmt--active_translation_request');
|
||||
await page.waitForSelector('.lmt--active_translation_request', { hidden: true });
|
||||
if(!_res) throw new Error('TRANSLATION_RESULT_UNAVAILABLE');
|
||||
|
||||
_res.target.translation = await page.$eval(targetSentenceField, node => node.value);
|
||||
if(!_res.target.translation) throw new Error('TRANSLATION_RESULT_UNAVAILABLE');
|
||||
return _res;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
translate: async (sentence, sourceLanguage = 'auto', targetLanguage) => {
|
||||
if(!/^(auto|[a-z]{2})$/.test(sourceLanguage))
|
||||
|
|
@ -21,70 +154,25 @@ module.exports = {
|
|||
if(!/^[a-z]{2}-[A-Z]{2}$/.test(targetLanguage))
|
||||
throw new Error('INVALID_TARGET_LANGUAGE');
|
||||
|
||||
const
|
||||
sourceLangSelect = '.lmt__language_select--source button',
|
||||
targetLangSelect = '.lmt__language_select--target button',
|
||||
sourceLangMenu = '.lmt__language_select__menu_source',
|
||||
targetLangMenu = '.lmt__language_select__menu_target',
|
||||
sourceLangButton = `${sourceLangMenu} [dl-test=translator-lang-option-${sourceLanguage}]`,
|
||||
targetLangButton = `${targetLangMenu} [dl-test=translator-lang-option-${targetLanguage}]`,
|
||||
originalSentenceField = '.lmt__source_textarea',
|
||||
targetSentenceField = '.lmt__target_textarea'/*,
|
||||
targetSentencesContainer = '.lmt__translations_as_text'*/;
|
||||
|
||||
const page = await getNewPage();
|
||||
|
||||
await page.goto(homepage);
|
||||
await page.click(sourceLangSelect);
|
||||
await page.waitForSelector(sourceLangMenu, { visible: true });
|
||||
try {
|
||||
await page.click(sourceLangButton);
|
||||
// Transient failures (missing node, exceeded timeout) get a fresh
|
||||
// page and another try instead of failing the whole translation.
|
||||
const maxAttempts = 2;
|
||||
let lastError;
|
||||
for(let attempt = 1; attempt <= maxAttempts; attempt ++){
|
||||
let page;
|
||||
try {
|
||||
page = await getNewPage();
|
||||
const result = await runTranslation(page, sentence, sourceLanguage, targetLanguage);
|
||||
await page.close().catch(() => {});
|
||||
return result;
|
||||
}
|
||||
catch(error){
|
||||
if(page) await page.close().catch(() => {});
|
||||
if(!isTransient(error) || attempt === maxAttempts) throw error;
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
catch(_){
|
||||
throw new Error('UNSUPPORTED_SOURCE_LANGUAGE');
|
||||
}
|
||||
await page.waitForSelector(sourceLangMenu, { hidden: true });
|
||||
|
||||
await page.click(targetLangSelect);
|
||||
await page.waitForSelector(targetLangMenu, { visible: true });
|
||||
try {
|
||||
await page.click(targetLangButton);
|
||||
}
|
||||
catch(_){
|
||||
throw new Error('UNSUPPORTED_TARGET_LANGUAGE');
|
||||
}
|
||||
await page.waitForSelector(targetLangMenu, { hidden: true });
|
||||
|
||||
await page.waitForSelector(originalSentenceField);
|
||||
await page.type(originalSentenceField, sentence);
|
||||
|
||||
let sentences = [];
|
||||
let _res = {};
|
||||
page.on('requestfinished', request => request.response().json().then(res => {
|
||||
if(!res['result']) return;
|
||||
sentences.push(...res['result']['translations'][0]['beams'].map(item => ({
|
||||
value: item['postprocessed_sentence'],
|
||||
confidence: item['totalLogProb']
|
||||
})));
|
||||
_res = {
|
||||
source: {
|
||||
lang: res['result']['source_lang'].toLowerCase(),
|
||||
...(sourceLanguage === 'auto' ? {
|
||||
confident: !!res['result']['source_lang_is_confident']
|
||||
} : {}),
|
||||
sentence
|
||||
},
|
||||
target: {
|
||||
lang: res['result']['target_lang'].toLowerCase(),
|
||||
sentences: sentences.sort((a, b) => a.confidence - b.confidence).map(item => item.value)
|
||||
}
|
||||
};
|
||||
}).catch(() => {}));
|
||||
await page.waitForSelector('.lmt--active_translation_request');
|
||||
await page.waitForSelector('.lmt--active_translation_request', { hidden: true });
|
||||
_res.target.translation = await page.$eval(targetSentenceField, el => el.value);
|
||||
page.close().catch(() => {});
|
||||
return _res;
|
||||
throw lastError;
|
||||
},
|
||||
quit: async () => {
|
||||
if(browser)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "deepl-scraper",
|
||||
"version": "1.0.11",
|
||||
"version": "1.0.12",
|
||||
"description": "Scrape data from DeepL translator without applying for the paid authenticated API",
|
||||
"keywords": [
|
||||
"deepl",
|
||||
|
|
|
|||
Reference in a new issue