fix(scripts): resolve i18n check script path and logic issues (#23069)
This commit is contained in:
lyzno1
@@ -1,5 +1,6 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const transpile = require('typescript').transpile
|
||||
const magicast = require('magicast')
|
||||
const { parseModule, generateCode, loadFile } = magicast
|
||||
@@ -22,11 +23,16 @@ const languageKeyMap = data.languages.reduce((map, language) => {
|
||||
}, {})
|
||||
|
||||
async function translateMissingKeyDeeply(sourceObj, targetObject, toLanguage) {
|
||||
const skippedKeys = []
|
||||
const translatedKeys = []
|
||||
|
||||
await Promise.all(Object.keys(sourceObj).map(async (key) => {
|
||||
if (targetObject[key] === undefined) {
|
||||
if (typeof sourceObj[key] === 'object') {
|
||||
targetObject[key] = {}
|
||||
await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage)
|
||||
const result = await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage)
|
||||
skippedKeys.push(...result.skipped)
|
||||
translatedKeys.push(...result.translated)
|
||||
}
|
||||
else {
|
||||
try {
|
||||
@@ -35,73 +41,198 @@ async function translateMissingKeyDeeply(sourceObj, targetObject, toLanguage) {
|
||||
targetObject[key] = ''
|
||||
return
|
||||
}
|
||||
// not support translate with '(' or ')'
|
||||
if (source.includes('(') || source.includes(')'))
|
||||
return
|
||||
|
||||
// Only skip obvious code patterns, not normal text with parentheses
|
||||
const codePatterns = [
|
||||
/\{\{.*\}\}/, // Template variables like {{key}}
|
||||
/\$\{.*\}/, // Template literals ${...}
|
||||
/<[^>]+>/, // HTML/XML tags
|
||||
/function\s*\(/, // Function definitions
|
||||
/=\s*\(/, // Assignment with function calls
|
||||
]
|
||||
|
||||
const isCodeLike = codePatterns.some(pattern => pattern.test(source))
|
||||
if (isCodeLike) {
|
||||
console.log(`⏭️ Skipping code-like content: "${source.substring(0, 50)}..."`)
|
||||
skippedKeys.push(`${key}: ${source}`)
|
||||
return
|
||||
}
|
||||
|
||||
console.log(`🔄 Translating: "${source}" to ${toLanguage}`)
|
||||
const { translation } = await translate(sourceObj[key], null, languageKeyMap[toLanguage])
|
||||
targetObject[key] = translation
|
||||
translatedKeys.push(`${key}: ${translation}`)
|
||||
console.log(`✅ Translated: "${translation}"`)
|
||||
}
|
||||
catch {
|
||||
console.error(`Error translating "${sourceObj[key]}" to ${toLanguage}. Key: ${key}`)
|
||||
catch (error) {
|
||||
console.error(`❌ Error translating "${sourceObj[key]}" to ${toLanguage}. Key: ${key}`, error.message)
|
||||
skippedKeys.push(`${key}: ${sourceObj[key]} (Error: ${error.message})`)
|
||||
|
||||
// Add retry mechanism for network errors
|
||||
if (error.message.includes('network') || error.message.includes('timeout')) {
|
||||
console.log(`🔄 Retrying translation for key: ${key}`)
|
||||
try {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000)) // Wait 1 second
|
||||
const { translation } = await translate(sourceObj[key], null, languageKeyMap[toLanguage])
|
||||
targetObject[key] = translation
|
||||
translatedKeys.push(`${key}: ${translation}`)
|
||||
console.log(`✅ Retry successful: "${translation}"`)
|
||||
}
|
||||
catch (retryError) {
|
||||
console.error(`❌ Retry failed for key ${key}:`, retryError.message)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (typeof sourceObj[key] === 'object') {
|
||||
targetObject[key] = targetObject[key] || {}
|
||||
await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage)
|
||||
const result = await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage)
|
||||
skippedKeys.push(...result.skipped)
|
||||
translatedKeys.push(...result.translated)
|
||||
}
|
||||
}))
|
||||
|
||||
return { skipped: skippedKeys, translated: translatedKeys }
|
||||
}
|
||||
async function autoGenTrans(fileName, toGenLanguage) {
|
||||
const fullKeyFilePath = path.join(__dirname, i18nFolder, targetLanguage, `${fileName}.ts`)
|
||||
const toGenLanguageFilePath = path.join(__dirname, i18nFolder, toGenLanguage, `${fileName}.ts`)
|
||||
// eslint-disable-next-line sonarjs/code-eval
|
||||
const fullKeyContent = eval(transpile(fs.readFileSync(fullKeyFilePath, 'utf8')))
|
||||
// if toGenLanguageFilePath is not exist, create it
|
||||
if (!fs.existsSync(toGenLanguageFilePath)) {
|
||||
fs.writeFileSync(toGenLanguageFilePath, `const translation = {
|
||||
async function autoGenTrans(fileName, toGenLanguage, isDryRun = false) {
|
||||
const fullKeyFilePath = path.resolve(__dirname, i18nFolder, targetLanguage, `${fileName}.ts`)
|
||||
const toGenLanguageFilePath = path.resolve(__dirname, i18nFolder, toGenLanguage, `${fileName}.ts`)
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(fullKeyFilePath, 'utf8')
|
||||
|
||||
// Create a safer module environment for vm
|
||||
const moduleExports = {}
|
||||
const context = {
|
||||
exports: moduleExports,
|
||||
module: { exports: moduleExports },
|
||||
require,
|
||||
console,
|
||||
__filename: fullKeyFilePath,
|
||||
__dirname: path.dirname(fullKeyFilePath),
|
||||
}
|
||||
|
||||
// Use vm.runInNewContext instead of eval for better security
|
||||
vm.runInNewContext(transpile(content), context)
|
||||
|
||||
const fullKeyContent = moduleExports.default || moduleExports
|
||||
|
||||
if (!fullKeyContent || typeof fullKeyContent !== 'object')
|
||||
throw new Error(`Failed to extract translation object from ${fullKeyFilePath}`)
|
||||
|
||||
// if toGenLanguageFilePath is not exist, create it
|
||||
if (!fs.existsSync(toGenLanguageFilePath)) {
|
||||
fs.writeFileSync(toGenLanguageFilePath, `const translation = {
|
||||
}
|
||||
|
||||
export default translation
|
||||
`)
|
||||
}
|
||||
// To keep object format and format it for magicast to work: const translation = { ... } => export default {...}
|
||||
const readContent = await loadFile(toGenLanguageFilePath)
|
||||
const { code: toGenContent } = generateCode(readContent)
|
||||
const mod = await parseModule(`export default ${toGenContent.replace('export default translation', '').replace('const translation = ', '')}`)
|
||||
const toGenOutPut = mod.exports.default
|
||||
}
|
||||
// To keep object format and format it for magicast to work: const translation = { ... } => export default {...}
|
||||
const readContent = await loadFile(toGenLanguageFilePath)
|
||||
const { code: toGenContent } = generateCode(readContent)
|
||||
const mod = await parseModule(`export default ${toGenContent.replace('export default translation', '').replace('const translation = ', '')}`)
|
||||
const toGenOutPut = mod.exports.default
|
||||
|
||||
await translateMissingKeyDeeply(fullKeyContent, toGenOutPut, toGenLanguage)
|
||||
const { code } = generateCode(mod)
|
||||
const res = `const translation =${code.replace('export default', '')}
|
||||
console.log(`\n🌍 Processing ${fileName} for ${toGenLanguage}...`)
|
||||
const result = await translateMissingKeyDeeply(fullKeyContent, toGenOutPut, toGenLanguage)
|
||||
|
||||
// Generate summary report
|
||||
console.log(`\n📊 Translation Summary for ${fileName} -> ${toGenLanguage}:`)
|
||||
console.log(` ✅ Translated: ${result.translated.length} keys`)
|
||||
console.log(` ⏭️ Skipped: ${result.skipped.length} keys`)
|
||||
|
||||
if (result.skipped.length > 0) {
|
||||
console.log(`\n⚠️ Skipped keys in ${fileName} (${toGenLanguage}):`)
|
||||
result.skipped.slice(0, 5).forEach(item => console.log(` - ${item}`))
|
||||
if (result.skipped.length > 5)
|
||||
console.log(` ... and ${result.skipped.length - 5} more`)
|
||||
}
|
||||
|
||||
const { code } = generateCode(mod)
|
||||
const res = `const translation =${code.replace('export default', '')}
|
||||
|
||||
export default translation
|
||||
`.replace(/,\n\n/g, ',\n').replace('};', '}')
|
||||
|
||||
fs.writeFileSync(toGenLanguageFilePath, res)
|
||||
if (!isDryRun) {
|
||||
fs.writeFileSync(toGenLanguageFilePath, res)
|
||||
console.log(`💾 Saved translations to ${toGenLanguageFilePath}`)
|
||||
}
|
||||
else {
|
||||
console.log(`🔍 [DRY RUN] Would save translations to ${toGenLanguageFilePath}`)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Error processing file ${fullKeyFilePath}:`, error.message)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// Add command line argument support
|
||||
const isDryRun = process.argv.includes('--dry-run')
|
||||
const targetFile = process.argv.find(arg => arg.startsWith('--file='))?.split('=')[1]
|
||||
const targetLang = process.argv.find(arg => arg.startsWith('--lang='))?.split('=')[1]
|
||||
|
||||
// Rate limiting helper
|
||||
function delay(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// const fileName = 'workflow'
|
||||
// Promise.all(Object.keys(languageKeyMap).map(async (toLanguage) => {
|
||||
// await autoGenTrans(fileName, toLanguage)
|
||||
// }))
|
||||
console.log('🚀 Starting auto-gen-i18n script...')
|
||||
console.log(`📋 Mode: ${isDryRun ? 'DRY RUN (no files will be modified)' : 'LIVE MODE'}`)
|
||||
|
||||
const files = fs
|
||||
.readdirSync(path.join(__dirname, i18nFolder, targetLanguage))
|
||||
.map(file => file.replace(/\.ts/, ''))
|
||||
.readdirSync(path.resolve(__dirname, i18nFolder, targetLanguage))
|
||||
.filter(file => /\.ts$/.test(file)) // Only process .ts files
|
||||
.map(file => file.replace(/\.ts$/, ''))
|
||||
.filter(f => f !== 'app-debug') // ast parse error in app-debug
|
||||
|
||||
await Promise.all(files.map(async (file) => {
|
||||
await Promise.all(Object.keys(languageKeyMap).map(async (language) => {
|
||||
// Filter by target file if specified
|
||||
const filesToProcess = targetFile ? files.filter(f => f === targetFile) : files
|
||||
const languagesToProcess = targetLang ? [targetLang] : Object.keys(languageKeyMap)
|
||||
|
||||
console.log(`📁 Files to process: ${filesToProcess.join(', ')}`)
|
||||
console.log(`🌍 Languages to process: ${languagesToProcess.join(', ')}`)
|
||||
|
||||
let totalTranslated = 0
|
||||
let totalSkipped = 0
|
||||
let totalErrors = 0
|
||||
|
||||
// Process files sequentially to avoid API rate limits
|
||||
for (const file of filesToProcess) {
|
||||
console.log(`\n📄 Processing file: ${file}`)
|
||||
|
||||
// Process languages with rate limiting
|
||||
for (const language of languagesToProcess) {
|
||||
try {
|
||||
await autoGenTrans(file, language)
|
||||
const result = await autoGenTrans(file, language, isDryRun)
|
||||
totalTranslated += result.translated.length
|
||||
totalSkipped += result.skipped.length
|
||||
|
||||
// Rate limiting: wait 500ms between language processing
|
||||
await delay(500)
|
||||
}
|
||||
catch (e) {
|
||||
console.error(`Error translating ${file} to ${language}`, e)
|
||||
console.error(`❌ Error translating ${file} to ${language}:`, e.message)
|
||||
totalErrors++
|
||||
}
|
||||
}))
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// Final summary
|
||||
console.log('\n🎉 Auto-translation completed!')
|
||||
console.log('📊 Final Summary:')
|
||||
console.log(` ✅ Total keys translated: ${totalTranslated}`)
|
||||
console.log(` ⏭️ Total keys skipped: ${totalSkipped}`)
|
||||
console.log(` ❌ Total errors: ${totalErrors}`)
|
||||
|
||||
if (isDryRun)
|
||||
console.log('\n💡 This was a dry run. To actually translate, run without --dry-run flag.')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const vm = require('node:vm')
|
||||
const transpile = require('typescript').transpile
|
||||
|
||||
const targetLanguage = 'en-US'
|
||||
const data = require('./languages.json')
|
||||
const languages = data.languages.filter(language => language.supported).map(language => language.value)
|
||||
|
||||
async function getKeysFromLanuage(language) {
|
||||
async function getKeysFromLanguage(language) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const folderPath = path.join(__dirname, '../i18n', language)
|
||||
let allKeys = []
|
||||
const folderPath = path.resolve(__dirname, '../i18n', language)
|
||||
const allKeys = []
|
||||
fs.readdir(folderPath, (err, files) => {
|
||||
if (err) {
|
||||
console.error('Error reading folder:', err)
|
||||
@@ -17,37 +18,61 @@ async function getKeysFromLanuage(language) {
|
||||
return
|
||||
}
|
||||
|
||||
files.forEach((file) => {
|
||||
// Filter only .ts and .js files
|
||||
const translationFiles = files.filter(file => /\.(ts|js)$/.test(file))
|
||||
|
||||
translationFiles.forEach((file) => {
|
||||
const filePath = path.join(folderPath, file)
|
||||
const fileName = file.replace(/\.[^/.]+$/, '') // Remove file extension
|
||||
const camelCaseFileName = fileName.replace(/[-_](.)/g, (_, c) =>
|
||||
c.toUpperCase(),
|
||||
) // Convert to camel case
|
||||
// console.log(camelCaseFileName)
|
||||
const content = fs.readFileSync(filePath, 'utf8')
|
||||
// eslint-disable-next-line sonarjs/code-eval
|
||||
const translationObj = eval(transpile(content))
|
||||
// console.log(translation)
|
||||
if(!translationObj || typeof translationObj !== 'object') {
|
||||
console.error(`Error parsing file: ${filePath}`)
|
||||
reject(new Error(`Error parsing file: ${filePath}`))
|
||||
return
|
||||
}
|
||||
const keys = Object.keys(translationObj)
|
||||
const nestedKeys = []
|
||||
const iterateKeys = (obj, prefix = '') => {
|
||||
for (const key in obj) {
|
||||
const nestedKey = prefix ? `${prefix}.${key}` : key
|
||||
nestedKeys.push(nestedKey)
|
||||
if (typeof obj[key] === 'object')
|
||||
iterateKeys(obj[key], nestedKey)
|
||||
}
|
||||
}
|
||||
iterateKeys(translationObj)
|
||||
|
||||
allKeys = [...keys, ...nestedKeys].map(
|
||||
key => `${camelCaseFileName}.${key}`,
|
||||
)
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8')
|
||||
|
||||
// Create a safer module environment for vm
|
||||
const moduleExports = {}
|
||||
const context = {
|
||||
exports: moduleExports,
|
||||
module: { exports: moduleExports },
|
||||
require,
|
||||
console,
|
||||
__filename: filePath,
|
||||
__dirname: folderPath,
|
||||
}
|
||||
|
||||
// Use vm.runInNewContext instead of eval for better security
|
||||
vm.runInNewContext(transpile(content), context)
|
||||
|
||||
// Extract the translation object
|
||||
const translationObj = moduleExports.default || moduleExports
|
||||
|
||||
if(!translationObj || typeof translationObj !== 'object') {
|
||||
console.error(`Error parsing file: ${filePath}`)
|
||||
reject(new Error(`Error parsing file: ${filePath}`))
|
||||
return
|
||||
}
|
||||
|
||||
const nestedKeys = []
|
||||
const iterateKeys = (obj, prefix = '') => {
|
||||
for (const key in obj) {
|
||||
const nestedKey = prefix ? `${prefix}.${key}` : key
|
||||
nestedKeys.push(nestedKey)
|
||||
if (typeof obj[key] === 'object' && obj[key] !== null)
|
||||
iterateKeys(obj[key], nestedKey)
|
||||
}
|
||||
}
|
||||
iterateKeys(translationObj)
|
||||
|
||||
// Fixed: accumulate keys instead of overwriting
|
||||
const fileKeys = nestedKeys.map(key => `${camelCaseFileName}.${key}`)
|
||||
allKeys.push(...fileKeys)
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Error processing file ${filePath}:`, error.message)
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
resolve(allKeys)
|
||||
})
|
||||
@@ -56,8 +81,8 @@ async function getKeysFromLanuage(language) {
|
||||
|
||||
async function main() {
|
||||
const compareKeysCount = async () => {
|
||||
const targetKeys = await getKeysFromLanuage(targetLanguage)
|
||||
const languagesKeys = await Promise.all(languages.map(language => getKeysFromLanuage(language)))
|
||||
const targetKeys = await getKeysFromLanguage(targetLanguage)
|
||||
const languagesKeys = await Promise.all(languages.map(language => getKeysFromLanguage(language)))
|
||||
|
||||
const keysCount = languagesKeys.map(keys => keys.length)
|
||||
const targetKeysCount = targetKeys.length
|
||||
|
||||
Reference in New Issue
Block a user