Bring languages into the tree (#1625)

* translations: bring languages into tree

* Update translation phrases changed since 2021

* Update packaging script to include all translations

* Update languages.cfg

* Add Latin American Spanish translations

This is a copy of spanish for now.

* Ignore "en" when looking for translation folders

English is the default and doesn't use a subfolder.

* Only add each translation folder once

Korean "ko" is in there twice.

* Compare language coverage to english

All phrases are compared to the english baseline files and any differences
are reported. The differences are pushed to a Github Project as well for
an easier overview.

Thank you to @nosoop for sharing the Python SMC parser!

* Add link to README

---------

Co-authored-by: Peace-Maker <[email protected]>
This commit is contained in:
Kyle Sanderson
2023-03-29 16:23:05 +02:00
committed by GitHub
co-authored by Peace-Maker
parent d8fd60b562
commit 48150e0c7a
696 changed files with 47789 additions and 22 deletions
@@ -0,0 +1,283 @@
#!/usr/bin/python3
# Copyright (c) 2023 Peace-Maker
from collections import defaultdict
from dataclasses import dataclass
import os
import pathlib
import re
from smc_parser import smc_string_to_dict
from typing import Dict, List, Union
from github_gql import GithubGQL
@dataclass
class Translation:
langid: str
translation: str
param_count: int
@dataclass
class Phrase:
key: str
format: Union[Translation, None]
translations: List[Translation]
@dataclass
class PhraseFile:
filename: str
phrases: List[Phrase]
error: Union[str, None] = None
@dataclass
class Language:
langid: str
name: str
files: List[PhraseFile]
@dataclass
class Report:
langid: str
filename: str
file_warning: str = ''
phrase_key: str = ''
phrase_warning: str = ''
def parse_translations(path: str):
param_regex = re.compile(r'\{[0-9]+\}', re.MULTILINE)
units = []
for file in pathlib.Path(path).glob('*.txt'):
if not file.is_file():
continue
try:
phrases = smc_string_to_dict(file.read_text('utf-8'))
except Exception as ex:
print(f'Error parsing {file.name}: {ex}')
units.append(PhraseFile(file.name, [], str(ex)))
continue
if 'Phrases' not in phrases:
print(f'File {file.name} does not start with a "Phrases" section')
continue
parsed_phrases = []
for phrase in phrases['Phrases']:
for phrase_ident, raw_translations in phrase.items():
translations = []
format_special = None
for child_langid, translation in raw_translations.items():
if child_langid == '#format':
format_special = Translation(
child_langid, translation,
translation.count(',') + 1)
else:
translations.append(
Translation(child_langid, translation,
len(param_regex.findall(translation))))
parsed_phrases.append(
Phrase(phrase_ident, format_special, translations))
units.append(PhraseFile(file.name, parsed_phrases))
return units
# Parse the languages.cfg file to know which languages could be available
print('Parsing languages.cfg...')
available_languages: Dict[str, Language] = {}
languages_cfg = smc_string_to_dict(
pathlib.Path('../../configs/languages.cfg').read_text('utf-8'))
for langid, lang in languages_cfg['Languages'][0].items():
available_languages[langid] = Language(langid, lang, [])
print(f'Available languages: {len(available_languages)}')
# Parse the english translation, since it doesn't use a subdirectory and is the baseline for all other translations
available_languages['en'].files = parse_translations('../../translations')
# Parse the other translations
for langid, lang in available_languages.items():
if langid == 'en':
continue
lang.files = parse_translations(f'../../translations/{langid}')
reports: Dict[str, Dict[str,
List[Report]]] = defaultdict(lambda: defaultdict(list))
# Compare the english translation with the other translations
english = available_languages['en']
for langid, lang in available_languages.items():
if langid == 'en':
continue
# See if this language has anything that English doesn't
for file in lang.files:
english_file = next(
(x for x in english.files if x.filename == file.filename), None)
if english_file is None:
reports[langid][file.filename].append(
Report(langid,
file.filename,
file_warning='File doesn\'t exist in English'))
continue
if not file.phrases:
reports[langid][file.filename].append(
Report(langid, file.filename, file_warning='File is empty'))
continue
for phrase in file.phrases:
if phrase.format:
reports[langid][file.filename].append(
Report(langid,
file.filename,
phrase_key=phrase.key,
phrase_warning='Includes a "#format" key'))
english_phrase = next(
(x for x in english_file.phrases if x.key == phrase.key), None)
if english_phrase is None:
# look for this phrase in a different english file
warning = 'Phrase doesn\'t exist in English'
for other_file in english.files:
other_phrase = next(
(x for x in other_file.phrases if x.key == phrase.key),
None)
if other_phrase:
warning = f'Phrase exists in a different file in English: {other_file.filename}'
break
reports[langid][file.filename].append(
Report(langid,
file.filename,
phrase_key=phrase.key,
phrase_warning=warning))
continue
translation_found = False
for translation in phrase.translations:
if translation.langid == langid:
translation_found = True
else:
reports[langid][file.filename].append(
Report(
langid,
file.filename,
phrase_key=phrase.key,
phrase_warning=
f'Includes a translation for language "{translation.langid}"'
))
if english_phrase.format and translation.param_count != english_phrase.format.param_count:
reports[langid][file.filename].append(
Report(
langid,
file.filename,
phrase_key=phrase.key,
phrase_warning=
f'Has {translation.param_count} format parameters, but English has {english_phrase.format.param_count}'
))
if not translation_found:
reports[langid][file.filename].append(
Report(langid,
file.filename,
phrase_key=phrase.key,
phrase_warning=
'Phrase available, but translation missing'))
# See if this language is missing anything that English has
for file in english.files:
lang_file = next(
(x for x in lang.files if x.filename == file.filename), None)
if lang_file is None:
reports[langid][file.filename].append(
Report(langid, file.filename, file_warning='File missing'))
continue
# The file doesn't contain any phrases. We reported that already, so don't spam every single missing phrase
if not lang_file.phrases:
continue
for phrase in file.phrases:
lang_phrase = next(
(x for x in lang_file.phrases if x.key == phrase.key), None)
if lang_phrase is None:
reports[langid][file.filename].append(
Report(langid,
file.filename,
phrase_key=phrase.key,
phrase_warning='Phrase missing'))
if langid not in reports:
print(f'No issues found for {lang.name} ({langid})')
else:
print(
f'Found {len(reports[langid])} issues for {lang.name} ({langid})')
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
if not GITHUB_TOKEN:
raise Exception('GITHUB_TOKEN environment variable not set')
ORGANIZATION = os.environ.get('ORGANIZATION')
if not ORGANIZATION:
raise Exception('ORGANIZATION environment variable not set')
PROJECT_NUMBER = os.environ.get('PROJECT_NUMBER')
if not PROJECT_NUMBER:
raise Exception('PROJECT_NUMBER environment variable not set')
# Get the project and its draft issues
print('Getting project and draft issues...')
githubgql = GithubGQL(GITHUB_TOKEN)
project = githubgql.get_project(ORGANIZATION, int(PROJECT_NUMBER))
project_id = project['id']
field_ids = project['fields']['nodes']
status_field = [field for field in field_ids if field['name'] == 'Status']
assert len(status_field) == 1, 'Status field not found'
status_field_id = status_field[0]['id']
status_field_option_ids = {
option['name']: option['id']
for option in status_field[0]['options']
}
if 'Incomplete' not in status_field_option_ids:
raise Exception('Incomplete status field option not found')
if 'Complete' not in status_field_option_ids:
raise Exception('Complete status field option not found')
draft_issues = project['items']['nodes']
# Generate the report markdown for the project draft issues
for langid, lang in available_languages.items():
markdown = ''
status = ''
if langid in reports:
print(f'Generating report for {lang.name} ({langid})...')
status = 'Incomplete'
for filename, problems in reports[langid].items():
markdown += f'## [{filename}](https://github.com/alliedmodders/sourcemod/blob/master/translations/{langid}/{filename})\n'
added_phrase_warning = False
for report in problems:
if report.file_warning:
markdown += f'**{report.file_warning}**\n'
print(f' {report.file_warning} ({report.filename})')
if report.phrase_warning:
if not added_phrase_warning:
markdown += '| Phrase | Issue |\n| ------- | --------- |\n'
added_phrase_warning = True
markdown += f'| `{report.phrase_key}` | {report.phrase_warning} |\n'
print(
f' {report.filename}: "{report.phrase_key}" -> {report.phrase_warning}'
)
markdown += '\n'
else:
status = 'Complete'
markdown = 'No issues found'
print(f'Updating draft issue for {lang.name} ({langid})...')
issue = next(
(x for x in draft_issues if x['content']['title'] == lang.name), None)
if issue is None:
issue = githubgql.add_draft_issue(project_id, lang.name, markdown)
else:
githubgql.update_draft_issue(issue['content']['id'], lang.name,
markdown)
githubgql.update_item_field_value_option(project_id, issue['id'],
status_field_id,
status_field_option_ids[status])
+156
View File
@@ -0,0 +1,156 @@
# Copyright (c) 2023 Peace-Maker
from gql import gql, Client
from gql.transport.aiohttp import AIOHTTPTransport
class GithubGQL:
def __init__(self, token):
transport = AIOHTTPTransport(
url="https://api.github.com/graphql",
headers={"Authorization": f"Bearer {token}"})
self.client = Client(transport=transport)
def get_project(self, orga, project_number):
query = gql("""
query getProjectId($login: String!, $projectNumber: Int!){
organization(login: $login) {
projectV2(number: $projectNumber) {
id
fields(first: 100) {
nodes {
... on ProjectV2Field {
id
name
}
... on ProjectV2SingleSelectField {
id
name
options {
id
name
}
}
... on ProjectV2SingleSelectField {
id
name
options {
id
name
}
}
}
}
items(first: 100) {
nodes {
id
fieldValues(first: 100) {
nodes {
... on ProjectV2ItemFieldTextValue {
text
field {
... on ProjectV2FieldCommon {
name
}
}
}
... on ProjectV2ItemFieldSingleSelectValue {
name
field {
... on ProjectV2FieldCommon {
name
}
}
}
}
}
content {
... on DraftIssue {
id
title
}
}
}
}
}
}
}
""")
variables = {"login": orga, "projectNumber": project_number}
result = self.client.execute(query, variable_values=variables)
# TODO: Handle pagination
return result["organization"]["projectV2"]
def add_draft_issue(self, project_id, title, body):
query = gql("""
mutation addDraftIssue($projectId: ID!, $title: String!, $body: String!){
addProjectV2DraftIssue(
input: {
projectId: $projectId,
title: $title,
body: $body
}
) {
projectItem {
id
content {
... on DraftIssue {
id
}
}
}
}
}
""")
variables = {"projectId": project_id, "title": title, "body": body}
result = self.client.execute(query, variable_values=variables)
return result["addProjectV2DraftIssue"]["projectItem"]
def update_draft_issue(self, issue_id, title, body):
query = gql("""
mutation updateDraftIssue($issueId: ID!, $title: String!, $body: String!){
updateProjectV2DraftIssue(
input: {
draftIssueId: $issueId,
title: $title,
body: $body
}
) {
draftIssue {
id
}
}
}
""")
variables = {"issueId": issue_id, "title": title, "body": body}
result = self.client.execute(query, variable_values=variables)
return result["updateProjectV2DraftIssue"]["draftIssue"]["id"]
def update_item_field_value_option(self, project_id, item_id, field_id,
option_id):
query = gql("""
mutation updateDraftIssueStatus($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!){
updateProjectV2ItemFieldValue(
input: {
projectId: $projectId,
itemId: $itemId,
fieldId: $fieldId,
value: {
singleSelectOptionId: $optionId
}
}
) {
projectV2Item {
id
}
}
}
""")
variables = {
"projectId": project_id,
"itemId": item_id,
"fieldId": field_id,
"optionId": option_id
}
result = self.client.execute(query, variable_values=variables)
return result["updateProjectV2ItemFieldValue"]["projectV2Item"]["id"]
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/python3
# BSD Zero Clause License
#
# Copyright (C) 2023 by nosoop
#
# Permission to use, copy, modify, and/or distribute this software for any purpose with or
# without fee is hereby granted.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
# SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
# OR PERFORMANCE OF THIS SOFTWARE.
# https://gist.github.com/nosoop/8c6ccaec11b1d33340bec8dbc8096658
import collections
import enum
import itertools
class SMCOperation(enum.Enum):
STRING = 1
SUBSECTION_START = 2
SUBSECTION_END = 3
COMMENT = 4
COMMENT_MULTILINE = 5
KEYVALUE = 6
# https://stackoverflow.com/a/70762559
def takewhile_inclusive(predicate, it):
for x in it:
if predicate(x):
yield x
else:
yield x
break
def _is_whitespace(ch):
return ch in (' ', '\t', '\n', '\r')
def _smc_stream_skip_whitespace(stream):
# consumes whitespace and returns the first non-whitespace character if any, or None if EOS
values = tuple(takewhile_inclusive(_is_whitespace, stream))
if not values:
return None
*ws, last = values
if not ws and not _is_whitespace(last):
return last
return last if ws and not _is_whitespace(last) else None
def _smc_stream_extract_multiline_comment(stream):
while True:
yield from itertools.takewhile(lambda ch: ch != '*', stream)
ch = next(stream, None)
if ch == '/':
return
yield '*'
yield ch
_escape_mapping = str.maketrans({
'"': '"',
'n': '\n',
'r': '\r',
't': '\t',
'\\': '\\',
})
def _smc_stream_extract_string(stream):
for ch in stream:
if ch == "\\":
ch = next(stream).translate(_escape_mapping)
elif ch == '"':
return
yield ch
def parse_smc_string(data):
stream = iter(data)
while True:
ch = _smc_stream_skip_whitespace(stream)
if ch is None:
return
elif ch == '"':
# consume until the next quote, then determine if:
# - the string marks the subsection name '{'
# - we have another string to consume, making this a key / value pair
key = ''.join(_smc_stream_extract_string(stream))
ch = _smc_stream_skip_whitespace(stream)
if ch == '{':
yield SMCOperation.SUBSECTION_START, key
elif ch == '"':
value = ''.join(_smc_stream_extract_string(stream))
yield SMCOperation.KEYVALUE, key, value
else:
raise ValueError(
f"Unexpected character {ch.encode('ascii', 'backslashreplace')} after end of string"
)
elif ch == '}':
yield SMCOperation.SUBSECTION_END, None
elif ch == '/':
ch = next(stream)
if ch == '/':
# single line comment: consume until the end of the line
value = ''.join(
itertools.takewhile(lambda ch: ch != '\n', stream))
yield SMCOperation.COMMENT, value
elif ch == '*':
# multi line comment: consume until the sequence '*/' is reached
value = ''.join(_smc_stream_extract_multiline_comment(stream))
yield SMCOperation.COMMENT_MULTILINE, value
else:
raise ValueError(
f"Unexpected character {ch.encode('ascii', 'backslashreplace')} at start of comment"
)
else:
raise ValueError(
f"Unexpected character {ch.encode('ascii', 'backslashreplace')}"
)
class MultiKeyDict(collections.defaultdict):
# a dict that supports supports one-to-many mappings
# init by passing keys pointing to a list of values
def __init__(self, *args, **kwargs):
super().__init__(list, *args, **kwargs)
# yields a key, value pair for every array item associated with a key
def items(self):
yield from ((k, iv) for k, v in super().items() for iv in v)
def smc_string_to_dict(data):
# returns a multidict instance
root_node = MultiKeyDict()
contexts = [root_node]
for event, *info in parse_smc_string(data):
if event == SMCOperation.SUBSECTION_START:
key, *_ = info
subkey = MultiKeyDict()
contexts[-1][key].append(subkey)
contexts.append(subkey)
elif event == SMCOperation.SUBSECTION_END:
contexts.pop()
elif event == SMCOperation.KEYVALUE:
key, value = info
contexts[-1][key].append(value)
return root_node
def main():
SMC_STRING = """
"thing"
{
// this is a comment node
"key" "value"
"subthing"
{
// and another
"subthing key" "subthing value"
"subthing key" "duplicate key value"
}
"subthing"
{
"duplicate subthing" "yes"
}
/**
* this is a multiline comment node
*/
"another key" "another value"
}
"""
# sections = []
# for event, *data in parse_smc_string(SMC_STRING):
# print(event, data, tuple(sections))
# if event == SMCOperation.SUBSECTION_START:
# section, *_ = data
# sections.append(section)
# elif event == SMCOperation.SUBSECTION_END:
# sections.pop()
# assert(not sections)
import json
import pathlib
# print(json.dumps(smc_string_to_dict(SMC_STRING), indent=4))
for f in pathlib.Path('translations').rglob('*.txt'):
print(f)
print(json.dumps(smc_string_to_dict(f.read_text('utf8')), indent = 4))
if __name__ == "__main__":
main()