Skip to content

Commit 8d72e8b

Browse files
michaeltlombardimichaeltlombardi
committed
(MAINT) Reimplement i18n tests
Prior to this change, the i18n tests used simple string matching to search for translation keys in the `en-us.toml` file and the project Rust code. This change defines a new `DscProjectRustTranslationInfo` class and `Get-TranslationKey` function in the `build.helpers` module. This change reimplements the i18n tests to use the new class and function for retrieving and analyzing translation strings for Rust projects, building in support for both TOML and YAML translation files. This is required to support the translation strings for JSON Schema annotations, which are lengthy. It enables us to keep the translations for messages in the code in the `en-us.toml` file and all schema-related translation strings in separate YAML files, where the hierarchy and multi-line strings fit more naturally. This change ensures that we can continue to accurately test our translation strings, checking: - That each project with a `locales` folder defines and uses at least one translation string. - That no translation keys are defined more than once across all translation files. - That every defined translation key is used in the Rust code. - That every translation key the Rust code references is defined in a translation file for `en-us`.
1 parent 8c24d70 commit 8d72e8b

File tree

2 files changed

+247
-39
lines changed

2 files changed

+247
-39
lines changed

‎build.helpers.psm1‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,16 @@ function Install-PowerShellTestPrerequisite {
644644
Write-Verbose "Installing module 'Pester'"
645645
Install-PSResource Pester -WarningAction Ignore -Repository $repository -TrustRepository
646646
}
647+
648+
if (-not (Get-Module -ListAvailable -Name YaYaml)) {
649+
Write-Verbose "Installing module 'YaYaml'"
650+
Install-PSResource YaYaml -WarningAction Ignore -Repository $repository -TrustRepository
651+
}
652+
653+
if (-not (Get-Module -ListAvailable -Name PSToml)) {
654+
Write-Verbose "Installing module 'PSToml'"
655+
Install-PSResource PSToml -WarningAction Ignore -Repository $repository -TrustRepository
656+
}
647657
}
648658
}
649659

@@ -1539,6 +1549,186 @@ function Export-RustDocs {
15391549
}
15401550
#endregion Documenting project functions
15411551

1552+
#region Rust i18n type definitions and functions
1553+
class DscProjectRustTranslationInfo {
1554+
[System.Collections.Generic.Dictionary[string, string]] $Table
1555+
[System.Collections.Generic.HashSet[string]] $DuplicateTranslations
1556+
[System.Collections.Generic.HashSet[string]] $MissingTranslations
1557+
[System.Collections.Generic.HashSet[string]] $UnusedTranslations
1558+
[System.Collections.Generic.HashSet[string]] $UsedTranslations
1559+
1560+
[string] GetTranslationString([string]$translationKey) {
1561+
if ($null -eq $this.Table) {
1562+
$this.Initialize()
1563+
}
1564+
return $this.Table[$translationKey]
1565+
}
1566+
1567+
[void] Initialize() {
1568+
if ($null -ne $this.Table) {
1569+
return
1570+
}
1571+
$this.Table = [System.Collections.Generic.Dictionary[string, string]]::new()
1572+
}
1573+
1574+
[void] ProcessData(
1575+
[System.Collections.Specialized.OrderedDictionary]$data,
1576+
[string]$prefix,
1577+
[int]$version
1578+
) {
1579+
foreach ($key in $data.Keys) {
1580+
$keyData = $data[$key]
1581+
$workingKey = @($prefix, $key) -join '.'
1582+
1583+
if ($keyData -is [System.Collections.Specialized.OrderedDictionary]) {
1584+
$this.ProcessData($keyData, $workingKey, $version)
1585+
} elseif ($keyData -is [string]) {
1586+
if ($key -match '^[a-zA-Z]+-[a-zA-Z]+' -and $version -eq 2) {
1587+
if ($key -eq 'en-us') {
1588+
if (-not [string]::IsNullOrEmpty($this.Table[$prefix])) {
1589+
$this.DuplicateTranslations.Add($prefix)
1590+
}
1591+
$this.Table[$prefix] = $keyData
1592+
}
1593+
continue
1594+
}
1595+
1596+
if (-not [string]::IsNullOrEmpty($this.Table[$workingKey])) {
1597+
$this.DuplicateTranslations.Add($workingKey)
1598+
}
1599+
$this.Table[$workingKey] = $keyData
1600+
}
1601+
}
1602+
}
1603+
1604+
[void] LoadData([System.Collections.Specialized.OrderedDictionary]$data) {
1605+
$this.Initialize()
1606+
1607+
[ValidateRange(1,2)][int]$version = 1
1608+
if ($data['_version'] -is [int]) {
1609+
$version = $data['_version']
1610+
}
1611+
1612+
foreach ($key in $data.Keys) {
1613+
if ($key -eq '_version') {
1614+
continue
1615+
}
1616+
$keyData = $data[$key]
1617+
1618+
if ($keyData -is [string]) {
1619+
if (-not [string]::IsNullOrEmpty($this.Table[$key])) {
1620+
$this.DuplicateTranslations.Add($key)
1621+
}
1622+
$this.Table[$key] = $keyData
1623+
} elseif ($keyData -is [System.Collections.Specialized.OrderedDictionary]) {
1624+
$this.ProcessData($keyData, $key, $version)
1625+
}
1626+
}
1627+
}
1628+
1629+
[void] LoadFile([System.IO.FileInfo]$file) {
1630+
$content = Get-Content -Path $file.FullName -Raw
1631+
$extension = $file.Extension.Substring(1)
1632+
1633+
$fileData = switch ($extension) {
1634+
'toml' {
1635+
$content | PSToml\ConvertFrom-Toml
1636+
break
1637+
}
1638+
'yaml' {
1639+
$content | YaYaml\ConvertFrom-Yaml
1640+
break
1641+
}
1642+
default {
1643+
throw "Unsupported translation file format '$extension' - must be TOML or YAML."
1644+
}
1645+
}
1646+
1647+
$this.LoadData($fileData)
1648+
}
1649+
1650+
[void] CheckTranslations([System.IO.DirectoryInfo]$projectFolder) {
1651+
$this.UsedTranslations = Get-TranslationKey -ProjectDirectory $projectFolder
1652+
$definedKeys = [System.Collections.Generic.HashSet[string]]$this.Table.Keys
1653+
$this.MissingTranslations = $this.UsedTranslations.Where({ $_ -notin $definedKeys })
1654+
$this.UnusedTranslations = $definedKeys.Where({ $_ -notin $this.UsedTranslations })
1655+
}
1656+
1657+
DscProjectRustTranslationInfo([System.Collections.Specialized.OrderedDictionary]$data) {
1658+
$this.LoadData($data)
1659+
}
1660+
1661+
DscProjectRustTranslationInfo([System.IO.FileInfo]$file) {
1662+
$this.LoadFile($file)
1663+
}
1664+
DscProjectRustTranslationInfo([System.IO.DirectoryInfo]$directory) {
1665+
$localesFolder = if ($directory.BaseName -eq 'locales') {
1666+
$directory
1667+
} else {
1668+
Join-Path -Path $directory -ChildPath 'locales'
1669+
}
1670+
$projectFolder = Split-Path -Path $localesFolder -Parent
1671+
1672+
if (-not (Test-Path $localesFolder)) {
1673+
throw "Unable to find valid locales folder from in directory '$directory'"
1674+
}
1675+
1676+
$tomlFile = Join-Path -Path $localesFolder -ChildPath 'en-us.toml'
1677+
if (Test-Path -Path $tomlFile) {
1678+
$this.LoadFile((Get-Item -Path $tomlFile))
1679+
}
1680+
$yamlFiles = Get-ChildItem -Path $localesFolder | Where-Object Extension -match 'ya?ml'
1681+
foreach ($yamlFile in $yamlFiles) {
1682+
$this.LoadFile($yamlFile)
1683+
}
1684+
1685+
$this.CheckTranslations($projectFolder)
1686+
}
1687+
}
1688+
1689+
function Get-TranslationKey {
1690+
[cmdletbinding()]
1691+
[OutputType([string[]])]
1692+
param(
1693+
[Parameter(Mandatory)]
1694+
[string]$ProjectDirectory
1695+
)
1696+
1697+
begin {
1698+
$patterns = @{
1699+
t = '(?s)\bt\!\(\s*"(?<key>.*?)".*?\)'
1700+
panic_t = '(?s)\bpanic_t\!\(\s*"(?<key>.*?)".*?\)'
1701+
assert_t = '(?s)\bassert_t\!\(\s*.*?,\s*"(?<key>.*?)".*?\)'
1702+
}
1703+
[string[]]$keys = @()
1704+
}
1705+
1706+
process {
1707+
if (-not (Test-Path $ProjectDirectory -PathType Container)) {
1708+
throw "Invalid target, '$ProjectDirectory' isn't a directory or doesn't exist."
1709+
}
1710+
1711+
Get-ChildItem -Recurse -Path $ProjectDirectory -Include *.rs -File | ForEach-Object {
1712+
$file = $_
1713+
$content = Get-Content -Path $file -Raw
1714+
foreach ($pattern in $patterns.keys) {
1715+
($content | Select-String -Pattern $patterns[$pattern] -AllMatches).Matches | ForEach-Object {
1716+
if ($null -ne $_) {
1717+
$key = $_.Groups['key'].Value
1718+
$keys += $key
1719+
}
1720+
}
1721+
}
1722+
}
1723+
}
1724+
1725+
end {
1726+
$keys
1727+
}
1728+
}
1729+
#endregion Rust i18n type definitions and functions
1730+
1731+
15421732
#region Test project functions
15431733
function Test-RustProject {
15441734
[CmdletBinding()]

‎dsc/tests/dsc_i18n.tests.ps1‎

Lines changed: 57 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,76 @@
11
# Copyright (c) Microsoft Corporation.
22
# Licensed under the MIT License.
33

4+
using module ../../build.helpers.psm1
5+
46
BeforeDiscovery {
5-
$tomls = Get-ChildItem $PSScriptRoot/../../en-us.toml -Recurse -File
7+
# Limit the folders to recursively search for rust i18n translation strings
8+
$rootFolders = @(
9+
'adapters'
10+
'dsc'
11+
'extensions'
12+
'grammars'
13+
'lib'
14+
'pal'
15+
'resources'
16+
'tools'
17+
'y2j'
18+
)
19+
$localeFolders = $rootFolders | ForEach-Object -Process {
20+
Get-ChildItem $PSScriptRoot/../../$_/locales -Recurse -Directory
21+
}
22+
623
$projects = @()
7-
$tomls | ForEach-Object {
8-
$projectName = (Split-Path $_ -Parent | Split-Path -Parent)
9-
$projects += @{ project = $projectName; toml = $_ }
24+
$localeFolders | ForEach-Object -Process {
25+
$projects += @{
26+
project = Split-Path $_ -Parent
27+
translationInfo = [DscProjectRustTranslationInfo]::new($_)
28+
}
1029
}
1130
}
1231

1332
Describe 'Internationalization tests' {
14-
It 'Project <project> uses i18n strings from <toml>' -TestCases $projects {
15-
param($project, $toml)
16-
17-
$i18n = [System.Collections.Hashtable]::new([System.StringComparer]::Ordinal)
18-
$prefix = ''
19-
Get-Content -Path $toml | ForEach-Object {
20-
if ($_ -match '\[(?<prefix>.*?)\]') {
21-
$prefix = $Matches['prefix']
22-
}
23-
elseif ($_ -match '^(?<key>\w+)\s?=\s?"(?<value>.*?)"') {
24-
$key = $prefix + '.' + $Matches['key']
25-
$i18n[$key] = 0
33+
Context '<project>' -ForEach $projects {
34+
It 'Uses translation strings' {
35+
$check = @{
36+
Not = $true
37+
BeNullOrEmpty = $true
38+
Because = "'$project' defines at least one translation file"
2639
}
40+
$translationInfo.UsedTranslations | Should @check
2741
}
42+
It 'Does not define any duplicate translation strings' {
43+
$check = @{
44+
BeNullOrEmpty = $true
45+
Because = (@(
46+
"The following translation keys are defined more than once:"
47+
$translationInfo.DuplicateTranslations | ConvertTo-Json -Depth 2
48+
) -join ' ')
49+
}
2850

29-
$patterns = @{
30-
t = '(?s)\bt\!\(\s*"(?<key>.*?)".*?\)'
31-
panic_t = '(?s)\bpanic_t\!\(\s*"(?<key>.*?)".*?\)'
32-
assert_t = '(?s)\bassert_t\!\(\s*.*?,\s*"(?<key>.*?)".*?\)'
51+
$translationInfo.DuplicateTranslations | Should @check
3352
}
3453

35-
$missing = @()
36-
Get-ChildItem -Recurse -Path $project -Include *.rs -File | ForEach-Object {
37-
$content = Get-Content -Path $_ -Raw
38-
foreach ($pattern in $patterns.keys) {
39-
($content | Select-String -Pattern $patterns[$pattern] -AllMatches).Matches | ForEach-Object {
40-
# write-verbose -verbose "Line: $_"
41-
if ($null -ne $_) {
42-
$key = $_.Groups['key'].Value
43-
if ($i18n.ContainsKey($key)) {
44-
$i18n[$key] = 1
45-
}
46-
else {
47-
$missing += $key
48-
}
49-
}
50-
}
54+
It 'Uses every defined translation string' {
55+
$check = @{
56+
BeNullOrEmpty = $true
57+
Because = (@(
58+
"The following translation keys are defined but not used:"
59+
$translationInfo.UnusedTranslations | ConvertTo-Json -Depth 2
60+
) -join ' ')
5161
}
62+
$translationInfo.UnusedTranslations | Should @check
5263
}
5364

54-
$missing | Should -BeNullOrEmpty -Because "The following i18n keys are missing from $toml :`n$($missing | Out-String)"
55-
$unused = $i18n.GetEnumerator() | Where-Object { $_.Value -eq 0 } | ForEach-Object { $_.Key }
56-
$unused | Should -BeNullOrEmpty -Because "The following i18n keys are unused in the project:`n$($unused | Out-String)"
65+
It 'Defines every used translation string' {
66+
$check = @{
67+
BeNullOrEmpty = $true
68+
Because = (@(
69+
"The following translation keys are used but not defined:"
70+
$translationInfo.MissingTranslations | ConvertTo-Json -Depth 2
71+
) -join ' ')
72+
}
73+
$translationInfo.MissingTranslations | Should @check
74+
}
5775
}
5876
}

0 commit comments

Comments
 (0)