Skip to content

Commit ae5d62d

Browse files
Add collation from ICU4X (#740)
* Add collation * Remove unused import * Update to merged * Setup submodule * Checkout submodules * Add build.dart * move bindings * Use path to build output dir * Fix build.dart * Enable test * Add help * Fix help * Make data default * Move collation test to general testing * Fix workflow * Run tests on dev * Fix imports * Add changelog and rev version * take health from branch * Add todo * Add top level analysis options * Fix collator options * Do checkout submodules * Use checkout submodules * Make casefirst non null * enable native assets experiment * Rev SDK * And env * set default mode * Add licenses * Adapt option matching * Update icu4x * Update icu4x ref * Add ignores * Update submodule * Remove cached * Remove branch * remove submodule * Add submodule * Remove submodules * Add submodule * Test current_repo * remove bracket * Add ignores * Add changelog * Add trailing newline * Incorporate build_libs.dart * Add simulator special casing * Update pkgs/intl4x/build.dart Co-authored-by: Robert Bastian <[email protected]> * Update pkgs/intl4x/build.dart Co-authored-by: Robert Bastian <[email protected]> * Reintroduce platform name for `ubuntu` --------- Co-authored-by: Robert Bastian <[email protected]>
1 parent fadcbfd commit ae5d62d

File tree

178 files changed

+773
-228
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

178 files changed

+773
-228
lines changed

.github/workflows/health.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,17 @@ on:
33
pull_request:
44
branches: [ main ]
55
types: [opened, synchronize, reopened, labeled, unlabeled]
6+
67
jobs:
78
health:
89
uses: dart-lang/ecosystem/.github/workflows/health.yaml@main
910
with:
1011
coverage_web: true
12+
checkout_submodules: true
13+
experiments: native-assets
14+
sdk: dev
15+
ignore_license: "**.g.dart,pkgs/intl_translation/example/lib/generated/**,pkgs/intl_translation/test/generate_localized/**,pkgs/intl_translation/test/two_components/**"
16+
ignore_coverage: "**.g.dart"
17+
ignore_packages: "submodules"
1118
permissions:
1219
pull-requests: write

.github/workflows/intl4x.yml

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,30 +18,37 @@ on:
1818
jobs:
1919
build:
2020
runs-on: ubuntu-latest
21+
22+
env:
23+
ICU4X_BUILD_MODE: fetch
24+
2125
defaults:
2226
run:
2327
working-directory: pkgs/intl4x
2428
strategy:
2529
matrix:
2630
sdk: [stable, dev] # {pkgs.versions}
2731
include:
28-
- sdk: stable
32+
- sdk: dev
2933
run-tests: true
3034
steps:
3135
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
36+
with:
37+
submodules: true
38+
3239
- uses: dart-lang/setup-dart@fedb1266e91cf51be2fdb382869461a434b920a3
3340
with:
3441
sdk: ${{matrix.sdk}}
3542

36-
- run: dart pub get
43+
- run: dart --enable-experiment=native-assets pub get
3744

38-
- run: dart analyze --fatal-infos
45+
- run: dart --enable-experiment=native-assets analyze --fatal-infos
3946

40-
- run: dart format --output=none --set-exit-if-changed .
47+
- run: dart --enable-experiment=native-assets format --output=none --set-exit-if-changed .
4148
if: ${{matrix.run-tests}}
4249

43-
- run: dart test
50+
- run: dart --enable-experiment=native-assets test
4451
if: ${{matrix.run-tests}}
4552

46-
- run: dart test -p chrome
53+
- run: dart --enable-experiment=native-assets test -p chrome
4754
if: ${{matrix.run-tests}}

.github/workflows/publish.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ jobs:
1414
uses: dart-lang/ecosystem/.github/workflows/publish.yaml@main
1515
with:
1616
write-comments: false
17+
checkout_submodules: true

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@
1515

1616
# Specific files
1717
pkgs/intl/test/number_format_compact_google3_icu_test.dart
18-
pkgs/intl/update_from_cldr_data.sh
18+
pkgs/intl/update_from_cldr_data.sh

.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "submodules/icu4x"]
2+
path = submodules/icu4x
3+
url = https://github.com/unicode-org/icu4x.git

analysis_options.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
analyzer:
2+
exclude:
3+
- "submodules/*"

pkgs/intl4x/CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 0.8.0
2+
3+
- Add ICU4X support for collation.
4+
15
## 0.7.1
26

37
- Export plural rules.

pkgs/intl4x/analysis_options.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,8 @@ linter:
77
- prefer_final_locals
88
- prefer_relative_imports
99
- unnecessary_parenthesis
10+
11+
analyzer:
12+
exclude:
13+
- "submodules/*"
14+
- "lib/src/bindings/*"

pkgs/intl4x/build.dart

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import 'package:archive/archive_io.dart';
8+
import 'package:native_assets_cli/native_assets_cli.dart';
9+
import 'package:path/path.dart' as path;
10+
11+
const crateName = 'icu_capi';
12+
const assetId = 'package:intl4x/src/bindings/lib.g.dart';
13+
14+
void main(List<String> args) async {
15+
final config = await BuildConfig.fromArgs(args);
16+
17+
final libFolder = path.join(config.outDir.path, 'release');
18+
Directory(libFolder).createSync();
19+
final libPath = path.join(
20+
libFolder,
21+
config.targetOs.dylibFileName(crateName.replaceAll('-', '_')),
22+
);
23+
24+
final buildMode = switch (Platform.environment['ICU4X_BUILD_MODE']) {
25+
'local' => LocalMode(libPath),
26+
'checkout' => CheckoutMode(config, libPath),
27+
'fetch' || null => FetchMode(libPath),
28+
String() => throw ArgumentError('''
29+
30+
31+
Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable with either `fetch`, `local`, or `checkout`.
32+
* fetch: Fetch the precompiled binary from a CDN.
33+
* local: Use a locally existing binary at the environment variable `LOCAL_ICU4X_BINARY`.
34+
* checkout: Build a fresh library from a local git checkout of the icu4x repository at the environment variable `LOCAL_ICU4X_CHECKOUT`.
35+
36+
'''),
37+
};
38+
39+
await buildMode.build();
40+
41+
await BuildOutput(
42+
assets: [
43+
Asset(
44+
id: assetId,
45+
linkMode: LinkMode.dynamic,
46+
target: Target.current,
47+
path: AssetAbsolutePath(Uri.file(libPath)),
48+
)
49+
],
50+
dependencies: Dependencies([Uri.file('build.dart')]),
51+
).writeToFile(outDir: config.outDir);
52+
}
53+
54+
void unzipFirstFile({required File input, required File output}) {
55+
final inputStream = InputFileStream(input.path);
56+
final archive = ZipDecoder().decodeBuffer(inputStream);
57+
final file = archive.files.firstOrNull;
58+
// If it's a file and not a directory
59+
if (file?.isFile ?? false) {
60+
final outputStream = OutputFileStream(output.path);
61+
file!.writeContent(outputStream);
62+
outputStream.close();
63+
}
64+
}
65+
66+
sealed class BuildMode {
67+
Future<void> build();
68+
}
69+
70+
final class FetchMode implements BuildMode {
71+
final String libPath;
72+
73+
FetchMode(this.libPath);
74+
75+
@override
76+
Future<void> build() async {
77+
// TODO: Get a nicer CDN than a generated link to a privately owned repo.
78+
final request = await HttpClient().getUrl(Uri.parse(
79+
'https://nightly.link/mosuem/i18n/workflows/intl4x_artifacts/main/lib-$platformName-latest.zip'));
80+
final response = await request.close();
81+
82+
final zippedDynamicLibrary =
83+
File(path.join(Directory.systemTemp.path, 'tmp.zip'));
84+
zippedDynamicLibrary.createSync();
85+
await response.pipe(zippedDynamicLibrary.openWrite());
86+
87+
final dynamicLibrary = File(libPath);
88+
dynamicLibrary.createSync(recursive: true);
89+
unzipFirstFile(input: zippedDynamicLibrary, output: dynamicLibrary);
90+
}
91+
92+
String get platformName {
93+
if (Platform.isMacOS) {
94+
return 'macos';
95+
} else if (Platform.isWindows) {
96+
return 'windows';
97+
} else {
98+
return 'ubuntu';
99+
}
100+
}
101+
}
102+
103+
final class LocalMode implements BuildMode {
104+
final String libPath;
105+
106+
LocalMode(this.libPath);
107+
108+
String get _localBinaryPath => Platform.environment['LOCAL_ICU4X_BINARY']!;
109+
110+
@override
111+
Future<void> build() async {
112+
await File(_localBinaryPath).copy(libPath);
113+
}
114+
}
115+
116+
final class CheckoutMode implements BuildMode {
117+
final BuildConfig config;
118+
final String libPath;
119+
CheckoutMode(this.config, this.libPath);
120+
121+
@override
122+
Future<void> build() async {
123+
final workingDirectory = Platform.environment['LOCAL_ICU4X_CHECKOUT'];
124+
if (workingDirectory == null) {
125+
throw ArgumentError('Specify the ICU4X checkout folder'
126+
'with the LOCAL_ICU4X_CHECKOUT variable');
127+
}
128+
final lib = await buildLib(
129+
config,
130+
workingDirectory,
131+
);
132+
await File(lib).copy(libPath);
133+
}
134+
}
135+
136+
Future<String> buildLib(
137+
BuildConfig config,
138+
String workingDirectory,
139+
) async {
140+
final rustTarget =
141+
config.target.asRustTarget(config.targetIOSSdk == IOSSdk.iPhoneSimulator);
142+
final isNoStd = config.target.isNoStdTarget;
143+
144+
if (!isNoStd) {
145+
final rustArguments = ['target', 'add', rustTarget];
146+
final rustup = await Process.run(
147+
'rustup',
148+
rustArguments,
149+
workingDirectory: workingDirectory,
150+
);
151+
152+
if (rustup.exitCode != 0) {
153+
throw ProcessException(
154+
'rustup',
155+
rustArguments,
156+
rustup.stderr.toString(),
157+
rustup.exitCode,
158+
);
159+
}
160+
}
161+
162+
final stdFeatures = [
163+
'default_components',
164+
'compiled_data',
165+
'buffer_provider',
166+
'logging',
167+
'simple_logger',
168+
];
169+
final noStdFeatures = [
170+
'default_components',
171+
'compiled_data',
172+
'buffer_provider',
173+
'libc-alloc',
174+
'panic-handler'
175+
];
176+
final tempDir = Directory.systemTemp.createTempSync();
177+
final linkModeType =
178+
config.linkModePreference.preferredLinkMode == LinkMode.static
179+
? 'staticlib'
180+
: 'cdylib';
181+
final arguments = [
182+
if (isNoStd) '+nightly',
183+
'rustc',
184+
'-p={crateName}',
185+
'--crate-type=$linkModeType',
186+
'--release',
187+
'--config=profile.release.panic="abort"',
188+
'--config=profile.release.codegen-units=1',
189+
'--no-default-features',
190+
if (!isNoStd) '--features=${stdFeatures.join(',')}',
191+
if (isNoStd) '--features=${noStdFeatures.join(',')}',
192+
if (isNoStd) '-Zbuild-std=core,alloc',
193+
if (isNoStd) '-Zbuild-std-features=panic_immediate_abort',
194+
'--target=$rustTarget',
195+
'--target-dir=${tempDir.path}'
196+
];
197+
final cargo = await Process.run(
198+
'cargo',
199+
arguments,
200+
workingDirectory: workingDirectory,
201+
);
202+
203+
if (cargo.exitCode != 0) {
204+
throw ProcessException(
205+
'cargo',
206+
arguments,
207+
cargo.stderr.toString(),
208+
cargo.exitCode,
209+
);
210+
}
211+
212+
final dylibFilePath = path.join(
213+
tempDir.path,
214+
rustTarget,
215+
'release',
216+
config.target.os.dylibFileName(crateName.replaceAll('-', '_')),
217+
);
218+
if (!File(dylibFilePath).existsSync()) {
219+
throw FileSystemException('Building the dylib failed', dylibFilePath);
220+
}
221+
return dylibFilePath;
222+
}
223+
224+
extension on Target {
225+
String asRustTarget(bool isSimulator) {
226+
if (this == Target.iOSArm64 && isSimulator) {
227+
return 'aarch64-apple-ios-sim';
228+
}
229+
return switch (this) {
230+
Target.androidArm => 'armv7-linux-androideabi',
231+
Target.androidArm64 => 'aarch64-linux-android',
232+
Target.androidIA32 => 'i686-linux-android',
233+
Target.androidRiscv64 => 'riscv64-linux-android',
234+
Target.androidX64 => 'x86_64-linux-android',
235+
Target.fuchsiaArm64 => 'aarch64-unknown-fuchsia',
236+
Target.fuchsiaX64 => 'x86_64-unknown-fuchsia',
237+
Target.iOSArm64 => 'aarch64-apple-ios',
238+
Target.iOSX64 => 'x86_64-apple-ios',
239+
Target.linuxArm => 'armv7-unknown-linux-gnueabihf',
240+
Target.linuxArm64 => 'aarch64-unknown-linux-gnu',
241+
Target.linuxIA32 => 'i686-unknown-linux-gnu',
242+
Target.linuxRiscv32 => 'riscv32gc-unknown-linux-gnu',
243+
Target.linuxRiscv64 => 'riscv64gc-unknown-linux-gnu',
244+
Target.linuxX64 => 'x86_64-unknown-linux-gnu',
245+
Target.macOSArm64 => 'aarch64-apple-darwin',
246+
Target.macOSX64 => 'x86_64-apple-darwin',
247+
Target.windowsArm64 => 'aarch64-pc-windows-msvc',
248+
Target.windowsIA32 => 'i686-pc-windows-msvc',
249+
Target.windowsX64 => 'x86_64-pc-windows-msvc',
250+
Target() => throw UnimplementedError('Target not available for rust'),
251+
};
252+
}
253+
254+
bool get isNoStdTarget =>
255+
[Target.androidRiscv64, Target.linuxRiscv32].contains(this);
256+
}

0 commit comments

Comments
 (0)