Skip to content

Commit 5556482

Browse files
WeiqunZhangWeiqunZhang
committed
Merge branch 'gitlab-ci' into mock_development
2 parents 76996ff + ec3e9ad commit 5556482

File tree

11 files changed

+204
-89
lines changed

11 files changed

+204
-89
lines changed

‎.gitlab/hpsf-gitlab-ci.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
Nvidia-H100-single-precision:
2+
stage: test
3+
tags: [nvidia-h100]
4+
image: nvcr.io/nvidia/cuda:12.8.0-devel-ubuntu24.04
5+
script:
6+
- apt-get update && apt-get install -y cmake git python3 libopenmpi-dev openmpi-bin
7+
- |
8+
cmake -S . -B build \
9+
-DCMAKE_VERBOSE_MAKEFILE=ON \
10+
-DAMReX_PRECISION=SINGLE \
11+
-DAMReX_PARTICLES_PRECISION=SINGLE \
12+
-DAMReX_SPACEDIM="1;2;3" \
13+
-DAMReX_FFT=ON \
14+
-DAMReX_EB=ON \
15+
-DAMReX_ENABLE_TESTS=ON \
16+
-DAMReX_FORTRAN=OFF \
17+
-DAMReX_GPU_BACKEND=CUDA \
18+
-DCMAKE_C_COMPILER=$(which gcc) \
19+
-DCMAKE_CXX_COMPILER=$(which g++) \
20+
-DCMAKE_CUDA_HOST_COMPILER=$(which g++) \
21+
-DAMReX_CUDA_ARCH="9.0" \
22+
-DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
23+
-DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
24+
- cmake --build build -j 16
25+
- export OMPI_ALLOW_RUN_AS_ROOT=1
26+
- export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
27+
- export AMREX_THE_ARENA_INIT_SIZE=1e9
28+
- ctest -j 2 --test-dir build --output-on-failure
29+
30+
AMD-MI300A:
31+
stage: test
32+
tags: [amd-mi300]
33+
image: rocm/dev-ubuntu-24.04:6.4.3-complete
34+
script:
35+
- apt-get update && apt-get install -y cmake git libopenmpi-dev openmpi-bin
36+
- export ROCM_PATH=/opt/rocm-6.4.3
37+
- export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:$PATH
38+
- export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$LD_LIBRARY_PATH
39+
- export HIP_PATH=$ROCM_PATH
40+
- export HIP_CLANG_PATH=$ROCM_PATH/llvm/bin
41+
- export CMAKE_PREFIX_PATH=$ROCM_PATH:$CMAKE_PREFIX_PATH
42+
- |
43+
cmake -S . -B build \
44+
-DCMAKE_VERBOSE_MAKEFILE=ON \
45+
-DAMReX_SPACEDIM="1;2;3" \
46+
-DAMReX_FFT=ON \
47+
-DAMReX_EB=ON \
48+
-DAMReX_ENABLE_TESTS=ON \
49+
-DAMReX_FORTRAN=OFF \
50+
-DAMReX_GPU_BACKEND=HIP \
51+
-DAMReX_AMD_ARCH=gfx942 \
52+
-DCMAKE_C_COMPILER=$(which clang) \
53+
-DCMAKE_CXX_COMPILER=$(which clang++) \
54+
-DCMAKE_CXX_STANDARD=17
55+
- cmake --build build -j 16
56+
- export OMPI_ALLOW_RUN_AS_ROOT=1
57+
- export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
58+
- export AMREX_THE_ARENA_INIT_SIZE=1e9
59+
- ctest -j 2 --test-dir build --output-on-failure
60+
61+
Intel-PVC:
62+
stage: test
63+
tags: [intel-data-center-max-1100]
64+
image: intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04
65+
script:
66+
- apt-get update && apt-get install -y git
67+
- sycl-ls
68+
- export ONEAPI_DEVICE_SELECTOR=level_zero:gpu
69+
- |
70+
cmake -S . -B build \
71+
-DCMAKE_VERBOSE_MAKEFILE=ON \
72+
-DAMReX_MPI=OFF \
73+
-DAMReX_SPACEDIM="1;2;3" \
74+
-DAMReX_FFT=ON \
75+
-DAMReX_EB=ON \
76+
-DAMReX_ENABLE_TESTS=ON \
77+
-DAMReX_FORTRAN=OFF \
78+
-DAMReX_GPU_BACKEND=SYCL \
79+
-DCMAKE_C_COMPILER=$(which icx) \
80+
-DCMAKE_CXX_COMPILER=$(which icpx) \
81+
-DAMReX_PARALLEL_LINK_JOBS=8
82+
- cmake --build build -j 16
83+
- export AMREX_THE_ARENA_INIT_SIZE=1e9
84+
- ctest -j 4 --test-dir build --output-on-failure

‎Docs/sphinx_documentation/source/GPU.rst‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -560,8 +560,16 @@ to :cpp:`The_Arena()` to reduce memory fragmentation.
560560
In :cpp:`amrex::Initialize`, a large amount of GPU device memory is
561561
allocated and is kept in :cpp:`The_Arena()`. The default is 3/4 of the
562562
total device memory, and it can be changed with a :cpp:`ParmParse`
563-
parameter, ``amrex.the_arena_init_size``, in the unit of bytes. The default
564-
initial size for other arenas is 8388608 (i.e., 8 MB). For
563+
parameter, ``amrex.the_arena_init_size``, in the unit of bytes. The default
564+
can also be changed with an environment variable
565+
``AMREX_THE_ARENA_INIT_SIZE=X``, where ``X`` is the number of bytes. When
566+
both the :cpp:`ParmParse` parameter and the environment variable are
567+
present, the former will override the latter. In both cases, the number
568+
string could have optional single quotes ``'`` as separators (e.g.,
569+
``10'000'000'000``). It may also use floating-point notation (``2.5e10``),
570+
as long as converting it does not introduce any loss of precision.
571+
572+
The default initial size for other arenas is 8388608 (i.e., 8 MB). For
565573
:cpp:`The_Managed_Arena()` and :cpp:`The_Device_Arena()`, it can be changed
566574
with ``amrex.the_managed_arena_init_size`` and
567575
``amrex.the_device_arena_init_size``, respectively, if they are not an alias

‎Src/Base/AMReX_Arena.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include <AMReX.H>
88
#include <AMReX_BLProfiler.H>
9+
#include <AMReX_IParser.H>
910
#include <AMReX_Print.H>
1011
#include <AMReX_ParallelDescriptor.H>
1112
#include <AMReX_ParmParse.H>
@@ -362,6 +363,13 @@ Arena::Initialize (bool minimal)
362363
the_pinned_arena_release_threshold = Gpu::Device::totalGlobalMem() / Gpu::Device::numDevicePartners() / 2L;
363364
#endif
364365

366+
// Overwrite the initial size with environment variables
367+
if (char const* init_size_p = std::getenv("AMREX_THE_ARENA_INIT_SIZE")) {
368+
IParser iparser(init_size_p);
369+
auto exe = iparser.compileHost<0>();
370+
the_arena_init_size = exe();
371+
}
372+
365373
ParmParse pp("amrex");
366374
pp.queryAdd( "the_arena_init_size", the_arena_init_size);
367375
pp.queryAdd( "the_device_arena_init_size", the_device_arena_init_size);

‎Tests/Algebra/GMRES/main.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,11 @@ int main (int argc, char* argv[])
113113
amrex::Axpy(xvec, Real(-1.0), exact);
114114
auto error = xvec.norminf();
115115
amrex::Print() << " Max norm error: " << error << "\n";
116+
#ifdef AMREX_USE_FLOAT
117+
AMREX_ALWAYS_ASSERT(error < eps);
118+
#else
116119
AMREX_ALWAYS_ASSERT(error*10 < eps);
120+
#endif
117121
}
118122
amrex::Finalize();
119123
}

‎Tests/CommType/main.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ int main(int argc, char* argv[])
1212
int ret_code = EXIT_SUCCESS;
1313

1414
{
15-
int ncells = 128;
15+
int ncells = 64;
1616
BoxArray ba(Box(IntVect(0), IntVect(ncells-1)));
17-
ba.maxSize(32);
17+
ba.maxSize(16);
1818
ba.convert(IntVect(1));
1919
DistributionMapping dm(ba);
2020

‎Tests/FFT/Batch/main.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ int main (int argc, char* argv[])
106106
auto error = mf2.norminf(0, batch_size, IntVect(0));
107107
amrex::Print() << " Expected to be close to zero: " << error << "\n";
108108
#ifdef AMREX_USE_FLOAT
109-
auto eps = 1.e-6f;
109+
auto eps = 3.e-6F;
110110
#else
111111
auto eps = 1.e-13;
112112
#endif
@@ -133,7 +133,7 @@ int main (int argc, char* argv[])
133133
auto error = errmf.norminf(0, batch_size, IntVect(0));
134134
amrex::Print() << " Expected to be close to zero: " << error << "\n";
135135
#ifdef AMREX_USE_FLOAT
136-
auto eps = 0.5e-6f;
136+
auto eps = 3.e-6F;
137137
#else
138138
auto eps = 1.e-15;
139139
#endif
@@ -156,7 +156,7 @@ int main (int argc, char* argv[])
156156
auto error = mf2.norminf(0, batch_size, IntVect(0));
157157
amrex::Print() << " Expected to be close to zero: " << error << "\n";
158158
#ifdef AMREX_USE_FLOAT
159-
auto eps = 1.e-6f;
159+
auto eps = 3.e-6F;
160160
#else
161161
auto eps = 1.e-13;
162162
#endif

‎Tests/FFT/RawPtr/main.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ int main (int argc, char* argv[])
6969
});
7070
amrex::Print() << " Expected to be close to zero: " << error << "\n";
7171
#ifdef AMREX_USE_FLOAT
72-
auto eps = 1.e-6f;
72+
auto eps = 3.e-6F;
7373
#else
7474
auto eps = 1.e-13;
7575
#endif
@@ -124,7 +124,7 @@ int main (int argc, char* argv[])
124124
});
125125
amrex::Print() << " Expected to be close to zero: " << error << "\n";
126126
#ifdef AMREX_USE_FLOAT
127-
auto eps = 1.e-6f;
127+
auto eps = 3.e-6F;
128128
#else
129129
auto eps = 1.e-13;
130130
#endif

‎Tests/LinearSolvers/NodalPoisson/MyTest.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,9 @@ MyTest::readParameters ()
166166
pp.query("max_iter", max_iter);
167167
pp.query("max_fmg_iter", max_fmg_iter);
168168
pp.query("reltol", reltol);
169+
#ifdef AMREX_USE_FLOAT
170+
reltol = std::max(reltol, 1.e-5F);
171+
#endif
169172

170173
pp.query("gpu_regtest", gpu_regtest);
171174

‎Tests/LinearSolvers/Nodal_Projection_EB/main.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,11 @@ int main (int argc, char* argv[])
249249
// nodal_solver.setBottomSolver(MLMG::BottomSolver::hypre);
250250

251251
// Define the relative tolerance
252+
#ifdef AMREX_USE_FLOAT
253+
Real reltol = 2.e-4;
254+
#else
252255
Real reltol = 1.e-8;
256+
#endif
253257

254258
// Define the absolute tolerance; note that this argument is optional
255259
Real abstol = 1.e-15;

0 commit comments

Comments
 (0)