Skip to content

Commit 20f6153

Browse files
kito-chengkito-cheng
authored
Merge pull request #1608 from CARV-ICS-FORTH/ci-updates
Update build workflow
2 parents a4b355b + 404de92 commit 20f6153

File tree

6 files changed

+234
-77
lines changed

6 files changed

+234
-77
lines changed

‎.github/cleanup-rootfs.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
3+
# There is no need to wait for this to finish, at least the rm part can be done
4+
# while the process moves forward (for apt we'll need to wait for it to finish
5+
# before we install dependencies later on, but it'll only give us a 1-3GBs so
6+
# we can skip it.
7+
WAIT=0
8+
RMONLY=1
9+
10+
PACKAGES=(
11+
"firefox"
12+
"google-chrome-stable"
13+
"microsoft-edge-stable"
14+
"php-pear"
15+
"ruby-full"
16+
"^aspnetcore-.*"
17+
"^dotnet-.*"
18+
"powershell*"
19+
)
20+
21+
PATHS=(
22+
"/opt/hostedtoolcache"
23+
"/usr/local/.ghcup/"
24+
"/usr/share/swift"
25+
"/usr/local/lib/android"
26+
"/usr/local/share/edge_driver"
27+
"/usr/local/share/gecko_driver"
28+
"/usr/local/share/chromedriver-linux64"
29+
"/usr/local/share/chromium"
30+
"/home/linuxbrew"
31+
"/usr/local/share/vcpkg"
32+
"/usr/share/kotlinc"
33+
"/usr/local/bin/minikube"
34+
)
35+
36+
37+
function cleanup_packages()
38+
{
39+
if [[ ${RMONLY} == 0 ]]; then
40+
apt-get purge -y "${PACKAGES[@]}"
41+
apt-get autoremove --purge -y
42+
apt-get clean
43+
fi
44+
}
45+
46+
function cleanup_paths()
47+
{
48+
for i in "${PATHS[@]}"; do
49+
rm -rf "${i}" &
50+
done
51+
if [[ ${WAIT} == 1 ]]; then
52+
wait
53+
fi
54+
}
55+
56+
if [[ ${WAIT} == 1 ]]; then
57+
echo "---=== Before ===---"
58+
df -hT
59+
cleanup_packages
60+
cleanup_paths
61+
echo "---=== After ===---"
62+
df -hT
63+
else
64+
cleanup_packages
65+
cleanup_paths
66+
fi

‎.github/dedup-dir.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
3+
function deduplicate_files() {
4+
local DUPLICATES=()
5+
local DIR=${1}
6+
local IFS=$'\n'
7+
local LINK_CHECK=""
8+
9+
readarray -t DUPLICATES < <(for i in `find ${DIR} -type f ! -empty`; do sha1sum ${i}; done | sort | uniq -w 40 --all-repeated=separate)
10+
11+
for ((i=1; i < ${#DUPLICATES[@]}; i++ )); do
12+
if [[ ${DUPLICATES[$i]} == "" ]]; then
13+
continue
14+
elif [[ ${DUPLICATES[$i-1]} = "" ]]; then
15+
continue
16+
else
17+
LINK_CHECK=$(ls -li "${DUPLICATES[$i]:42}" "${DUPLICATES[$i-1]:42}" |awk '{print $1}' | uniq | wc -l)
18+
if [[ ${LINK_CHECK} != "1" ]]; then
19+
ln -f "${DUPLICATES[$i-1]:42}" "${DUPLICATES[$i]:42}"
20+
fi
21+
fi
22+
done
23+
24+
return 0
25+
}
26+
27+
deduplicate_files ${1}

‎.github/workflows/build.yaml‎

Lines changed: 111 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,67 @@
11
name: Build
22

33
on:
4+
workflow_dispatch:
45
push:
56
branches:
67
- master
78
pull_request:
89
branches:
910
- master
1011

12+
env:
13+
submodule_paths: |
14+
binutils
15+
dejagnu
16+
gcc
17+
gdb
18+
glibc
19+
llvm
20+
musl
21+
newlib
22+
pk
23+
qemu
24+
spike
25+
uclibc-ng
26+
.git/modules
27+
1128
jobs:
29+
submodule_cache:
30+
name: Initialize submodule cache
31+
runs-on: ubuntu-latest
32+
outputs:
33+
key: ${{ steps.keygen.outputs.smcache_key }}
34+
steps:
35+
- uses: actions/checkout@v4
36+
37+
- name: Remove unneeded frameworks to recover disk space
38+
run: sudo ./.github/cleanup-rootfs.sh
39+
40+
- name: Generate submodule cache key
41+
id: keygen
42+
run: echo "smcache_key=smcache-$(printf $(git submodule | sha1sum))" >> $GITHUB_OUTPUT
43+
44+
- name: Setup submodule cache
45+
id: smcache
46+
uses: actions/cache@v4
47+
with:
48+
path: ${{ env.submodule_paths }}
49+
key: ${{ steps.keygen.outputs.smcache_key }}
50+
51+
- name: Checkout required submodules
52+
if: steps.smcache.outputs.cache-hit != 'true'
53+
run: git submodule update --init -j $(nproc) --depth 1 $(echo ${submodule_paths} | sed '$d' | tr '\n' ' ')
54+
55+
- name: Storage size optimization
56+
if: steps.smcache.outputs.cache-hit != 'true'
57+
run: |
58+
git submodule foreach 'git maintenance run'
59+
1260
build:
1361
runs-on: ${{ matrix.os }}
62+
needs: [submodule_cache]
63+
env:
64+
smcache_key: ${{ needs.submodule_cache.outputs.key }}
1465
strategy:
1566
matrix:
1667
os: [ubuntu-22.04, ubuntu-24.04]
@@ -23,47 +74,46 @@ jobs:
2374
- mode: uclibc
2475
compiler: llvm
2576
steps:
26-
- name: Remove unneeded frameworks to recover disk space
27-
run: |
28-
echo "-- Before --"
29-
df -h
30-
sudo rm -rf /usr/share/dotnet
31-
sudo rm -rf /usr/local/lib/android
32-
echo "-- After --"
33-
df -h
34-
3577
- uses: actions/checkout@v4
3678

79+
- name: Remove unneeded frameworks to recover disk space
80+
run: sudo ./.github/cleanup-rootfs.sh
81+
3782
- name: install dependencies
3883
run: sudo ./.github/setup-apt.sh
3984

85+
- name: Load submodule cache
86+
uses: actions/cache/restore@v4
87+
with:
88+
path: ${{ env.submodule_paths }}
89+
key: ${{ env.smcache_key }}
90+
4091
- name: build toolchain
4192
run: |
4293
TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
43-
BUILD_TOOLCHAIN="./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}"
94+
BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}"
4495
if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm
4596
$BUILD_TOOLCHAIN --enable-llvm
4697
else
4798
$BUILD_TOOLCHAIN
4899
fi
49-
sudo make -j $(nproc) ${{ matrix.mode }}
100+
sudo mkdir /mnt/riscv
101+
sudo chown runner:runner /mnt/riscv
102+
make -j $(nproc) ${{ matrix.mode }}
103+
104+
- name: tarball build
105+
run: |
106+
du -s -h /mnt/riscv
107+
./.github/dedup-dir.sh /mnt/riscv/
108+
XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/
50109
51110
- name: make report
52111
if: |
53112
matrix.os == 'ubuntu-24.04'
54113
&& (matrix.mode == 'linux' || matrix.mode == 'newlib')
55114
&& matrix.compiler == 'gcc'
56115
run: |
57-
sudo make report-${{ matrix.mode }} -j $(nproc)
58-
59-
- name: recover space
60-
run: |
61-
sudo du -hs / 2> /dev/null || true
62-
sudo rm -rf binutils dejagnu gcc gdb glibc llvm musl newlib pk qemu spike uclibc-ng || true
63-
sudo du -hs / 2> /dev/null || true
64-
65-
- name: tarball build
66-
run: tar czvf riscv.tar.gz -C /opt/ riscv/
116+
make report-${{ matrix.mode }} -j $(nproc)
67117
68118
- name: generate prebuilt toolchain name
69119
id: toolchain-name-generator
@@ -84,35 +134,40 @@ jobs:
84134
- uses: actions/upload-artifact@v4
85135
with:
86136
name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }}
87-
path: riscv.tar.gz
137+
path: riscv.tar.xz
88138

89139
test-sim:
90140
runs-on: ${{ matrix.os }}
141+
needs: [submodule_cache]
142+
env:
143+
smcache_key: ${{ needs.submodule_cache.outputs.key }}
91144
strategy:
92145
matrix:
93146
os: [ubuntu-24.04]
94147
mode: [newlib]
95148
target: [rv64gc-lp64d]
96149
sim: [spike]
97150
steps:
98-
- name: Remove unneeded frameworks to recover disk space
99-
run: |
100-
echo "-- Before --"
101-
df -h
102-
sudo rm -rf /usr/share/dotnet
103-
sudo rm -rf /usr/local/lib/android
104-
echo "-- After --"
105-
df -h
106-
107151
- uses: actions/checkout@v4
108152

153+
- name: Remove unneeded frameworks to recover disk space
154+
run: sudo ./.github/cleanup-rootfs.sh
155+
109156
- name: install dependencies
110157
run: sudo ./.github/setup-apt.sh
111158

159+
- name: Load submodule cache
160+
uses: actions/cache/restore@v4
161+
with:
162+
path: ${{ env.submodule_paths }}
163+
key: ${{ env.smcache_key }}
164+
112165
- name: build toolchain
113166
run: |
114167
TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
115-
./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }}
168+
./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }}
169+
sudo mkdir /mnt/riscv
170+
sudo chown runner:runner /mnt/riscv
116171
make -j $(nproc) ${{ matrix.mode }}
117172
118173
- name: make report
@@ -121,38 +176,46 @@ jobs:
121176
build-multilib:
122177
if: ${{ false }} # Disable until multilib errors are triaged
123178
runs-on: ${{ matrix.os }}
179+
needs: [submodule_cache]
180+
env:
181+
smcache_key: ${{ needs.submodule_cache.outputs.key }}
124182
strategy:
125183
matrix:
126184
os: [ubuntu-24.04]
127185
mode: [newlib, linux]
128186
target: [rv64gc-lp64d]
129187
steps:
130-
- name: Remove unneeded frameworks to recover disk space
131-
run: |
132-
echo "-- Before --"
133-
df -h
134-
sudo rm -rf /usr/share/dotnet
135-
sudo rm -rf /usr/local/lib/android
136-
echo "-- After --"
137-
df -h
138-
139188
- uses: actions/checkout@v4
140189

190+
- name: Remove unneeded frameworks to recover disk space
191+
run: sudo ./.github/cleanup-rootfs.sh
192+
141193
- name: install dependencies
142194
run: sudo ./.github/setup-apt.sh
143195

196+
- name: Load submodule cache
197+
uses: actions/cache/restore@v4
198+
with:
199+
path: ${{ env.submodule_paths }}
200+
key: ${{ env.smcache_key }}
201+
144202
- name: build toolchain
145203
run: |
146204
TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
147-
./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib
148-
sudo make -j $(nproc) ${{ matrix.mode }}
205+
./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib
206+
sudo mkdir /mnt/riscv
207+
sudo chown runner:runner /mnt/riscv
208+
make -j $(nproc) ${{ matrix.mode }}
149209
150-
- name: make report
210+
- name: tarball build
151211
run: |
152-
sudo make report-${{ matrix.mode }} -j $(nproc)
212+
du -s -h /mnt/riscv
213+
./.github/dedup-dir.sh /mnt/riscv/
214+
XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/
153215
154-
- name: tarball build
155-
run: tar czvf riscv.tar.gz -C /opt/ riscv/
216+
- name: make report
217+
run: |
218+
make report-${{ matrix.mode }} -j $(nproc)
156219
157220
- name: generate prebuilt toolchain name
158221
id: toolchain-name-generator
@@ -173,4 +236,4 @@ jobs:
173236
- uses: actions/upload-artifact@v4
174237
with:
175238
name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }}
176-
path: riscv.tar.gz
239+
path: riscv.tar.xz

0 commit comments

Comments
 (0)