Skip to content

Commit 12e9db0

Browse files
tanjinxvitess-bot[bot]arthurschreiber
authored
[slack-22.0] ci backports from release v22.0.2 (#745)
* [release-22.0] ci: use the newest mysql apt config package (vitessio#18790) (vitessio#18793) Signed-off-by: Arthur Schreiber <[email protected]> Co-authored-by: Arthur Schreiber <[email protected]> Co-authored-by: Arthur Schreiber <[email protected]> Signed-off-by: Tanjin Xu <[email protected]> * [release-22.0] [CI] Use the draft state from the event payload instead of calling `curl`. (vitessio#18650) (vitessio#18652) Signed-off-by: Arthur Schreiber <[email protected]> Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com> Signed-off-by: Tanjin Xu <[email protected]> * [release-22.0] ci: extract os tuning (vitessio#18824) (vitessio#18826) Signed-off-by: Arthur Schreiber <[email protected]> Co-authored-by: Arthur Schreiber <[email protected]> Signed-off-by: Tanjin Xu <[email protected]> * [release-22.0] ci: extract os tuning (vitessio#18824) (vitessio#18826) Signed-off-by: Arthur Schreiber <[email protected]> Co-authored-by: Arthur Schreiber <[email protected]> Signed-off-by: Tanjin Xu <[email protected]> * [release-22.0] ci: DRY up MySQL Setup (vitessio#18815) (vitessio#18836) Signed-off-by: Arthur Schreiber <[email protected]> Co-authored-by: Arthur Schreiber <[email protected]> Signed-off-by: Tanjin Xu <[email protected]> * fix typo Signed-off-by: Tanjin Xu <[email protected]> * delete mysql57 tests Signed-off-by: Tanjin Xu <[email protected]> * setup secrets Signed-off-by: Tanjin Xu <[email protected]> * fix tests Signed-off-by: Tanjin Xu <[email protected]> * fix typo Signed-off-by: Tanjin Xu <[email protected]> * rm java docker test Signed-off-by: Tanjin Xu <[email protected]> --------- Signed-off-by: Arthur Schreiber <[email protected]> Signed-off-by: Tanjin Xu <[email protected]> Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com> Co-authored-by: Arthur Schreiber <[email protected]> Co-authored-by: Arthur Schreiber <[email protected]>
1 parent c1acf7a commit 12e9db0

File tree

105 files changed

+2055
-3540
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+2055
-3540
lines changed

‎.github/actions/setup-mysql/action.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: 'Setup MySQL'
2+
description: 'Set up MySQL server for use in GitHub Actions workflows'
3+
inputs:
4+
flavor:
5+
description: 'The flavor of MySQL to use (e.g. mysql-5.7, mysql-8.0, mysql-8.4)'
6+
required: true
7+
runs:
8+
using: "composite"
9+
steps:
10+
- name: Setup MySQL
11+
shell: bash
12+
run: |
13+
export DEBIAN_FRONTEND="noninteractive"
14+
sudo apt-get update
15+
16+
# Uninstall any previously installed MySQL first
17+
# sudo systemctl stop apparmor
18+
sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
19+
sudo apt-get -y autoremove
20+
sudo apt-get -y autoclean
21+
# sudo deluser mysql
22+
sudo rm -rf /var/lib/mysql
23+
sudo rm -rf /etc/mysql
24+
25+
# We have to install this old version of libaio1. See also:
26+
# https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
27+
wget http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb && \
28+
sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb && \
29+
rm libaio1_0.3.112-13build1_amd64.deb
30+
31+
# Get key to latest MySQL repo
32+
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
33+
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb
34+
35+
if [[ "${{ inputs.flavor }}" == "mysql-5.7" ]]; then
36+
# Bionic packages are still compatible for Jammy since there's no MySQL 5.7
37+
# packages for Jammy.
38+
echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections
39+
echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections
40+
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
41+
sudo apt-get update
42+
# libtinfo5 is also needed for older MySQL 5.7 builds.
43+
curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
44+
sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
45+
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6
46+
elif [[ "${{ inputs.flavor }}" == "mysql-8.0" ]]; then
47+
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
48+
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
49+
sudo apt-get update
50+
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
51+
elif [[ "${{ inputs.flavor }}" == "mysql-8.4" ]]; then
52+
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections
53+
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
54+
sudo apt-get update
55+
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
56+
else
57+
echo "Unsupported MySQL flavor: ${{ inputs.flavor }}"
58+
exit 1
59+
fi
60+
61+
sudo service mysql stop
62+
sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
63+
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
64+
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"

‎.github/actions/tune-os/action.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "Tune OS"
2+
description: "Tune the OS for better performance during CI runs"
3+
runs:
4+
using: "composite"
5+
steps:
6+
- name: Tune the OS
7+
shell: bash
8+
run: |
9+
# Install eatmydata and ensure it's used for every operation
10+
sudo apt-get update && sudo apt-get install -y eatmydata
11+
echo "/usr/\${LIB}/libeatmydata.so" | sudo tee -a /etc/ld.so.preload
12+
13+
# Increase the range of ephemeral ports.
14+
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
15+
16+
# Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
17+
echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
18+
sudo sysctl -p /etc/sysctl.conf
19+
20+
# Don't waste a bunch of time processing man-db triggers
21+
echo "set man-db/auto-update false" | sudo debconf-communicate
22+
sudo dpkg-reconfigure man-db

‎.github/workflows/check_make_vtadmin_authz_testgen.yml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,20 +48,18 @@ jobs:
4848
- 'go/vt/vtadmin/**'
4949
- '.github/workflows/check_make_vtadmin_authz_testgen.yml'
5050
51-
- name: Set up Go
52-
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
53-
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true'
54-
with:
55-
go-version-file: go.mod
56-
5751
- name: Setup GitHub access token
5852
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true'
5953
run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/
6054

6155
- name: Tune the OS
62-
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true'
63-
run: |
64-
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
56+
if: steps.changes.outputs.vtadmin_changes == 'true'
57+
uses: ./.github/actions/tune-os
58+
59+
- name: Set up Go
60+
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
61+
if: steps.changes.outputs.vtadmin_changes == 'true'
62+
uses: ./.github/actions/tune-os
6563

6664
- name: Get dependencies
6765
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true'

‎.github/workflows/check_make_vtadmin_web_proto.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ jobs:
3131
with:
3232
persist-credentials: 'false'
3333

34+
- name: Tune the OS
35+
if: steps.skip-workflow.outputs.skip-workflow == 'false'
36+
uses: ./.github/actions/tune-os
37+
3438
- name: Check for changes in relevant files
3539
if: steps.skip-workflow.outputs.skip-workflow == 'false'
3640
uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1

‎.github/workflows/cluster_endtoend_12.yml‎

Lines changed: 20 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,6 @@ jobs:
3838
echo Skip ${skip}
3939
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
4040
41-
if [[ "${{github.event.pull_request}}" != "" ]]; then
42-
PR_DATA=$(curl -s\
43-
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
44-
-H "Accept: application/vnd.github.v3+json" \
45-
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
46-
draft=$(echo "$PR_DATA" | jq .draft -r)
47-
echo "is_draft=${draft}" >> $GITHUB_OUTPUT
48-
fi
49-
5041
- name: Check out code
5142
if: steps.skip-workflow.outputs.skip-workflow == 'false'
5243
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -91,50 +82,34 @@ jobs:
9182
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
9283

9384
- name: Tune the OS
94-
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
95-
run: |
96-
# Limit local port range to not use ports that overlap with server side
97-
# ports that we listen on.
98-
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
99-
# Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
100-
echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
101-
sudo sysctl -p /etc/sysctl.conf
85+
if: steps.changes.outputs.end_to_end == 'true'
86+
uses: ./.github/actions/tune-os
10287

88+
- name: Setup MySQL
89+
if: steps.changes.outputs.end_to_end == 'true'
90+
uses: ./.github/actions/setup-mysql
91+
with:
92+
flavor: mysql-8.0
93+
10394
- name: Get dependencies
10495
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
10596
timeout-minutes: 10
10697
run: |
10798
108-
# Get key to latest MySQL repo
109-
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
110-
# Setup MySQL 8.0
111-
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb
112-
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
113-
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
114-
sudo apt-get -qq update
115-
116-
# We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also:
117-
# https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
118-
curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb
119-
sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb
120-
# libtinfo5 is also needed for older MySQL 5.7 builds.
121-
curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
122-
sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
99+
sudo apt-get -qq install -y mysql-shell
123100
124101
# Install everything else we need, and configure
125-
sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6
102+
sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6
126103
127-
sudo service mysql stop
128104
sudo service etcd stop
129-
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
130-
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
105+
131106
go mod download
132107
133108
# install JUnit report formatter
134109
go install github.com/vitessio/go-junit-report@HEAD
135110
136111
- name: Setup launchable dependencies
137-
if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
112+
if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
138113
run: |
139114
# Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
140115
pip3 install --user launchable~=1.0 > /dev/null
@@ -159,16 +134,17 @@ jobs:
159134
# Some of these tests require specific locales to be installed.
160135
# See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5
161136
# run the tests however you normally do, then produce a JUnit XML file
162-
eatmydata -- go run test.go -docker=false -follow -shard 12 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
137+
go run test.go -docker=false -follow -shard 12 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
163138
164-
- name: Print test output and Record test result in launchable if PR is not a draft
165-
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()
139+
- name: Record test results in launchable if PR is not a draft
140+
if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always()
166141
run: |
167-
if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then
168-
# send recorded tests to launchable
169-
launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
170-
fi
142+
# send recorded tests to launchable
143+
launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
171144
145+
- name: Print test output
146+
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()
147+
run: |
172148
# print test output
173149
cat output.txt
174150

‎.github/workflows/cluster_endtoend_13.yml‎

Lines changed: 20 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,6 @@ jobs:
3838
echo Skip ${skip}
3939
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
4040
41-
if [[ "${{github.event.pull_request}}" != "" ]]; then
42-
PR_DATA=$(curl -s\
43-
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
44-
-H "Accept: application/vnd.github.v3+json" \
45-
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
46-
draft=$(echo "$PR_DATA" | jq .draft -r)
47-
echo "is_draft=${draft}" >> $GITHUB_OUTPUT
48-
fi
49-
5041
- name: Check out code
5142
if: steps.skip-workflow.outputs.skip-workflow == 'false'
5243
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -91,50 +82,34 @@ jobs:
9182
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
9283

9384
- name: Tune the OS
94-
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
95-
run: |
96-
# Limit local port range to not use ports that overlap with server side
97-
# ports that we listen on.
98-
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
99-
# Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
100-
echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
101-
sudo sysctl -p /etc/sysctl.conf
85+
if: steps.changes.outputs.end_to_end == 'true'
86+
uses: ./.github/actions/tune-os
10287

88+
- name: Setup MySQL
89+
if: steps.changes.outputs.end_to_end == 'true'
90+
uses: ./.github/actions/setup-mysql
91+
with:
92+
flavor: mysql-8.0
93+
10394
- name: Get dependencies
10495
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
10596
timeout-minutes: 10
10697
run: |
10798
108-
# Get key to latest MySQL repo
109-
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
110-
# Setup MySQL 8.0
111-
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb
112-
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
113-
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
114-
sudo apt-get -qq update
115-
116-
# We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also:
117-
# https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
118-
curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb
119-
sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb
120-
# libtinfo5 is also needed for older MySQL 5.7 builds.
121-
curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
122-
sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
99+
sudo apt-get -qq install -y mysql-shell
123100
124101
# Install everything else we need, and configure
125-
sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6
102+
sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6
126103
127-
sudo service mysql stop
128104
sudo service etcd stop
129-
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
130-
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
105+
131106
go mod download
132107
133108
# install JUnit report formatter
134109
go install github.com/vitessio/go-junit-report@HEAD
135110
136111
- name: Setup launchable dependencies
137-
if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
112+
if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
138113
run: |
139114
# Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
140115
pip3 install --user launchable~=1.0 > /dev/null
@@ -159,16 +134,17 @@ jobs:
159134
# Some of these tests require specific locales to be installed.
160135
# See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5
161136
# run the tests however you normally do, then produce a JUnit XML file
162-
eatmydata -- go run test.go -docker=false -follow -shard 13 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
137+
go run test.go -docker=false -follow -shard 13 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
163138
164-
- name: Print test output and Record test result in launchable if PR is not a draft
165-
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()
139+
- name: Record test results in launchable if PR is not a draft
140+
if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always()
166141
run: |
167-
if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then
168-
# send recorded tests to launchable
169-
launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
170-
fi
142+
# send recorded tests to launchable
143+
launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
171144
145+
- name: Print test output
146+
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()
147+
run: |
172148
# print test output
173149
cat output.txt
174150

0 commit comments

Comments
 (0)