Skip to content

Commit cade7ae

Browse files
dennisbaderdennisbader
authored
Update CI/CD pipelines Part 1 (#2577)
* reduce tests to failing macos * retry * retry * retry * add back cache file * add comment * try different caching * fix bug * remove venv * add back codecov upload and ubuntu os * add linting, docs, and example checks * fix papermill missing * change to exmaples workdir for example checks * improve caching, update merge workflow * update merge workflow to use cache * add update cache step whenever a branch is merged to master
1 parent ad93612 commit cade7ae

File tree

3 files changed

+216
-172
lines changed

3 files changed

+216
-172
lines changed

‎.github/workflows/develop.yml‎

Lines changed: 86 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,23 @@ jobs:
99
lint:
1010
runs-on: ubuntu-latest
1111
steps:
12-
- name: "1. Clone repository"
13-
uses: actions/checkout@v2
12+
- name: "Clone repository"
13+
uses: actions/checkout@v4
1414

15-
- name: "2. Set up Python 3.9"
16-
uses: actions/setup-python@v1
15+
- name: "Set up Python 3.9"
16+
uses: actions/setup-python@v5
1717
with:
1818
python-version: '3.9'
1919

20-
# downloading gradle multiple times in parallel can yield to connection errors
21-
- name: "3. Cache gradle distribution"
22-
uses: actions/cache@v2
23-
with:
24-
path: ~/.gradle/wrapper/dists
25-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
26-
27-
- name: "3.1 Cache gradle packages"
28-
uses: actions/cache@v2
29-
with:
30-
path: ~/.gradle/caches
31-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
20+
- name: "Install Dev Dependencies"
21+
run: |
22+
python -m pip install --upgrade pip
23+
pip install -r requirements/dev.txt
3224
33-
- name: "4. Lint"
25+
- name: "Lint"
3426
run: |
35-
./gradlew lint
27+
pre-commit run --all-files
28+
3629
3730
tests:
3831
runs-on: ${{ matrix.os }}
@@ -43,47 +36,39 @@ jobs:
4336
flavour: ['all']
4437

4538
steps:
46-
- name: "1. Clone repository"
47-
uses: actions/checkout@v2
39+
- name: "Clone repository"
40+
uses: actions/checkout@v4
4841

49-
- name: "2. Set up Python ${{ matrix.python-version }}"
50-
uses: actions/setup-python@v1
42+
- name: "Set up Python ${{ matrix.python-version }}"
43+
uses: actions/setup-python@v5
5144
with:
5245
python-version: ${{ matrix.python-version }}
5346

54-
# downloading gradle multiple times in parallel can yield to connection errors
55-
- name: "3. Cache gradle distribution"
56-
uses: actions/cache@v2
47+
- name: "Cache python environment"
48+
uses: actions/cache@v4
49+
id: pythonenv-cache
5750
with:
58-
path: ~/.gradle/wrapper/dists
59-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
51+
path: ${{ env.pythonLocation }}
52+
key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
6053

61-
- name: "3.1 Cache gradle packages"
62-
uses: actions/cache@v2
63-
with:
64-
path: ~/.gradle/caches
65-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
54+
- name: "Setup Pip"
55+
run: |
56+
python -m pip install --upgrade pip
6657
67-
- name: "4. Setup pip"
58+
- name: "Install Dependencies"
6859
run: |
69-
./gradlew installPipLatest
60+
# install latest dependencies (potentially updating cached dependencies)
61+
pip install -U -r requirements/dev-all.txt
7062
71-
- name: "5. Install libomp (for LightGBM)"
63+
- name: "Install libomp (for LightGBM)"
7264
run: |
7365
./.github/scripts/libomp-${{ runner.os }}.sh
7466
75-
- name: "6. Attach cache for pip"
76-
uses: actions/cache@v1
77-
id: cache
78-
with:
79-
path: ~/.cache/pip
80-
key: tests-${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements-latest.txt') }}
81-
82-
- name: "7. Tests"
67+
- name: "Run tests"
8368
run: |
84-
./gradlew "test_${{matrix.flavour}}"
69+
pytest --durations=50 --cov=darts --cov-config=.coveragerc --cov-report=xml darts/tests
8570
86-
- name: "8. Codecov upload"
71+
- name: "Codecov upload"
8772
if: ${{ matrix.flavour == 'all' }}
8873
uses: codecov/codecov-action@v4
8974
with:
@@ -93,73 +78,87 @@ jobs:
9378
docs:
9479
runs-on: ubuntu-latest
9580
steps:
96-
- name: "1. Clone repository"
97-
uses: actions/checkout@v2
81+
- name: "Clone repository"
82+
uses: actions/checkout@v4
9883

99-
- name: "2. Set up Python 3.9"
100-
uses: actions/setup-python@v1
84+
- name: "Set up Python 3.9"
85+
uses: actions/setup-python@v5
10186
with:
10287
python-version: '3.9'
10388

104-
- name: "3. Install pandoc"
89+
# only restore cache but do not upload
90+
- name: "Restore cached python environment"
91+
uses: actions/cache/restore@v4
92+
id: pythonenv-cache
93+
with:
94+
path: ${{ env.pythonLocation }}
95+
key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
96+
97+
- name: "Install pandoc"
10598
run: |
10699
sudo apt-get install -y pandoc
107100
108-
# downloading gradle multiple times in parallel can yield to connection errors
109-
- name: "4. Cache gradle distribution"
110-
uses: actions/cache@v2
111-
with:
112-
path: ~/.gradle/wrapper/dists
113-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
101+
- name: "Setup Pip"
102+
run: |
103+
python -m pip install --upgrade pip
114104
115-
- name: "4.1 Cache gradle packages"
116-
uses: actions/cache@v2
117-
with:
118-
path: ~/.gradle/caches
119-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
105+
- name: "Install Dependencies"
106+
run: |
107+
# install latest dependencies (potentially updating cached dependencies)
108+
pip install -U -r requirements/dev-all.txt
120109
121-
- name: "5. Setup pip"
110+
- name: "Install libomp (for LightGBM)"
122111
run: |
123-
./gradlew setupPip
112+
./.github/scripts/libomp-${{ runner.os }}.sh
124113
125-
- name: "6. Attach cache for pip"
126-
uses: actions/cache@v1
127-
id: cache
128-
with:
129-
path: ~/.cache/pip
130-
key: tests-${{ runner.os }}-3.9-pip-${{ hashFiles('requirements/core.txt', 'requirements/release.txt') }}
114+
- name: "Install Locally"
115+
run: |
116+
pip install .
131117
132-
- name: "7. Build docs"
118+
- name: "Build docs"
133119
run: |
134-
./gradlew buildDocs
120+
make --directory ./docs build-all-docs
135121
136122
check-examples:
137123
runs-on: ubuntu-latest
138124
strategy:
139125
matrix:
140126
example-name: [03-FFT-examples.ipynb, 04-RNN-examples.ipynb, 00-quickstart.ipynb, 02-data-processing.ipynb, 01-multi-time-series-and-covariates.ipynb]
141127
steps:
142-
- name: "1. Clone repository"
143-
uses: actions/checkout@v2
128+
- name: "Clone repository"
129+
uses: actions/checkout@v4
144130

145-
- name: "2. Set up Python 3.9"
146-
uses: actions/setup-python@v1
131+
- name: "Set up Python 3.9"
132+
uses: actions/setup-python@v5
147133
with:
148134
python-version: '3.9'
149135

150-
# downloading gradle multiple times in parallel can yield to connection errors
151-
- name: "3. Cache gradle distribution"
152-
uses: actions/cache@v2
136+
# only restore cache but do not upload
137+
- name: "Restore cached python environment"
138+
uses: actions/cache/restore@v4
139+
id: pythonenv-cache
153140
with:
154-
path: ~/.gradle/wrapper/dists
155-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
141+
path: ${{ env.pythonLocation }}
142+
key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
156143

157-
- name: "3.1 Cache gradle packages"
158-
uses: actions/cache@v2
159-
with:
160-
path: ~/.gradle/caches
161-
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
144+
- name: "Setup Pip"
145+
run: |
146+
python -m pip install --upgrade pip
147+
148+
- name: "Install Dependencies"
149+
run: |
150+
# install latest dependencies (potentially updating cached dependencies)
151+
pip install -U -r requirements/dev-all.txt
152+
153+
- name: "Install libomp (for LightGBM)"
154+
run: |
155+
./.github/scripts/libomp-${{ runner.os }}.sh
156+
157+
- name: "Install Locally"
158+
run: |
159+
pip install .
162160
163-
- name: "4. Run examples ${{matrix.example-name}}"
161+
- name: "Run example ${{matrix.example-name}}"
162+
working-directory: ./examples
164163
run: |
165-
./gradlew checkExample -PexampleName=${{matrix.example-name}}
164+
papermill ${{matrix.example-name}} ${{matrix.example-name}}

0 commit comments

Comments
 (0)