Skip to content

Commit b25a180

Browse files
author
Ryan Faircloth
authored
Merge pull request #16 from splunk/test/common-template-rollout-changes
Deprecate splunk_env_indexer submodule
2 parents 5cc89c4 + 6dac3bd commit b25a180

File tree

11 files changed

+349
-17
lines changed

11 files changed

+349
-17
lines changed

.circleci/config.yml

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,9 @@ jobs:
297297
docker cp project_splunk_1:${diag//[$'\t\r\n']} /tmp/diag
298298
# Check with btool
299299
docker-compose -f docker-compose-ci.yml exec splunk sudo /opt/splunk/bin/splunk cmd btool check
300-
docker-compose -f docker-compose-ci.yml up -d splunk
300+
docker-compose -f docker-compose-ci.yml up -d splunk uf
301301
until docker-compose -f docker-compose-ci.yml logs splunk | grep "Ansible playbook complete" ; do sleep 1; done
302+
until docker-compose -f docker-compose-ci.yml logs uf | grep "Ansible playbook complete" ; do sleep 1; done
302303
303304
# Normal Inspecting
304305
[ -z $BROWSER ] || [ "$UI_TEST_HEADLESS" = "true" ] || docker-compose -f docker-compose-ci.yml logs sauceconnect
@@ -491,9 +492,47 @@ jobs:
491492
- store_test_results:
492493
path: test-results
493494

495+
semgrep:
496+
environment:
497+
SEMGREP_REPO_URL: << pipeline.project.git_url >>
498+
SEMGREP_BRANCH: << pipeline.git.branch >>
499+
docker:
500+
- image: returntocorp/semgrep-agent:v1
501+
user: root
502+
steps:
503+
- checkout
504+
- run:
505+
name: "Install Dependencies"
506+
command: |
507+
pip3 install --upgrade semgrep
508+
- run:
509+
name: "Semgrep Scan"
510+
no_output_timeout: 1h
511+
command: |
512+
export SEMGREP_REPO_NAME=splunk/${CIRCLE_PROJECT_REPONAME}
513+
python -m semgrep_agent --publish-deployment ${SEMGREP_DEPLOYMENT_ID} --publish-token ${SEMGREP_PUBLISH_TOKEN}
514+
515+
# Generate the Semgrep Dashboard URL
516+
export REPO_BRANCH=$(echo "<< pipeline.git.branch >>")
517+
DASHBOARD_URL=$(python3 -c "from urllib.parse import quote; import os; print('https://semgrep.dev/manage/findings?repo=' + quote(os.environ['SEMGREP_REPO_NAME'], safe='') + '&tab=findings&ref_type=branch&ref=' + quote(os.environ['REPO_BRANCH'], safe=''))")
518+
echo "View Result at Semgrep Dashboard: $DASHBOARD_URL"
519+
520+
# Semgrep Exclude files
521+
dos2unix .semgrepignore
522+
SEMGREP_EXCLUDE=$(sed "/^#/d" .semgrepignore | sed "/^:/d" | sed -r '/^\s*$/d' | sed ':a;N;$!ba;s/\n/ --exclude /g')
523+
echo "Excluding Semgrep Files: --exclude $SEMGREP_EXCLUDE"
524+
525+
# Generate xml file
526+
semgrep --config="p/r2c-ci" --config="p/r2c-security-audit" --config="p/bandit" --error --strict --timeout=0 --junit-xml -o /root/project/test-results/semgrep-scan.xml --exclude $SEMGREP_EXCLUDE
527+
- store_artifacts:
528+
path: test-results
529+
destination: test-results
530+
- store_test_results:
531+
path: test-results
532+
494533
release:
495534
docker:
496-
- image: circleci/node:11
535+
- image: circleci/node:12
497536
steps:
498537
- setup_remote_docker:
499538
docker_layer_caching: true
@@ -545,7 +584,6 @@ jobs:
545584
PACKAGE_ID=$(crudini --get package/default/app.conf id name)
546585
[[ << pipeline.git.tag >> =~ ^v[0-9]*.[0-9]*.[0-9]*$ ]] || export ISPRE=-prerelease
547586
[ "${ISPRE}" == "-prerelease" ] && SPLUNKBASE_VIS="false" || SPLUNKBASE_VIS="true"
548-
export SPLUNKBASE_SPLUNK_VERSION=$(echo << pipeline.git.tag >> | sed 's/^v\(.*\)/\1/')
549587
curl -u ${SPLUNKBASE_USERNAME}:${SPLUNKBASE_PASSWORD} --request POST https://splunkbase.splunk.com/api/v1/app/${SPLUNKBASE_ID}/new_release/ -F "files[]=@${PACKAGE}" -F "filename=${PACKAGE_ID}.spl" -F "splunk_versions=${SPLUNKBASE_SPLUNK_VERSION}" -F "visibility=${SPLUNKBASE_VIS}"
550588
reuse:
551589
docker:
@@ -570,6 +608,12 @@ jobs:
570608
workflows:
571609
build:
572610
jobs:
611+
- semgrep:
612+
context:
613+
- gdi-semgrep
614+
filters:
615+
branches:
616+
only: /.*/
573617
- reuse:
574618
filters:
575619
branches:
@@ -708,4 +752,4 @@ workflows:
708752
branches:
709753
ignore: /.*/
710754
tags:
711-
only: /^v\d*\.\d*\.\d*.*$/
755+
only: /^v\d*\.\d*\.\d*$/

.gitmodules

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,3 @@
44
[submodule "deps/apps/Splunk_SA_CIM"]
55
path = deps/apps/Splunk_SA_CIM
66
url = [email protected]:splunk/addonfactory-splunk_sa_cim.git
7-
[submodule "deps/apps/splunk_env_indexer"]
8-
path = deps/apps/splunk_env_indexer
9-
url = [email protected]:splunk/addonfactory-splunk_env_indexer.git

.reuse/dep5

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,6 @@ Files: semtag
4949
Copyright: $YEAR $NAME <$CONTACT>
5050
License: LicenseRef-Splunk-1-2020
5151

52-
Files: README.*
52+
Files: README*
5353
Copyright: $YEAR $NAME <$CONTACT>
5454
License: LicenseRef-Splunk-1-2020

.semgrepignore

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
## Default semgrep ignore
2+
# Ignore git items
3+
.gitignore
4+
.git/
5+
:include .gitignore
6+
7+
# Common large directories
8+
node_modules/
9+
build/
10+
dist/
11+
vendor/
12+
env/
13+
.env/
14+
venv/
15+
.venv/
16+
*.min.js
17+
18+
# Common test directories
19+
test/
20+
tests/
21+
22+
# Semgrep rules folder
23+
.semgrep
24+
25+
## Additional files to be ignored
26+
.circleci/
27+
.github/
28+
.reuse/
29+
.vscode/

Dockerfile-splunk-local

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
ARG SPLUNK_VERSION=latest
2+
FROM splunk/splunk:$SPLUNK_VERSION
3+
ARG SPLUNK_APP_ID=TA_UNKNOWN
4+
ARG SPLUNK_APP_PACKAGE=$SPLUNK_APP_PACKAGE
5+
RUN echo Splunk VERSION=$SPLUNK_VERSION
6+
7+
COPY deps/apps /opt/splunk/etc/apps/
8+
COPY deps/build/addonfactory_test_matrix_splunk/packages/all/common /opt/splunk/etc/apps/
9+
COPY deps/build/addonfactory_test_matrix_splunk/packages/all/sh /opt/splunk/etc/apps/
10+
COPY deps/build/addonfactory_test_matrix_splunk/packages/all/addon_factory_web /opt/splunk/etc/system/local
11+
COPY $SPLUNK_APP_PACKAGE /opt/splunk/etc/apps/$SPLUNK_APP_ID
12+
13+
COPY .pytest.expect /home/circleci/work_backend/.pytest.expect
14+
COPY tests /home/circleci/work_backend/tests

Dockerfile-uf

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
ARG SPLUNK_VERSION=latest
2+
FROM splunk/universalforwarder:$SPLUNK_VERSION
3+
ARG SPLUNK_VERSION=latest
4+
ARG SPLUNK_APP_ID=TA_UNKNOWN
5+
ARG SPLUNK_APP_PACKAGE=$SPLUNK_APP_PACKAGE
6+
RUN echo Splunk VERSION=$SPLUNK_VERSION
7+
8+
COPY $SPLUNK_APP_PACKAGE /opt/splunkforwarder/etc/apps/$SPLUNK_APP_ID

README-test.md

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
2+
# Run Addon tests in local environment
3+
4+
## With Docker
5+
6+
### Prerequisitory:
7+
- Git
8+
- Python3 (>=3.7)
9+
- Python2
10+
- crudini
11+
- Docker
12+
- Docker-compose
13+
14+
### Steps:
15+
16+
1. Clone the repository
17+
```bash
18+
git clone [email protected]:splunk/<repo name>.git
19+
cd <repo dir>
20+
git submodule update --init --recursive
21+
```
22+
23+
2. Install Requirements and Generate Addon
24+
```bash
25+
pip3 install -r requirements_dev.txt
26+
ucc-gen
27+
28+
# Execute only if TEST_TYPE is modinput_functional or modinput_others
29+
curl -s https://api.github.com/repos/splunk/splunk-add-on-for-modinput-test/releases/latest | grep "Splunk_TA.*tar.gz" | grep -v search_head | grep -v indexer | grep -v forwarder | cut -d : -f 2,3 | tr -d \" | wget -qi -; tar -xvzf *.tar.gz -C deps/apps/
30+
```
31+
32+
3. Set Variables
33+
```bash
34+
export SPLUNK_VERSION=<splunk_version> [i.e. latest, 8.1.0]
35+
export SPLUNK_APP_ID=$(crudini --get package/default/app.conf id name) [i.e. Splunk_TA_addon-name]
36+
export SPLUNK_APP_PACKAGE=output/$(ls output/) [i.e. output/Splunk_TA_addon-name]
37+
export TEST_TYPE=<knowledge|ui|modinput_functional|modinput_others>
38+
export TEST_SET=tests/$TEST_TYPE [i.e. tests/knowledge]
39+
export IMAGE_TAG="3.7-browsers"
40+
export SC4S_VERSION=<sc4s_version> [i.e. latest, 1.51.0]
41+
42+
# If TEST_TYPE is ui also set the following variables
43+
export TEST_BROWSER=<browser_name> [i.e. chrome, firefox]
44+
export JOB_NAME=<LocalRun::[addon_name]-[browser]>
45+
export SAUCE_USERNAME=<sauce_username>
46+
export SAUCE_PASSWORD=<sauce_password>
47+
export SAUCE_IDENTIFIER=$SAUCE_IDENTIFIER-$(cat /proc/sys/kernel/random/uuid)
48+
export UI_TEST_HEADLESS="true"
49+
```
50+
**Note:** If TEST_TYPE is `modinput_functional`, `modinput_others` or `ui`, also set all variables in [test_credentials.env](test_credentials.env) file with appropriate values encoded with base64.
51+
52+
4. Docker Build and test execution:
53+
```bash
54+
docker-compose -f docker-compose.yml build
55+
56+
# Execute only if TEST_TYPE is ui
57+
[ -z $BROWSER ] || [ "$UI_TEST_HEADLESS" = "true" ] || docker-compose -f docker-compose.yml up -d sauceconnect
58+
59+
docker-compose -f docker-compose.yml up -d splunk
60+
until docker-compose -f docker-compose.yml logs splunk | grep "Ansible playbook complete" ; do sleep 1; done
61+
docker-compose -f docker-compose.yml up --abort-on-container-exit test
62+
```
63+
64+
65+
## With External
66+
67+
### Prerequisitory:
68+
- Git
69+
- Python3 (>=3.7)
70+
- Python2
71+
- Splunk along with addon installed and HEC token created
72+
- If Addon support the syslog data ingestion(sc4s)
73+
- Docker
74+
- Docker-compose
75+
76+
### Steps:
77+
78+
1. Clone the repository
79+
```bash
80+
git clone [email protected]:splunk/<repo name>.git
81+
cd <repo dir>
82+
git submodule update --init --recursive
83+
```
84+
85+
2. Install Requirements
86+
```bash
87+
pip3 install -r requirements_dev.txt
88+
```
89+
90+
3. Set Variables (only if Addon supports sc4s)
91+
92+
**Note:** Stop the existing splunk if it is running. New splunk will up after executing following steps with `Changed@11` splunk password.
93+
```bash
94+
export SPLUNK_VERSION=<splunk_version> [i.e. latest, 8.1.0]
95+
export SPLUNK_APP_ID=<Addon_id> [i.e. Splunk_TA_addon-name]
96+
export SPLUNK_APP_PACKAGE=<splunk_package>
97+
export IMAGE_TAG="3.7-browsers"
98+
export SC4S_VERSION=<sc4s_version>
99+
100+
docker-compose -f docker-compose.yml build
101+
docker-compose -f docker-compose.yml up -d splunk
102+
```
103+
104+
4. Run Tests
105+
106+
- Knowledge
107+
108+
```bash
109+
pytest -vv --splunk-type=external --splunk-app=<path-to-addon-package> --splunk-data-generator=<path to pytest-splunk-addon-data.conf file> --splunk-host=<hostname> --splunk-port=<splunk-management-port> --splunk-user=<username> --splunk-password=<password> --splunk-hec-token=<splunk_hec_token> --sc4s-host=<sc4s_host> --sc4s-port=<sc4s_port>
110+
```
111+
112+
- UI:
113+
114+
1. Set all variables in environment mentioned at [test_credentials.env](test_credentials.env) file with appropriate values encoded with base64.
115+
2. Download Browser's specific driver
116+
- For Chrome: download chromedriver
117+
- For Firefox: download geckodriver
118+
- For IE: download IEdriverserver
119+
3. Put the downloaded driver into `test/ui/` directory, make sure that it is within the environment's PATH variable, and that it is executable
120+
4. For Internet explorer, The steps mentioned at below link must be performed:
121+
https://github.com/SeleniumHQ/selenium/wiki/InternetExplorerDriver#required-configuration
122+
5. Execute the test cases:
123+
```script
124+
pytest -vv --browser=<browser> --local --splunk-host=<web_url> --splunk-port=<mgmt_url> --splunk-user=<username> --splunk-password=<password>
125+
```
126+
127+
- Modinput:
128+
129+
Install [splunk-add-on-for-modinput-test](https://github.com/splunk/splunk-add-on-for-modinput-test/releases/latest/) addon in splunk and set all variables in environment mentioned at [test_credentials.env](test_credentials.env) file with appropriate values encoded with base64 or add variables in pytest command mentioned in conftest file.
130+
```bash
131+
pytest -vv --username=<splunk_username> --password=<splunk_password> --splunk-url=<splunk_url> --remote
132+
```

deps/apps/splunk_env_indexer

Lines changed: 0 additions & 1 deletion
This file was deleted.

docker-compose-ci.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,25 @@ services:
144144
- PROXY_PASSWORD=123456
145145
tty: true
146146

147+
uf:
148+
build:
149+
context: .
150+
dockerfile: Dockerfile-uf
151+
args:
152+
SPLUNK_APP_ID: ${SPLUNK_APP_ID}
153+
SPLUNK_APP_PACKAGE: ${SPLUNK_APP_PACKAGE}
154+
SPLUNK_VERSION: ${SPLUNK_VERSION}
155+
hostname: uf
156+
ports:
157+
- "8089"
158+
links:
159+
- splunk
160+
environment:
161+
- SPLUNK_PASSWORD=Chang3d!
162+
- SPLUNK_START_ARGS=--accept-license
163+
volumes:
164+
- results:/home/circleci/work
165+
147166
volumes:
148167
results:
149168
external: false

0 commit comments

Comments
 (0)