chore: running with initial ansible changes

.github/workflows/ci-env-images.yaml

@@ -0,0 +1,81 @@
+name: Build and Publish CI Environment Images
+
+on:
+  pull_request:
+    branches:
+      - develop
+    paths:
+      - 'tests/container_images/docker/ubuntu_24/Dockerfile'
+      - 'tests/container_images/docker/redhat_9/Dockerfile'
+
+
+jobs:
+  build-and-push-ci-images:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write # Required to push to GHCR
+
+    strategy:
+      fail-fast: false # Set to false to build all images even if one fails
+      matrix:
+        os_variant: [ubuntu, rhel]
+        include:
+          - os_variant: ubuntu
+            dockerfile_path: tests/container_images/docker/ubuntu_24/Dockerfile
+            image_name_suffix: ci-env-ubuntu
+            # Build args specific to Ubuntu
+            ansible_version: "6.1.0"
+            ansible_core_version: "2.13.13"
+            poetry_installer_version: "1.5.1"
+          - os_variant: rhel
+            dockerfile_path: tests/container_images/docker/redhat_9/Dockerfile
+            image_name_suffix: ci-env-rhel
+            # Build args specific to RHEL
+            ansible_version: "6.1.0"
+            ansible_core_version: "2.13.13"
+            poetry_installer_version: "1.5.1"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }} # Your Docker Hub username
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.image_name_suffix }}
+          tags: |
+            type=raw,value=latest
+
+      - name: Build and push Docker image for ${{ matrix.os_variant }}
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ matrix.dockerfile_path }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          secrets: |
+            pipconf=${{ secrets.PIP_CONF_CONTENT }} # Pass pip.conf content as a secret
+          build-args: |
+            ANSIBLE_VERSION=${{ matrix.ansible_version }}
+            ANSIBLE_CORE_VERSION=${{ matrix.ansible_core_version }}
+            POETRY_INSTALLER_VERSION=${{ matrix.poetry_installer_version }}

.github/workflows/ci-main.yaml

@@ -178,14 +178,13 @@ jobs:
           scanners: "vuln"
 
   test-container:
+    strategy:
+      matrix: 
+        deployment_type: ["docker", "podman"]
     runs-on: ubuntu-latest
     needs:
       - meta
       - build_action
-    # runs all of the steps inside the specified container rather than on the VM host.
-    # Because of this the network configuration changes from host based network to a container network.
-    container:
-      image: python:3.9-buster
 
     services:
       splunk:
@@ -200,42 +199,64 @@ jobs:
           SPLUNK_START_ARGS: --accept-license
           SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz
 
-      sc4s:
-        image: ${{ needs.meta.outputs.container_base }}
-        ports:
-          - 514:514
-          - 601:601
-          - 5614:5514
-          - 5601:5601
-          - 6000:6000
-          - 6002:6002
-          - 9000:9000
-        env:
-          SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088
-          SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630
-          SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no"
-          SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes"
-          SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000
-          SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514
-          SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514
-          SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601
-          SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002
-          SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000
-          SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001
-          SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes"
-          SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes"
-          TEST_SC4S_ACTIVATE_EXAMPLES: "yes"
-          SC4S_DEBUG_CONTAINER: "yes"
-          SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes"
-          SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes"
-          SC4S_USE_VPS_CACHE: "yes"
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: false
           persist-credentials: false
+      - name: Install Ansible and other dependencies as python package
+        run: |
+          pip install ansible~=6.1.0 --no-cache-dir \
+          && pip install pywinrm>=0.4.2 --no-cache-dir \
+          && pip install ansible-lint>=6.0.0 --no-cache-dir \
+          && pip install docker
+      - name: Configure Ansible Environment Variables
+        env:
+          ANSIBLE_CONFIG: ansible.cfg
+          ANSIBLE_HOST_KEY_CHECKING: False
+        run: |
+          echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG"
+          echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING"
+      - name: Update inventory file
+        run: |
+          cat << EOF > ansible/inventory/inventory.yaml
+          ---
+          all:
+            hosts:
+            children:
+              node:
+                hosts:
+                  node_1:
+                    ansible_host: 127.0.0.1
+                    ansible_connection: local
+                    ansible_user: root
+      - name: Update env_file
+        run: |
+          echo "Updating ansible/inventory/inventory.yaml"
+          cat << EOF > ansible/resources/env_file
+          SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088
+          SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630
+          SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no
+
+      - name: Debug
+        run: |
+          whoami
+          uname -a
+          cat /etc/os-release
+          cat ansible/inventory/inventory.yaml
+          cat ansible/resources/env_file
+          docker ps
+      - name: Run Ansible Playbook
+        run: |
+          ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml 
+      - name: Debug 2
+        run: |
+          whoami
+          uname -a
+          docker ps
+          systemctl status sc4s
+          docker images
       - name: Run tests
         run: |
           pip3 install poetry
@@ -244,11 +265,17 @@ jobs:
           poetry run pytest -v --tb=long \
             --splunk_type=external \
             --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \
-            --splunk_host=splunk \
-            --sc4s_host=sc4s \
+            --splunk_host=127.0.0.1 \
+            --sc4s_host=127.0.0.1 \
             --junitxml=test-results/test.xml \
             -n 14 \
             -k 'not lite and not name_cache'
+      - name: artifact-test-results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-xml
+          path: test-results/test.xml
+        if: ${{ !cancelled() }}
 
   test-ipv4-name-cache:
     runs-on: ubuntu-latest
@@ -462,4 +489,4 @@ jobs:
             @google/[email protected]
             [email protected]
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }}

ansible/playbooks/docker.yml

@@ -4,6 +4,10 @@
   vars:
     iface: "{{ swarm_iface | default('eth0') }}"
   tasks:
+    - name: Check if 'docker' command exists on target (if not using docker connection)
+      ansible.builtin.command: which docker
+      register: docker_command_check
+      ignore_errors: true
     - name: Docker installation role
       include_role:
         name: install_docker

ansible/resources/env_file

@@ -1,4 +1,4 @@
-SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://xxx.xxx.xxx.xxx:8088
+SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://xxx.xxx.xxx.xxx:8088
 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=xxxxxxxxxxxxxxxxxx
 #Uncomment the following line if using untrusted SSL certificates
 #SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no

ansible/roles/install_docker/tasks/install_docker_debian.yml

@@ -23,10 +23,16 @@
     repo: deb https://download.docker.com/linux/ubuntu focal stable
     state: present
 
+- name: Check if 'docker' command exists
+  ansible.builtin.command: which docker
+  register: docker_command_check
+  ignore_errors: true
+
 - name: Install docker-ce
   apt:
     name: docker-ce
     state: latest
+  when: docker_command_check.rc != 0
 
 - name: Install Docker Module for Python
   pip:

ansible/roles/install_docker/tasks/install_docker_rhel.yml

@@ -28,11 +28,11 @@
     state: latest
     update_cache: true
 
-- name: Install Docker Module for Python
-  pip:
-    name:
-      - docker
-      - jsondiff
+# - name: Install Docker Module for Python
+#   pip:
+#     name:
+#       - docker
+#       - jsondiff
 
 - name: Start Docker
   ansible.builtin.service:

ansible/tasks/docker/deploy_app.yml

@@ -12,15 +12,15 @@
 
 - name: Copying unit file on the server
   copy:
-    src: "/opt/ansible/resources/docker_sc4s.service"
+    src: "../../../ansible/resources/docker_sc4s.service"
     dest: "/lib/systemd/system/sc4s.service"
     owner: "{{ ansible_user }}"
     group: "{{ ansible_user }}"
     mode: u=rw,g=rw
 
 - name: Copying env_file file on the server
   copy:
-    src: "/opt/ansible/resources/env_file"
+    src: "../../../ansible/resources/env_file"
     dest: "/opt/sc4s/env_file"
     owner: "{{ ansible_user }}"
     group: "{{ ansible_user }}"

ansible/tasks/podman/deploy_app.yml

@@ -12,15 +12,15 @@
 
 - name: Copying unit file on the server
   copy:
-    src: "/opt/ansible/resources/podman_sc4s.service"
+    src: "../../../ansible/resources/podman_sc4s.service"
     dest: "/lib/systemd/system/sc4s.service"
     owner: "{{ ansible_user }}"
     group: "{{ ansible_user }}"
     mode: u=rw,g=rw
 
 - name: Copying env_file file on the server
   copy:
-    src: "/opt/ansible/resources/env_file"
+    src: "../../../ansible/resources/env_file"
     dest: "/opt/sc4s/env_file"
     owner: "{{ ansible_user }}"
     group: "{{ ansible_user }}"

docs/gettingstarted/ansible-docker-podman.md

@@ -23,16 +23,16 @@ docker exec -it ansible_sc4s /bin/bash
 
 * To authenticate with username and password:
 ``` bash 
-ansible-playbook -i path/to/inventory.yaml -u <username> --ask-pass path/to/playbooks/docker.yml
+ansible-playbook -i ansible/inventory/inventory.yaml -u <username> --ask-pass ansible/playbooks/playbooks/docker.yml
 or
-ansible-playbook -i path/to/inventory.yaml -u <username> --ask-pass path/to/playbooks/podman.yml
+ansible-playbook -i ansible/inventory/inventory.yaml -u <username> --ask-pass ansible/playbooks/playbooks/podman.yml
 
 ```
 * To authenticate using a key pair:
 ``` bash 
-ansible-playbook -i path/to/inventory.yaml -u <username> --key-file <key_file> path/to/playbooks/docker.yml
+ansible-playbook -i ansible/inventory/inventory.yaml -u <username> --key-file <key_file> ansible/playbooks/playbooks/docker.yml
 or
-ansible-playbook -i path/to/inventory.yaml -u <username> --key-file <key_file> path/to/playbooks/podman.yml
+ansible-playbook -i ansible/inventory/inventory.yaml -u <username> --key-file <key_file> ansible/playbooks/playbooks/podman.yml
 ```
 
 # Step 3: Validate your configuration

docs/gettingstarted/ansible-docker-swarm.md

@@ -51,7 +51,7 @@ ansible-playbook -i path/to/inventory_swarm.yaml -u <username> --key-file <key_f
 |sc4s    | 1        | Swarm        |
 
 * To scale your number of services:
-```sudo docker service update --replicas 2 sc4s_sc4s```
+```sudo docker service update --replicas 2 SC4S_sc4s```
 
 * To see services running in a given stack: 
 ```sudo docker stack services sc4s```