Add command line tool to run tests (#1)

* Add command line tool to run tests

* Fix errors & warnings in tests

* Support filter option by test names

* ops: setup pipeline to analyze code

* ops: fix branch name

* ops: fix pipeline

* Terraform fmt

* ops: added missing step

* ops: addedstep to download modules

* Revert "ops: addedstep to download modules"

Author: arnaud-tincelin

.github/workflows/PolicyTesting.yml

@@ -0,0 +1,41 @@
+name: 'Analysis'
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+jobs:
+  lint:
+    name: Check linter
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v1
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Terraform fmt
+      run: terraform fmt -check -recursive
+    - name: Golang lint
+      uses: golangci/golangci-lint-action@v2
+  codeql:
+    name: CodeQL
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: go
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/analysis.yml

@@ -2,11 +2,46 @@
 
 This repository is a command line tool for testing Azure Policies.
 
-This repository is organized as follow:
+## Usage
 
-- custom_policies : contains 1 folder (1 [terraform Module](https://www.terraform.io/docs/configuration/modules.html)) per custom Azure Policy. 
-- prod_policies : this folder contains the terraform configuration applied on Azure environments to match security requirements.
-- test : for each custom Azure Policy, automated tests are located here.
+For each test case, the following directory structure must be created:
+
+```txt
++-- your_test_name
+|   +-- setup
+|   |   +-- terraform files to setup the policy
+|   +-- terraform files for test cases
+```
+
+Additionally, a `.yaml` configuration file must describe the test as following:
+
+```yaml
+name: Name of the test
+cases:
+- variables:
+  - key: variable name in TF
+    value: val
+  errorExpected: true
+- variables:
+  - key: variable name in TF
+    value: val2
+  errorExpected: false
+terraformDir: relative path to the folder structure described above
+errorMessage: Error message from Azure (ex 'Error creating Network Interface')
+errorCode: Error code from Azure (ex 'Error Code=\"RequestDisallowedByPolicy\"')
+```
+
+To run from the source code:
+
+```bash
+go run ./cmd/policyTester/ -config ./test/ -test.v
+```
+
+Note: any args from the `go test` command are available for use. For instance:
+
+```bash
+go run ./cmd/policyTester/ -config ./test/ -test.v -test.parallel=10 -test.run Location
+```
 
 ## About Azure Policies
 

README.md

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/microsoft/AzurePolicyTestFramework/pkg/tester"
+)
+
+var (
+	cases *string
+)
+
+func init() {
+	cases = flag.String("config", "", "path to the test cases")
+}
+
+func main() {
+	testing.Init()
+	flag.Parse()
+
+	code, err := tester.RunGoTest(*cases)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Exit(code)
+}

cmd/policyTester/main.go

@@ -0,0 +1,53 @@
+module github.com/microsoft/AzurePolicyTestFramework
+
+go 1.17
+
+require (
+	github.com/gruntwork-io/terratest v0.31.0
+	github.com/hashicorp/terraform-exec v0.15.0
+	github.com/stretchr/testify v1.7.0
+	gopkg.in/yaml.v2 v2.3.0
+)
+
+require (
+	cloud.google.com/go v0.51.0 // indirect
+	cloud.google.com/go/storage v1.0.0 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/aws/aws-sdk-go v1.27.1 // indirect
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/hashicorp/go-checkpoint v0.5.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-getter v1.5.3 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
+	github.com/hashicorp/go-uuid v1.0.0 // indirect
+	github.com/hashicorp/go-version v1.3.0 // indirect
+	github.com/hashicorp/terraform-json v0.13.0 // indirect
+	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/klauspost/compress v1.11.2 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/ulikunitz/xz v0.5.8 // indirect
+	github.com/zclconf/go-cty v1.9.1 // indirect
+	go.opencensus.io v0.22.2 // indirect
+	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
+	golang.org/x/exp v0.0.0-20191227195350-da58074b4299 // indirect
+	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	golang.org/x/net v0.0.0-20210326060303-6b1517762897 // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/tools v0.0.0-20200113040837-eac381796e91 // indirect
+	google.golang.org/api v0.15.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
+	google.golang.org/grpc v1.27.0 // indirect
+	google.golang.org/protobuf v1.24.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+	honnef.co/go/tools v0.0.1-2019.2.3 // indirect
+)