diff --git a/.github/workflows/environment-outputs.yaml b/.github/workflows/environment-outputs.yaml new file mode 100644 index 000000000..2089c3789 --- /dev/null +++ b/.github/workflows/environment-outputs.yaml @@ -0,0 +1,65 @@ +name: environment-outputs + +on: + pull_request: + paths: + - environment-outputs/** + - '*.json' + - '*.yaml' + - .github/workflows/environment-outputs.yaml + push: + branches: + - main + paths: + - environment-outputs/** + - '*.json' + - '*.yaml' + - .github/workflows/environment-outputs.yaml + +defaults: + run: + working-directory: environment-outputs + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + with: + node-version: 20 + - run: corepack enable pnpm + - run: pnpm i + - run: pnpm test + + e2e-test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + with: + node-version: 20 + - run: corepack enable pnpm + - run: pnpm i + - run: pnpm build + - uses: ./environment-outputs + id: environment + with: + service: example + rules: | + - pull_request: + base: '**' + head: '**' + outputs: + overlay: pr + namespace: pr-${{ github.event.pull_request.number }} + - push: + ref: refs/heads/main + outputs: + overlay: development + namespace: development + - run: echo 'overlay=${{ steps.environment.outputs.overlay }}' + - run: echo 'namespace=${{ steps.environment.outputs.namespace }}' + - run: echo 'github-deployment-url=${{ steps.environment.outputs.github-deployment-url }}' diff --git a/environment-outputs/README.md b/environment-outputs/README.md new file mode 100644 index 000000000..3bdb15c47 --- /dev/null +++ b/environment-outputs/README.md @@ -0,0 +1,146 @@ +# environment-outputs [![environment-outputs](https://github.com/quipper/monorepo-deploy-actions/actions/workflows/environment-outputs.yaml/badge.svg)](https://github.com/quipper/monorepo-deploy-actions/actions/workflows/environment-outputs.yaml) + +This action generates outputs to deploy a service to the corresponding environment. + +## Getting Started + +Let's think about the following example: + +- When a pull request is created, deploy it to `pr-NUMBER` namespace +- When `main` branch is pushed, deploy it to `development` namespace + +It can be descibed as the following rules: + +```yaml +- pull_request: + base: '**' + head: '**' + outputs: + overlay: pr + namespace: pr-${{ github.event.pull_request.number }} +- push: + ref: refs/heads/main + outputs: + overlay: development + namespace: development +``` + +This action finds a rule matched to the current context. +If any rule is matched, this action returns the outputs corresponding to the rule. +For example, when `main` branch is pushed, this action returns the following outputs: + +```yaml +overlay: development +namespace: development +``` + +This action finds a rule in order. +If no rule is matched, this action fails. + +## GitHub Deployment + +This action supports [GitHub Deployment](https://docs.github.com/en/rest/deployments/deployments) to receive the deployment status from an external system, such as Argo CD. + +It creates a GitHub Deployment for each environment in the form of `{overlay}/{namespace}/{service}`, +if the following fields are given: + +- `overlay` (in `environments`) +- `namespace` (in `environments`) +- `service` (in the inputs) + +If an old deployment exists, this action deletes it and recreates new one. + +This action sets `github-deployment-url` field to the output. +For example, the below inputs are given, + +```yaml +- uses: quipper/monorepo-deploy-actions/environment-outputs@v1 + with: + service: backend + rules: | + - pull_request: + base: '**' + head: '**' + outputs: + overlay: pr + namespace: pr-${{ github.event.pull_request.number }} +``` + +this action creates a GitHub Deployment of `pr/pr-1/backend` and returns the following outputs: + +```yaml +overlay: pr +namespace: pr-1 +github-deployment-url: https://api.github.com/repos/octocat/example/deployments/1 +``` + +## Example + +Here is the example workflow. + +```yaml +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: quipper/monorepo-deploy-actions/environment-outputs@v1 + id: environment + with: + service: example + rules: | + - pull_request: + base: '**' + head: '**' + outputs: + overlay: pr + namespace: pr-${{ github.event.pull_request.number }} + - push: + ref: refs/heads/main + outputs: + overlay: development + namespace: development + - uses: quipper/monorepo-deploy-actions/git-push-service@v1 + with: + manifests: # (omit in this example) + overlay: ${{ steps.environment.outputs.overlay }} + namespace: ${{ steps.environment.outputs.namespace }} + service: example + application-annotations: | + argocd-commenter.int128.github.io/deployment-url=${{ steps.environment.outputs.github-deployment-url }} +``` + +## Spec + +### Inputs + +| Name | Default | Description | +| --------- | -------------- | ----------------------------------------------------------- | +| `rules` | (required) | YAML string of rules | +| `service` | (optional) | Name of service to deploy. If set, create GitHub Deployment | +| `token` | `github.token` | GitHub token, required if `service` is set | + +The following fields are available in the rules YAML. + +```yaml +- pull_request: # on pull_request event + base: # base branch name (wildcard available) + head: # head branch name (wildcard available) + outputs: # map +- push: # on push event + ref: refs/heads/main # ref name (wildcard available) + outputs: # map +``` + +It supports the wildcard pattern. +See https://github.com/isaacs/minimatch for details. + +### Outputs + +This actions returns the outputs corresponding to the rule. + +It also returns the below outputs. + +| Name | Description | +| ----------------------- | --------------------------------------------------------------------- | +| `github-deployment-url` | URL of GitHub Deployment. Available if `service` is set in the inputs | diff --git a/environment-outputs/action.yaml b/environment-outputs/action.yaml new file mode 100644 index 000000000..272202015 --- /dev/null +++ b/environment-outputs/action.yaml @@ -0,0 +1,19 @@ +name: environment-matrix +description: generate a JSON for matrix deploy +inputs: + rules: + description: YAML string of rules + required: true + service: + description: Name of service. If set, create GitHub Deployment + required: false + token: + description: GitHub token, required if service is set + required: false + default: ${{ github.token }} +outputs: + github-deployment-url: + description: URL of the GitHub Deployment, e.g. https://api.github.com/repos/octocat/example/deployments/1 +runs: + using: 'node20' + main: 'dist/index.js' diff --git a/environment-outputs/jest.config.js b/environment-outputs/jest.config.js new file mode 100644 index 000000000..4f97e5f78 --- /dev/null +++ b/environment-outputs/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + clearMocks: true, + testEnvironment: 'node', + testMatch: ['**/*.test.ts'], + verbose: true, +} diff --git a/environment-outputs/package.json b/environment-outputs/package.json new file mode 100644 index 000000000..9185ca4be --- /dev/null +++ b/environment-outputs/package.json @@ -0,0 +1,22 @@ +{ + "name": "environment-outputs", + "version": "0.0.0", + "private": true, + "scripts": { + "build": "ncc build --source-map --license licenses.txt src/main.ts", + "test": "jest" + }, + "dependencies": { + "@actions/core": "1.10.1", + "@actions/github": "6.0.0", + "@octokit/plugin-retry": "6.0.1", + "@octokit/request-error": "5.0.1", + "ajv": "8.12.0", + "js-yaml": "4.1.0", + "minimatch": "9.0.3" + }, + "devDependencies": { + "@types/js-yaml": "4.0.9", + "@types/minimatch": "5.1.2" + } +} diff --git a/environment-outputs/src/deployment.ts b/environment-outputs/src/deployment.ts new file mode 100644 index 000000000..49fd0239e --- /dev/null +++ b/environment-outputs/src/deployment.ts @@ -0,0 +1,82 @@ +import * as core from '@actions/core' +import * as github from '@actions/github' +import { RequestError } from '@octokit/request-error' +import { Octokit, assertPullRequestPayload } from './github' +import assert from 'assert' + +type Context = Pick + +export const createDeployment = async ( + octokit: Octokit, + context: Context, + overlay: string, + namespace: string, + service: string, +) => { + const environment = `${overlay}/${namespace}/${service}` + + core.info(`Finding the old deployments for environment ${environment}`) + const oldDeployments = await octokit.rest.repos.listDeployments({ + owner: context.repo.owner, + repo: context.repo.repo, + environment, + }) + + core.info(`Deleting ${oldDeployments.data.length} deployment(s)`) + for (const deployment of oldDeployments.data) { + try { + await octokit.rest.repos.deleteDeployment({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: deployment.id, + }) + } catch (error) { + if (error instanceof RequestError) { + core.warning(`Could not delete the old deployment ${deployment.url}: ${error.status} ${error.message}`) + continue + } + throw error + } + core.info(`Deleted the old deployment ${deployment.url}`) + } + core.info(`Deleted ${oldDeployments.data.length} deployment(s)`) + + const ref = getDeploymentRef(context) + core.info(`Creating a deployment for environment=${environment}, ref=${ref}`) + const created = await octokit.rest.repos.createDeployment({ + owner: context.repo.owner, + repo: context.repo.repo, + ref, + environment, + auto_merge: false, + required_contexts: [], + transient_environment: context.eventName === 'pull_request', + payload: { overlay, namespace, service }, + }) + assert.strictEqual(created.status, 201) + core.info(`Created a deployment ${created.data.url}`) + + // If the deployment is not deployed for a while, it will cause the following error: + // This branch had an error being deployed + // 1 abandoned deployment + // + // To avoid this, we set the deployment status to inactive immediately. + core.info(`Setting the deployment status to inactive`) + await octokit.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: created.data.id, + state: 'inactive', + }) + core.info(`Set the deployment status to inactive`) + return created.data.url +} + +const getDeploymentRef = (context: Context): string => { + if (context.eventName === 'pull_request') { + // Set the head ref to associate a deployment with the pull request + assertPullRequestPayload(context.payload.pull_request) + return context.payload.pull_request.head.ref + } + return context.ref +} diff --git a/environment-outputs/src/github.ts b/environment-outputs/src/github.ts new file mode 100644 index 000000000..54dc1cac3 --- /dev/null +++ b/environment-outputs/src/github.ts @@ -0,0 +1,36 @@ +import assert from 'assert' +import * as github from '@actions/github' +import * as pluginRetry from '@octokit/plugin-retry' + +export type Octokit = ReturnType + +export const getOctokit = (token: string): Octokit => { + return github.getOctokit(token, { previews: ['ant-man', 'flash'] }, pluginRetry.retry) +} + +// picked from https://docs.github.com/en/rest/pulls/pulls#get-a-pull-request +export type PullRequestPayload = { + head: { + ref: string + } + base: { + ref: string + } +} + +export function assertPullRequestPayload(x: unknown): asserts x is PullRequestPayload { + assert(typeof x === 'object') + assert(x != null) + + assert('base' in x) + assert(typeof x.base === 'object') + assert(x.base != null) + assert('ref' in x.base) + assert(typeof x.base.ref === 'string') + + assert('head' in x) + assert(typeof x.head === 'object') + assert(x.head != null) + assert('ref' in x.head) + assert(typeof x.head.ref === 'string') +} diff --git a/environment-outputs/src/main.ts b/environment-outputs/src/main.ts new file mode 100644 index 000000000..34f3f2792 --- /dev/null +++ b/environment-outputs/src/main.ts @@ -0,0 +1,24 @@ +import * as core from '@actions/core' +import { run } from './run' + +const main = async (): Promise => { + const outputs = await run({ + rules: core.getInput('rules', { required: true }), + service: core.getInput('service'), + token: core.getInput('token'), + }) + core.info('Setting outputs:') + for (const [k, v] of Object.entries(outputs.outputs)) { + core.info(`${k}=${v}`) + core.setOutput(k, v) + } + if (outputs.githubDeploymentURL) { + core.info(`github-deployment-url=${outputs.githubDeploymentURL}`) + core.setOutput('github-deployment-url', outputs.githubDeploymentURL) + } +} + +main().catch((e: Error) => { + core.setFailed(e) + console.error(e) +}) diff --git a/environment-outputs/src/matcher.ts b/environment-outputs/src/matcher.ts new file mode 100644 index 000000000..4df00ce82 --- /dev/null +++ b/environment-outputs/src/matcher.ts @@ -0,0 +1,28 @@ +import * as github from '@actions/github' +import { minimatch } from 'minimatch' +import { Rule, Rules } from './rule' +import { assertPullRequestPayload } from './github' + +type Context = Pick + +export const find = (context: Context, rules: Rules): Record | undefined => { + for (const rule of rules) { + if (match(context, rule)) { + return rule.outputs + } + } +} + +const match = (context: Context, rule: Rule): boolean => { + if (context.eventName === 'pull_request' && rule.pull_request !== undefined) { + assertPullRequestPayload(context.payload.pull_request) + return ( + minimatch(context.payload.pull_request.base.ref, rule.pull_request.base) && + minimatch(context.payload.pull_request.head.ref, rule.pull_request.head) + ) + } + if (context.eventName === 'push' && rule.push !== undefined) { + return minimatch(context.ref, rule.push.ref) + } + return false +} diff --git a/environment-outputs/src/rule.ts b/environment-outputs/src/rule.ts new file mode 100644 index 000000000..b4fcd6c95 --- /dev/null +++ b/environment-outputs/src/rule.ts @@ -0,0 +1,64 @@ +import * as yaml from 'js-yaml' +import Ajv, { JTDSchemaType } from 'ajv/dist/jtd' + +export type Rule = { + pull_request?: { + base: string + head: string + } + push?: { + ref: string + } + outputs: Record +} + +const RuleSchema: JTDSchemaType = { + properties: { + outputs: { + values: { + type: 'string', + }, + }, + }, + optionalProperties: { + pull_request: { + properties: { + base: { + type: 'string', + }, + head: { + type: 'string', + }, + }, + }, + push: { + properties: { + ref: { + type: 'string', + }, + }, + }, + }, +} + +export type Rules = Rule[] + +const rulesSchema: JTDSchemaType = { + elements: RuleSchema, +} + +const ajv = new Ajv() +export const validateRules = ajv.compile(rulesSchema) + +export const parseRulesYAML = (s: string): Rules => { + const rules = yaml.load(s) + if (!validateRules(rules)) { + if (validateRules.errors) { + throw new Error( + `invalid rules YAML: ${validateRules.errors.map((e) => `${e.instancePath} ${e.message || ''}`).join(', ')}`, + ) + } + throw new Error('invalid rules YAML') + } + return rules +} diff --git a/environment-outputs/src/run.ts b/environment-outputs/src/run.ts new file mode 100644 index 000000000..fa36f38a5 --- /dev/null +++ b/environment-outputs/src/run.ts @@ -0,0 +1,40 @@ +import assert from 'assert' +import * as core from '@actions/core' +import * as github from '@actions/github' +import { parseRulesYAML } from './rule' +import { find } from './matcher' +import { createDeployment } from './deployment' +import { getOctokit } from './github' + +type Inputs = { + rules: string + service: string + token: string +} + +type Outputs = { + outputs: Record + githubDeploymentURL?: string +} + +export const run = async (inputs: Inputs): Promise => { + const rules = parseRulesYAML(inputs.rules) + core.info(`rules: ${JSON.stringify(rules, undefined, 2)}`) + const outputs = find(github.context, rules) + if (outputs === undefined) { + throw new Error(`no environment to deploy`) + } + + if (!inputs.service) { + return { outputs } + } + + core.info(`Creating a GitHub Deployment for the environment`) + const { overlay, namespace } = outputs + assert(overlay, `overlay is required in the rule outputs`) + assert(namespace, `namespace is required in the rule outputs`) + assert(inputs.token, `inputs.token is required`) + const octokit = getOctokit(inputs.token) + const githubDeploymentURL = await createDeployment(octokit, github.context, overlay, namespace, inputs.service) + return { outputs, githubDeploymentURL } +} diff --git a/environment-outputs/tests/matcher.test.ts b/environment-outputs/tests/matcher.test.ts new file mode 100644 index 000000000..4d50ffe3f --- /dev/null +++ b/environment-outputs/tests/matcher.test.ts @@ -0,0 +1,91 @@ +import { find } from '../src/matcher' +import { Rules } from '../src/rule' + +const rules: Rules = [ + { + pull_request: { + head: '*/qa', + base: '*/production', + }, + outputs: { + overlay: 'pr', + namespace: 'pr-2', + }, + }, + { + pull_request: { + head: '**', + base: '**', + }, + outputs: { + overlay: 'pr', + namespace: 'pr-1', + }, + }, + { + push: { + ref: 'refs/heads/main', + }, + outputs: { + overlay: 'development', + namespace: 'development', + }, + }, +] + +test('pull_request with any branches', () => { + const context = { + eventName: 'pull_request', + payload: { + pull_request: { + number: 1, + head: { ref: 'topic' }, + base: { ref: 'main' }, + }, + }, + ref: 'refs/pull/1/merge', + } + expect(find(context, rules)).toStrictEqual({ + overlay: 'pr', + namespace: 'pr-1', + }) +}) + +test('pull_request with patterns', () => { + const context = { + eventName: 'pull_request', + payload: { + pull_request: { + number: 2, + head: { ref: 'microservice/qa' }, + base: { ref: 'microservice/production' }, + }, + }, + ref: 'refs/pull/2/merge', + } + expect(find(context, rules)).toStrictEqual({ + overlay: 'pr', + namespace: 'pr-2', + }) +}) + +test('push', () => { + const context = { + eventName: 'push', + payload: {}, + ref: 'refs/heads/main', + } + expect(find(context, rules)).toStrictEqual({ + overlay: 'development', + namespace: 'development', + }) +}) + +test('push with no match', () => { + const context = { + eventName: 'push', + payload: {}, + ref: 'refs/tags/v1.0.0', + } + expect(find(context, rules)).toBeUndefined() +}) diff --git a/environment-outputs/tests/rule.test.ts b/environment-outputs/tests/rule.test.ts new file mode 100644 index 000000000..99eee8f33 --- /dev/null +++ b/environment-outputs/tests/rule.test.ts @@ -0,0 +1,53 @@ +import { Rules, parseRulesYAML } from '../src/rule' + +test('parse a valid YAML', () => { + const yaml = ` +- pull_request: + base: '**' + head: '**' + outputs: + overlay: pr + namespace: pr-1 +- push: + ref: refs/heads/main + outputs: + overlay: development + namespace: development +` + expect(parseRulesYAML(yaml)).toStrictEqual([ + { + pull_request: { + base: '**', + head: '**', + }, + outputs: { + overlay: 'pr', + namespace: 'pr-1', + }, + }, + { + push: { + ref: 'refs/heads/main', + }, + outputs: { + overlay: 'development', + namespace: 'development', + }, + }, + ]) +}) + +test('parse an empty string', () => { + expect(() => parseRulesYAML('')).toThrow(`invalid rules YAML: must be array`) +}) + +test('parse an invalid string', () => { + const yaml = ` +- pull_request: + base: '**' + outputs: + overlay: pr + namespace: pr-1 +` + expect(() => parseRulesYAML(yaml)).toThrow(`invalid rules YAML: /0/pull_request must have property 'head'`) +}) diff --git a/environment-outputs/tsconfig.json b/environment-outputs/tsconfig.json new file mode 100644 index 000000000..a854be8f7 --- /dev/null +++ b/environment-outputs/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "./lib" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 879dfeca8..3a3dcea6b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -112,6 +112,8 @@ importers: specifier: 5.0.1 version: 5.0.1 + delete-pull-request-namespaces/dist: {} + environment-matrix: dependencies: '@actions/core': @@ -143,6 +145,37 @@ importers: specifier: 5.1.2 version: 5.1.2 + environment-outputs: + dependencies: + '@actions/core': + specifier: 1.10.1 + version: 1.10.1 + '@actions/github': + specifier: 6.0.0 + version: 6.0.0 + '@octokit/plugin-retry': + specifier: 6.0.1 + version: 6.0.1(@octokit/core@5.1.0) + '@octokit/request-error': + specifier: 5.0.1 + version: 5.0.1 + ajv: + specifier: 8.12.0 + version: 8.12.0 + js-yaml: + specifier: 4.1.0 + version: 4.1.0 + minimatch: + specifier: 9.0.3 + version: 9.0.3 + devDependencies: + '@types/js-yaml': + specifier: 4.0.9 + version: 4.0.9 + '@types/minimatch': + specifier: 5.1.2 + version: 5.1.2 + git-push-namespace: dependencies: '@actions/core':