Integration Doc Generator #105
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Doc Generator | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| target_branch: | |
| description: 'Branch in appsmith-docs to create PR against' | |
| required: true | |
| default: 'main' | |
| type: string | |
| jobs: | |
| generate_docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checkout the docs repo on the target branch | |
| - name: Checkout appsmith-docs (target branch) | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| ref: ${{ github.event.inputs.target_branch }} | |
| fetch-depth: 0 | |
| # Checkout the integration-resources production repo | |
| - name: Checkout integration-resources repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: appsmithorg/integration-resources | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| ref: main | |
| path: integration-resources | |
| fetch-depth: 2 | |
| # Detect changed files | |
| - name: Get changed files in uqi_configs | |
| id: changed-files | |
| uses: tj-actions/changed-files@v46 | |
| with: | |
| path: integration-resources/Generic UQI Creation/uqi_configs | |
| since_last_remote_commit: true | |
| # Save list of changed files | |
| - name: Save list of changed files | |
| run: | | |
| mkdir -p scripts | |
| echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' > files_to_process.txt | |
| if [ ! -s files_to_process.txt ]; then | |
| echo "No new or modified files to process. Exiting." | |
| exit 0 | |
| fi | |
| echo "changes_found=true" >> $GITHUB_ENV | |
| # Process each changed file with OpenAI | |
| - name: Process files with OpenAI | |
| if: env.changes_found == 'true' | |
| run: | | |
| mkdir -p generated_docs | |
| PROCESSED_COUNT=0 | |
| while IFS= read -r FILE_PATH; do | |
| FILE_NAME=$(basename "$FILE_PATH") | |
| echo "⏳ Processing $FILE_NAME" | |
| # URL‐encode the file name for raw.githubusercontent | |
| FILE_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME" | |
| echo "Fetching content from: $FILE_URL" | |
| curl -fsSL --max-time 60 "$FILE_URL" -o input_file.json || { | |
| echo "❌ Failed to fetch $FILE_NAME" | |
| continue | |
| } | |
| SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt \ | |
| || echo "Extract important integration details.") | |
| USER_CONTENT=$(cat input_file.json) | |
| # Prompt 1: extract key details | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$USER_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 2000, | |
| temperature: 0 | |
| }') | |
| RESPONSE1=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| if echo "$RESPONSE1" | jq -e '.error' > /dev/null; then | |
| echo "❌ OpenAI error on Prompt 1" | |
| echo "$RESPONSE1" | jq . | |
| continue | |
| fi | |
| echo "$RESPONSE1" | jq -r '.choices[0].message.content' > extracted_info.md | |
| # Prompt 2: generate markdown doc | |
| SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt \ | |
| || echo "Generate reference documentation in markdown.") | |
| EXTRACTED_CONTENT=$(cat extracted_info.md) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$EXTRACTED_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 4000, | |
| temperature: 0.3 | |
| }') | |
| RESPONSE2=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| if echo "$RESPONSE2" | jq -e '.error' > /dev/null; then | |
| echo "❌ OpenAI error on Prompt 2" | |
| echo "$RESPONSE2" | jq . | |
| continue | |
| fi | |
| echo "$RESPONSE2" | jq -r '.choices[0].message.content' > generated_doc.md | |
| INTEGRATION=$(echo "$FILE_NAME" \ | |
| | sed 's/_uqi_config\.json//' \ | |
| | tr '[:upper:]' '[:lower:]') | |
| FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md" | |
| mkdir -p "$(dirname "$FINAL_PATH")" | |
| cp generated_doc.md "$FINAL_PATH" | |
| echo "$FILE_NAME" >> scripts/processed_files.txt | |
| PROCESSED_COUNT=$((PROCESSED_COUNT + 1)) | |
| echo "✅ Finished $FILE_NAME" | |
| done < files_to_process.txt | |
| echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV | |
| if [ "$PROCESSED_COUNT" -gt 0 ]; then | |
| echo "content_generated=true" >> $GITHUB_ENV | |
| else | |
| echo "content_generated=false" >> $GITHUB_ENV | |
| fi | |
| rm -f input_file.json extracted_info.md generated_doc.md | |
| # Commit & open the PR | |
| - name: Commit and open PR | |
| if: env.content_generated == 'true' | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| title: "docs: update integration docs for ${{ github.event.inputs.target_branch }}" | |
| commit-message: | | |
| docs: automated generation for ${{ github.event.inputs.target_branch }} | |
| Processed changed files from integration-resources. | |
| branch: "docs-update/${{ github.event.inputs.target_branch }}-${{ github.run_id }}" | |
| base: ${{ github.event.inputs.target_branch }} | |
| add-paths: | | |
| website/docs/connect-data/reference/ | |
| scripts/processed_files.txt | |
| body: | | |
| ✅ Auto-generated PR based on file changes in `appsmithorg/integration-resources`. | |
| **Target Branch:** `${{ github.event.inputs.target_branch }}` | |
| **Source Repo:** `appsmithorg/integration-resources` |