Integration Doc Generator #87
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Doc Generator | |
| # Trigger manually from GitHub UI | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| target_branch: | |
| description: 'Branch in appsmith-docs to create PR against' | |
| required: true | |
| default: 'main' # Updated default from 'docs-staging' to 'main' | |
| type: string | |
| jobs: | |
| generate_docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Step 1: Checkout appsmith-docs repo (where docs will be committed) | |
| - name: Checkout appsmith-docs (target branch) | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| ref: ${{ github.event.inputs.target_branch }} | |
| fetch-depth: 0 | |
| # Step 2: Checkout integration-resources prod repo (detect changed integration files) | |
| - name: Checkout integration-resources repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: appsmithorg/integration-resources | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| ref: main | |
| path: integration-resources | |
| fetch-depth: 2 # Get last 2 commits to check diff | |
| # Step 3: Detect changed files in the target directory | |
| - name: Get changed files in uqi_configs | |
| id: changed-files | |
| uses: tj-actions/changed-files@v46 | |
| with: | |
| path: integration-resources/Generic UQI Creation/uqi_configs | |
| since_last_remote_commit: true | |
| # Step 4: Save changed file paths to a list | |
| - name: Save list of changed files | |
| run: | | |
| mkdir -p scripts | |
| echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' > files_to_process.txt | |
| if [ ! -s files_to_process.txt ]; then | |
| echo "No new or modified files to process. Exiting." | |
| exit 0 | |
| fi | |
| echo "changes_found=true" >> $GITHUB_ENV | |
| # Step 5: Extract data and generate markdown docs using OpenAI | |
| - name: Process files with OpenAI | |
| if: env.changes_found == 'true' | |
| run: | | |
| mkdir -p generated_docs | |
| PROCESSED_COUNT=0 | |
| while IFS= read -r FILE_PATH; do | |
| FILE_NAME=$(basename "$FILE_PATH") | |
| echo "⏳ Processing $FILE_NAME" | |
| FILE_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME" | |
| echo "Fetching content from: $FILE_URL" | |
| curl -fsSL --max-time 60 "$FILE_URL" -o input_file.json || { | |
| echo "❌ Failed to fetch $FILE_NAME" | |
| continue | |
| } | |
| SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt || echo "Extract important integration details.") | |
| USER_CONTENT=$(cat input_file.json) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$USER_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 2000, | |
| temperature: 0 | |
| }') | |
| RESPONSE1=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| if echo "$RESPONSE1" | jq -e '.error' > /dev/null; then | |
| echo "❌ OpenAI error on Prompt 1" | |
| echo "$RESPONSE1" | jq . | |
| continue | |
| fi | |
| echo "$RESPONSE1" | jq -r '.choices[0].message.content' > extracted_info.md | |
| SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate reference documentation in markdown.") | |
| EXTRACTED_CONTENT=$(cat extracted_info.md) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$EXTRACTED_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 4000, | |
| temperature: 0.3 | |
| }') | |
| RESPONSE2=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| if echo "$RESPONSE2" | jq -e '.error' > /dev/null; then | |
| echo "❌ OpenAI error on Prompt 2" | |
| echo "$RESPONSE2" | jq . | |
| continue | |
| fi | |
| echo "$RESPONSE2" | jq -r '.choices[0].message.content' > generated_doc.md | |
| INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]') | |
| FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md" | |
| mkdir -p "$(dirname "$FINAL_PATH")" | |
| cp generated_doc.md "$FINAL_PATH" | |
| echo "$FILE_NAME" >> scripts/processed_files.txt | |
| PROCESSED_COUNT=$((PROCESSED_COUNT + 1)) | |
| echo "✅ Finished $FILE_NAME" | |
| done < files_to_process.txt | |
| echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV | |
| if [ "$PROCESSED_COUNT" -gt 0 ]; then | |
| echo "content_generated=true" >> $GITHUB_ENV | |
| else | |
| echo "content_generated=false" >> $GITHUB_ENV | |
| fi | |
| rm -f input_file.json extracted_info.md generated_doc.md | |
| # Step 6: Commit and open a pull request with generated docs | |
| - name: Commit and open PR | |
| if: env.content_generated == 'true' | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| title: "docs: update integration docs for ${{ github.event.inputs.target_branch }}" | |
| commit-message: "docs: automated generation for ${{ github.event.inputs.target_branch }}\n\nProcessed changed files from integration-resources." | |
| branch: "docs-update/${{ github.event.inputs.target_branch }}-${{ github.run_id }}" | |
| base: ${{ github.event.inputs.target_branch }} | |
| add-paths: | | |
| website/docs/connect-data/reference/ | |
| scripts/processed_files.txt | |
| body: | | |
| ✅ Auto-generated PR based on file changes in `appsmithorg/integration-resources`. | |
| **Target Branch:** `${{ github.event.inputs.target_branch }}` | |
| **Source Repo:** `appsmithorg/integration-resources` | |
| This PR includes: | |
| - New/updated docs from changed integration files. | |
| - Updated tracking list in `scripts/processed_files.txt`. |