Integration Doc Generator #96
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Doc Generator | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| target_branch: | |
| description: 'Branch in appsmith-docs to create PR against' | |
| required: true | |
| default: 'main' | |
| type: string | |
| jobs: | |
| generate_docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Step 1: Checkout appsmith-docs repo (target branch) | |
| - name: Checkout appsmith-docs (target branch) | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| ref: ${{ github.event.inputs.target_branch }} | |
| fetch-depth: 0 | |
| # Step 2: Checkout integration-resources repo | |
| - name: Checkout integration-resources repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: appsmithorg/integration-resources | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| ref: main | |
| path: integration-resources | |
| fetch-depth: 20 # Ensure recent commit history is available | |
| # Step 3: Always get latest 3 committed *_uqi_config.json files | |
| - name: Get latest committed JSON files | |
| id: detect-files | |
| run: | | |
| mkdir -p scripts | |
| OUTPUT_FILE="files_to_process.txt" | |
| # Get the 3 most recently committed *_uqi_config.json files | |
| cd integration-resources | |
| git log --pretty=format: --name-only -n 20 \ | |
| | grep 'Generic UQI Creation/uqi_configs/.*_uqi_config.json' \ | |
| | sort -u \ | |
| | head -n 3 \ | |
| > ../$OUTPUT_FILE | |
| cd .. | |
| echo "Files to process:" | |
| cat $OUTPUT_FILE || echo "None found" | |
| if [ ! -s "$OUTPUT_FILE" ]; then | |
| echo "❌ No recent JSON files found. Exiting." | |
| exit 0 | |
| fi | |
| echo "changes_found=true" >> $GITHUB_ENV | |
| # Step 4: Process each file with OpenAI to generate docs | |
| - name: Process files with OpenAI | |
| if: env.changes_found == 'true' | |
| run: | | |
| mkdir -p generated_docs | |
| PROCESSED_COUNT=0 | |
| while IFS= read -r FILE_PATH; do | |
| FILE_NAME=$(basename "$FILE_PATH") | |
| echo "⏳ Processing $FILE_NAME" | |
| FILE_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME" | |
| echo "Fetching content from: $FILE_URL" | |
| curl -fsSL --max-time 60 "$FILE_URL" -o input_file.json || { | |
| echo "❌ Failed to fetch $FILE_NAME" | |
| continue | |
| } | |
| SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt || echo "Extract important integration details.") | |
| USER_CONTENT=$(cat input_file.json) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$USER_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 2000, | |
| temperature: 0 | |
| }') | |
| RESPONSE1=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| if echo "$RESPONSE1" | jq -e '.error' > /dev/null; then | |
| echo "❌ OpenAI error on Prompt 1" | |
| echo "$RESPONSE1" | jq . | |
| continue | |
| fi | |
| echo "$RESPONSE1" | jq -r '.choices[0].message.content' > extracted_info.md | |
| SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate reference documentation in markdown.") | |
| EXTRACTED_CONTENT=$(cat extracted_info.md) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$EXTRACTED_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 4000, | |
| temperature: 0.3 | |
| }') | |
| RESPONSE2=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| if echo "$RESPONSE2" | jq -e '.error' > /dev/null; then | |
| echo "❌ OpenAI error on Prompt 2" | |
| echo "$RESPONSE2" | jq . | |
| continue | |
| fi | |
| echo "$RESPONSE2" | jq -r '.choices[0].message.content' > generated_doc.md | |
| INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]') | |
| FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md" | |
| mkdir -p "$(dirname "$FINAL_PATH")" | |
| cp generated_doc.md "$FINAL_PATH" | |
| echo "$FILE_NAME" >> scripts/processed_files.txt | |
| PROCESSED_COUNT=$((PROCESSED_COUNT + 1)) | |
| echo "✅ Finished $FILE_NAME" | |
| done < files_to_process.txt | |
| echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV | |
| if [ "$PROCESSED_COUNT" -gt 0 ]; then | |
| echo "content_generated=true" >> $GITHUB_ENV | |
| else | |
| echo "content_generated=false" >> $GITHUB_ENV | |
| fi | |
| rm -f input_file.json extracted_info.md generated_doc.md | |
| # Step 5: Commit and raise a PR | |
| - name: Commit and open PR | |
| if: env.content_generated == 'true' | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.REPO_ACCESS_TOKEN }} | |
| title: "docs: update integration docs for ${{ github.event.inputs.target_branch }}" | |
| commit-message: "docs: automated generation for ${{ github.event.inputs.target_branch }}\n\nGenerated from latest committed integration configs." | |
| branch: "docs-update/${{ github.event.inputs.target_branch }}-${{ github.run_id }}" | |
| base: ${{ github.event.inputs.target_branch }} | |
| add-paths: | | |
| website/docs/connect-data/reference/ | |
| scripts/processed_files.txt | |
| body: | | |
| ✅ Auto-generated PR from `appsmithorg/integration-resources`. | |
| **Target Branch:** `${{ github.event.inputs.target_branch }}` | |
| **Source Repo:** `appsmithorg/integration-resources` | |
| This PR includes: | |
| - New/updated docs from the latest committed integration files. | |
| - Tracking file update in `scripts/processed_files.txt`. |