Update integration-doc-generator.yml

Author: harshilp24

.github/workflows/integration-doc-generator.yml

@@ -34,4 +34,143 @@ jobs:
 
     # Step 3: Always get latest 3 committed *_uqi_config.json files
     - name: Get latest committed JSON files
-      id: detect-
+      id: detect-files
+      run: |
+        mkdir -p scripts
+        OUTPUT_FILE="files_to_process.txt"
+
+        # Get the 3 most recently committed *_uqi_config.json files
+        cd integration-resources
+        git log --pretty=format: --name-only -n 20 \
+          | grep 'Generic UQI Creation/uqi_configs/.*_uqi_config.json' \
+          | sort -u \
+          | head -n 3 \
+          > ../$OUTPUT_FILE
+        cd ..
+
+        echo "Files to process:"
+        cat $OUTPUT_FILE || echo "None found"
+
+        if [ ! -s "$OUTPUT_FILE" ]; then
+          echo "❌ No recent JSON files found. Exiting."
+          exit 0
+        fi
+
+        echo "changes_found=true" >> $GITHUB_ENV
+
+    # Step 4: Process each file with OpenAI to generate docs
+    - name: Process files with OpenAI
+      if: env.changes_found == 'true'
+      run: |
+        mkdir -p generated_docs
+        PROCESSED_COUNT=0
+
+        while IFS= read -r FILE_PATH; do
+          FILE_NAME=$(basename "$FILE_PATH")
+          echo "⏳ Processing $FILE_NAME"
+
+          FILE_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME"
+          echo "Fetching content from: $FILE_URL"
+          curl -fsSL --max-time 60 "$FILE_URL" -o input_file.json || {
+            echo "❌ Failed to fetch $FILE_NAME"
+            continue
+          }
+
+          SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt || echo "Extract important integration details.")
+          USER_CONTENT=$(cat input_file.json)
+
+          PAYLOAD=$(jq -n \
+            --arg system "$SYSTEM_PROMPT" \
+            --arg user "$USER_CONTENT" \
+            '{
+              model: "gpt-4-1106-preview",
+              messages: [
+                {"role": "system", "content": $system},
+                {"role": "user", "content": $user}
+              ],
+              max_tokens: 2000,
+              temperature: 0
+            }')
+
+          RESPONSE1=$(curl -s https://api.openai.com/v1/chat/completions \
+            -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD")
+
+          if echo "$RESPONSE1" | jq -e '.error' > /dev/null; then
+            echo "❌ OpenAI error on Prompt 1"
+            echo "$RESPONSE1" | jq .
+            continue
+          fi
+          echo "$RESPONSE1" | jq -r '.choices[0].message.content' > extracted_info.md
+
+          SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate reference documentation in markdown.")
+          EXTRACTED_CONTENT=$(cat extracted_info.md)
+
+          PAYLOAD=$(jq -n \
+            --arg system "$SYSTEM_PROMPT" \
+            --arg user "$EXTRACTED_CONTENT" \
+            '{
+              model: "gpt-4-1106-preview",
+              messages: [
+                {"role": "system", "content": $system},
+                {"role": "user", "content": $user}
+              ],
+              max_tokens: 4000,
+              temperature: 0.3
+            }')
+
+          RESPONSE2=$(curl -s https://api.openai.com/v1/chat/completions \
+            -H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD")
+
+          if echo "$RESPONSE2" | jq -e '.error' > /dev/null; then
+            echo "❌ OpenAI error on Prompt 2"
+            echo "$RESPONSE2" | jq .
+            continue
+          fi
+          echo "$RESPONSE2" | jq -r '.choices[0].message.content' > generated_doc.md
+
+          INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]')
+          FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md"
+
+          mkdir -p "$(dirname "$FINAL_PATH")"
+          cp generated_doc.md "$FINAL_PATH"
+
+          echo "$FILE_NAME" >> scripts/processed_files.txt
+          PROCESSED_COUNT=$((PROCESSED_COUNT + 1))
+          echo "✅ Finished $FILE_NAME"
+
+        done < files_to_process.txt
+
+        echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV
+        if [ "$PROCESSED_COUNT" -gt 0 ]; then
+          echo "content_generated=true" >> $GITHUB_ENV
+        else
+          echo "content_generated=false" >> $GITHUB_ENV
+        fi
+        rm -f input_file.json extracted_info.md generated_doc.md
+
+    # Step 5: Commit and raise a PR
+    - name: Commit and open PR
+      if: env.content_generated == 'true'
+      uses: peter-evans/create-pull-request@v6
+      with:
+        token: ${{ secrets.REPO_ACCESS_TOKEN }}
+        title: "docs: update integration docs for ${{ github.event.inputs.target_branch }}"
+        commit-message: "docs: automated generation for ${{ github.event.inputs.target_branch }}\n\nGenerated from latest committed integration configs."
+        branch: "docs-update/${{ github.event.inputs.target_branch }}-${{ github.run_id }}"
+        base: ${{ github.event.inputs.target_branch }}
+        add-paths: |
+          website/docs/connect-data/reference/
+          scripts/processed_files.txt
+        body: |
+          ✅ Auto-generated PR from `appsmithorg/integration-resources`.
+
+          **Target Branch:** `${{ github.event.inputs.target_branch }}`
+          **Source Repo:** `appsmithorg/integration-resources`
+
+          This PR includes:
+          - New/updated docs from the latest committed integration files.
+          - Tracking file update in `scripts/processed_files.txt`.