Update integration-doc-generator.yml

Author: harshilp24

.github/workflows/integration-doc-generator.yml

@@ -14,32 +14,22 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      # 1. Checkout the docs repo on the target branch
+      # Step 1: Checkout appsmith-docs repo
       - name: Checkout appsmith-docs
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}
           ref: ${{ github.event.inputs.target_branch }}
           fetch-depth: 0
 
-      # 2. Checkout the integration-resources repo
-      - name: Checkout integration-resources
-        uses: actions/checkout@v4
-        with:
-          repository: appsmithorg/integration-resources
-          token: ${{ secrets.REPO_ACCESS_TOKEN }}
-          ref: main
-          path: integration-resources
-          fetch-depth: 1
-
-      # 3. Hardcode the two files to process
+      # Step 2: Set local files to process
       - name: Set files to process manually
         run: |
           echo "intercom_uqi_config.json"   > files_to_process.txt
           echo "sharepoint_uqi_config.json" >> files_to_process.txt
           cat files_to_process.txt
 
-      # 4. Generate docs via OpenAI
+      # Step 3: Generate Markdown docs using OpenAI
       - name: Generate docs with OpenAI
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -49,45 +39,62 @@ jobs:
           while IFS= read -r FILE_NAME; do
             echo "⏳ Processing $FILE_NAME"
 
-            # Build and URL-encode the full relative path
-            REL="Generic UQI Creation/uqi_configs/${FILE_NAME}"
-            ENC=$(echo "$REL" | sed 's/ /%20/g')
-            RAW_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/$ENC"
-            echo "Fetching $RAW_URL"
-            curl -fsSL "$RAW_URL" -o input.json || {
-              echo "❌ Failed to fetch $FILE_NAME"
+            INPUT_PATH=".github/resources/integration-configs/$FILE_NAME"
+            if [ ! -f "$INPUT_PATH" ]; then
+              echo "❌ File not found: $INPUT_PATH"
               exit 1
-            }
+            fi
 
             # Step 1: extract integration details
-            SYS1=$(< .github/prompts/extract_prompt.txt)
-            USR1=$(< input.json)
-            PAY1=$(jq -nc --arg sys "$SYS1" --arg usr "$USR1" \
-              '{model:"gpt-4-1106-preview",messages:[{role:"system",content:$sys},{role:"user",content:$usr}],max_tokens:2000,temperature:0}')
-            RES1=$(curl -s -H "Authorization: Bearer $OPENAI_API_KEY" \
-                        -H "Content-Type: application/json" \
-                        -d "$PAY1" \
-                        https://api.openai.com/v1/chat/completions)
-            EXTRACTED=$(echo "$RES1" | jq -r '.choices[0].message.content')
+            SYSTEM_PROMPT_1=$(cat .github/prompts/extract_prompt.txt)
+            USER_INPUT_1=$(cat "$INPUT_PATH")
+
+            PAYLOAD_1=$(jq -nc \
+              --arg sys "$SYSTEM_PROMPT_1" \
+              --arg usr "$USER_INPUT_1" \
+              '{model: "gpt-4-1106-preview", messages: [{"role": "system", "content": $sys}, {"role": "user", "content": $usr}], max_tokens: 2000, temperature: 0}')
+
+            RESPONSE_1=$(curl -s https://api.openai.com/v1/chat/completions \
+              -H "Authorization: Bearer $OPENAI_API_KEY" \
+              -H "Content-Type: application/json" \
+              -d "$PAYLOAD_1")
+
+            if echo "$RESPONSE_1" | jq -e '.error' > /dev/null; then
+              echo "❌ OpenAI error on extract"
+              echo "$RESPONSE_1" | jq .
+              continue
+            fi
+
+            EXTRACTED_CONTENT=$(echo "$RESPONSE_1" | jq -r '.choices[0].message.content')
 
             # Step 2: generate markdown
-            SYS2=$(< .github/prompts/generate_prompt.txt)
-            PAY2=$(jq -nc --arg sys "$SYS2" --arg usr "$EXTRACTED" \
-              '{model:"gpt-4-1106-preview",messages:[{role:"system",content:$sys},{role:"user",content:$usr}],max_tokens:4000,temperature:0.3}')
-            RES2=$(curl -s -H "Authorization: Bearer $OPENAI_API_KEY" \
-                        -H "Content-Type: application/json" \
-                        -d "$PAY2" \
-                        https://api.openai.com/v1/chat/completions)
-            MD=$(echo "$RES2" | jq -r '.choices[0].message.content')
-
-            # Save markdown
-            NAME=$(basename "$FILE_NAME" "_uqi_config.json" | tr '[:upper:]' '[:lower:]')
-            OUT="website/docs/connect-data/reference/${NAME}.md"
-            echo "$MD" > "$OUT"
-            echo "✅ Wrote $OUT"
+            SYSTEM_PROMPT_2=$(cat .github/prompts/generate_prompt.txt)
+
+            PAYLOAD_2=$(jq -nc \
+              --arg sys "$SYSTEM_PROMPT_2" \
+              --arg usr "$EXTRACTED_CONTENT" \
+              '{model: "gpt-4-1106-preview", messages: [{"role": "system", "content": $sys}, {"role": "user", "content": $usr}], max_tokens: 4000, temperature: 0.3}')
+
+            RESPONSE_2=$(curl -s https://api.openai.com/v1/chat/completions \
+              -H "Authorization: Bearer $OPENAI_API_KEY" \
+              -H "Content-Type: application/json" \
+              -d "$PAYLOAD_2")
+
+            if echo "$RESPONSE_2" | jq -e '.error' > /dev/null; then
+              echo "❌ OpenAI error on generate"
+              echo "$RESPONSE_2" | jq .
+              continue
+            fi
+
+            FINAL_MD=$(echo "$RESPONSE_2" | jq -r '.choices[0].message.content')
+
+            INTEGRATION_NAME=$(basename "$FILE_NAME" "_uqi_config.json" | tr '[:upper:]' '[:lower:]')
+            OUTPUT_FILE="website/docs/connect-data/reference/${INTEGRATION_NAME}.md"
+            echo "$FINAL_MD" > "$OUTPUT_FILE"
+            echo "✅ Wrote $OUTPUT_FILE"
           done < files_to_process.txt
 
-      # 5. Commit & open PR
+      # Step 4: Create a PR with generated docs
       - name: Commit & Create Pull Request
         uses: peter-evans/create-pull-request@v6
         with: