Test Doc Generator #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Doc Generator | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| generate_docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout appsmith-docs | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.test_REPO_ACCESS_TOKEN }} | |
| - name: Create exclusion list | |
| run: echo > saas_exclusions.txt | |
| - name: Ensure scripts directory exists | |
| run: | | |
| mkdir -p scripts | |
| [ -f scripts/processed_files.txt ] || touch scripts/processed_files.txt | |
| [ -f scripts/file_hashes.json ] || echo "{}" > scripts/file_hashes.json | |
| - name: Fetch file list from test repo | |
| id: fetch_files | |
| run: | | |
| curl -s --max-time 30 -H "Authorization: Bearer ${{ secrets.test_REPO_ACCESS_TOKEN }}" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| https://api.github.com/repos/harshilp24/integration-resources-test/contents/Generic%20UQI%20Creation/uqi_configs \ | |
| -o response.json | |
| jq -r '.[] | select(.type=="file") | [.name, .sha] | @tsv' response.json > latest_files_with_sha.txt | |
| jq -r '.[] | select(.type=="file") | .name' response.json > latest_files.txt | |
| echo "files_found=true" >> $GITHUB_ENV | |
| - name: Identify new and modified files | |
| id: detect_changes | |
| run: | | |
| PREV_HASHES=$(cat scripts/file_hashes.json) | |
| NEW_FILES=$(comm -23 <(sort latest_files.txt) <(sort scripts/processed_files.txt) || true) | |
| MODIFIED_FILES="" | |
| while IFS=$'\t' read -r FILE_NAME FILE_SHA; do | |
| PREV_SHA=$(echo "$PREV_HASHES" | jq -r --arg file "$FILE_NAME" '.[$file] // ""') | |
| if [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$FILE_SHA" ] && grep -q "^$FILE_NAME$" scripts/processed_files.txt; then | |
| MODIFIED_FILES="$MODIFIED_FILES$FILE_NAME"$'\n' | |
| fi | |
| done < latest_files_with_sha.txt | |
| { echo "$NEW_FILES"; echo "$MODIFIED_FILES"; } | grep -v "^$" > files_to_process.txt | |
| if [ -s files_to_process.txt ]; then | |
| echo "changes_found=true" >> $GITHUB_ENV | |
| else | |
| echo "changes_found=false" >> $GITHUB_ENV | |
| fi | |
| - name: Exit if no files to process | |
| if: env.changes_found != 'true' | |
| run: exit 0 | |
| - name: Process files with OpenAI | |
| run: | | |
| mkdir -p generated_docs | |
| HASHES_JSON=$(cat scripts/file_hashes.json) | |
| PROCESSED_COUNT=0 | |
| while IFS= read -r FILE_NAME; do | |
| echo "⏳ Processing $FILE_NAME" | |
| FILE_URL="https://raw.githubusercontent.com/harshilp24/integration-resources-test/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME" | |
| curl -sSL --max-time 30 "$FILE_URL" -o input_file.json | |
| FILE_SHA=$(grep "$FILE_NAME" latest_files_with_sha.txt | cut -f2) | |
| HASHES_JSON=$(echo "$HASHES_JSON" | jq --arg file "$FILE_NAME" --arg sha "$FILE_SHA" '.[$file] = $sha') | |
| # Prompt 1: Extract Info | |
| SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt || echo "Extract important integration details.") | |
| USER_CONTENT=$(cat input_file.json) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$USER_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 2000, | |
| temperature: 0 | |
| }') | |
| RESPONSE=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.test_OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| echo "$RESPONSE" | jq '.' | |
| echo "$RESPONSE" | jq -r '.choices[0].message.content' > extracted_info.md | |
| # Prompt 2: Generate Markdown | |
| SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate reference documentation in markdown.") | |
| EXTRACTED_CONTENT=$(cat extracted_info.md) | |
| PAYLOAD=$(jq -n \ | |
| --arg system "$SYSTEM_PROMPT" \ | |
| --arg user "$EXTRACTED_CONTENT" \ | |
| '{ | |
| model: "gpt-4-1106-preview", | |
| messages: [ | |
| {"role": "system", "content": $system}, | |
| {"role": "user", "content": $user} | |
| ], | |
| max_tokens: 4000, | |
| temperature: 0.3 | |
| }') | |
| RESPONSE=$(curl -s https://api.openai.com/v1/chat/completions \ | |
| -H "Authorization: Bearer ${{ secrets.test_OPENAI_API_KEY }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| echo "$RESPONSE" | jq '.' | |
| echo "$RESPONSE" | jq -r '.choices[0].message.content' > generated_doc.md | |
| INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]') | |
| FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md" | |
| mkdir -p "$(dirname "$FINAL_PATH")" | |
| cp generated_doc.md "$FINAL_PATH" | |
| cp generated_doc.md "generated_docs/${INTEGRATION}.md" | |
| echo "$FILE_NAME" >> scripts/processed_files.txt | |
| PROCESSED_COUNT=$((PROCESSED_COUNT + 1)) | |
| echo "✅ Finished $FILE_NAME" | |
| done < files_to_process.txt | |
| echo "$HASHES_JSON" > scripts/file_hashes.json | |
| echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV | |
| echo "content_generated=true" >> $GITHUB_ENV | |
| - name: Commit and open PR | |
| if: env.content_generated == 'true' | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ secrets.test_REPO_ACCESS_TOKEN }} | |
| title: "test: generate integration docs from test repo" | |
| commit-message: "test: generated docs from harshilp24/integration-resources-test" | |
| branch: "test/docs-update-${{ github.run_id }}" | |
| base: main | |
| add-paths: | | |
| website/docs/connect-data/reference/ | |
| scripts/processed_files.txt | |
| scripts/file_hashes.json | |
| body: | | |
| ✅ Test PR: Generated integration documentation from your test repo. | |
| Source: [harshilp24/integration-resources-test](https://github.com/harshilp24/integration-resources-test/tree/main/Generic%20UQI%20Creation/uqi_configs) |