Set up helper to fetch SDK artifacts from Github

Author: pschutz93

examples/docusaurus/speakeasy.glean.config.github.mjs

@@ -0,0 +1,31 @@
+import { withGithubSdk } from "@speakeasy-api/docs-md";
+
+export default {
+  spec: "../specs/glean.yaml",
+  output: {
+    pageOutDir: "./docs/glean/api",
+    embedOutDir: "./src/components/glean-embeds",
+    framework: "docusaurus",
+  },
+  display: {
+    maxNestingLevel: 2,
+  },
+  codeSamples: [
+    withGithubSdk(
+      {
+        language: "typescript",
+        owner: "gleanwork",
+        repo: "api-client-typescript",
+        version: "v0.11.2",
+        tryItNow: {
+          outDir: "./public/glean-try-it-now",
+          urlPrefix: "/glean-try-it-now",
+        },
+      },
+      process.env.GITHUB_TOKEN
+    ),
+    {
+      language: "curl",
+    },
+  ],
+};

packages/compiler/package.json

@@ -36,6 +36,8 @@
     "@microsoft/api-extractor": "^7.52.15",
     "@speakeasy-api/docs-md-react": "0.2.36",
     "@speakeasy-api/docs-md-shared": "0.2.36",
+    "@octokit/request-error": "^6.1.8",
+    "@octokit/rest": "^21.0.2",
     "arg": "^5.0.2",
     "chalk": "^5.3.0",
     "change-case": "^5.4.4",
@@ -54,4 +56,4 @@
     "globals": "^16.3.0",
     "typescript": "^5.8.3"
   }
-}
+}

packages/compiler/src/cli/cli.ts

@@ -132,7 +132,8 @@ async function getSettings(): Promise<Settings> {
   configFileImport.output ??= {};
 
   // Parse the settings using Zod to ensure accuracy
-  const configFileContents = settingsSchema.safeParse(configFileImport);
+  const configFileContents =
+    await settingsSchema.safeParseAsync(configFileImport);
   if (!configFileContents.success) {
     reportError(
       `Error parsing config file "${configFilePath}": ${z.prettifyError(configFileContents.error)}`

packages/compiler/src/compiler.ts

@@ -1,4 +1,5 @@
 export { getSettings } from "./settings.ts";
 export { escapeText } from "./renderers/util.ts";
+export { withGithubSdk } from "./fetchers/withGithub.ts";
 export type { FrameworkConfig } from "./types/FrameworkConfig.ts";
 export type { Settings } from "./settings.ts";

packages/compiler/src/fetchers/withGithub.ts

@@ -0,0 +1,106 @@
+import { randomUUID } from "node:crypto";
+import { createWriteStream, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { pipeline } from "node:stream/promises";
+
+import { RequestError } from "@octokit/request-error";
+import { Octokit } from "@octokit/rest";
+
+import { error, info } from "../logging.ts";
+import type { CodeSampleWithSDK } from "../settings.ts";
+import { InternalError } from "../util/internalError.ts";
+
+type GithubSdkConfig = {
+  owner: string;
+  repo: string;
+  version: string;
+} & CodeSampleWithSDK;
+
+export async function withGithubSdk(
+  sdk: GithubSdkConfig,
+  token?: string
+): Promise<CodeSampleWithSDK> {
+  const githubToken = token ?? process.env.GITHUB_TOKEN;
+  if (githubToken === undefined) {
+    error(
+      "GitHub Personal Access Token is required. Please pass to withGithubSdks or set GITHUB_TOKEN environment variable"
+    );
+    process.exit(1);
+  }
+
+  const octokit = new Octokit({
+    auth: githubToken,
+  });
+
+  const { owner, repo } = sdk;
+
+  info(
+    `Fetching ${sdk.language} SDK from ${owner}/${repo}@${sdk.version ? sdk.version : "latest"}`
+  );
+
+  const tarballUrl = await getTarballUrl(octokit, owner, repo, sdk.version);
+
+  const tempDir = join(tmpdir(), `speakeasy-github-${randomUUID()}`);
+  mkdirSync(tempDir, { recursive: true });
+
+  const sdkTarballPath = join(tempDir, `${repo}-${sdk.version}.tar.gz`);
+
+  const response = await fetch(tarballUrl, {
+    headers: {
+      Authorization: `token ${token}`,
+      Accept: "application/vnd.github.v3+json",
+    },
+  });
+
+  if (!response.ok || !response.body) {
+    error(
+      `Failed to download archive for repo ${repo}}: ${response.status} ${response.statusText}`
+    );
+    process.exit(1);
+  }
+
+  const fileStream = createWriteStream(sdkTarballPath);
+  await pipeline(response.body as unknown as NodeJS.ReadableStream, fileStream);
+
+  return { ...sdk, sdkTarballPath };
+}
+
+async function getTarballUrl(
+  octokit: Octokit,
+  owner: string,
+  repo: string,
+  version?: string
+): Promise<string> {
+  try {
+    const { data } =
+      version && version !== "latest"
+        ? await octokit.repos.getReleaseByTag({
+            owner,
+            repo,
+            tag: version,
+          })
+        : await octokit.repos.getLatestRelease({
+            owner,
+            repo,
+          });
+
+    if (!data.tarball_url) {
+      error(`No tarball URL found for release ${version} in ${owner}/${repo}`);
+      process.exit(1);
+    }
+    return data.tarball_url;
+  } catch (e) {
+    if (e instanceof RequestError) {
+      if (e.status === 404) {
+        error(
+          `Release ${version} not found in ${owner}/${repo}. Make sure the tag exists and the token has access to the repository.`
+        );
+        process.exit(1);
+      }
+    }
+    throw new InternalError(
+      `Failed to fetch artifact from Github: ${(e as Error).message}`
+    );
+  }
+}

packages/compiler/src/settings.ts

@@ -42,6 +42,11 @@ export function setInternalSetting<Key extends keyof InternalSettings>(
   internalSettings[key] = value;
 }
 
+// helper function for use with z.preprocess to unwrap promises for settings values
+async function valueOrPromise(value: unknown) {
+  return await Promise.resolve(value);
+}
+
 const sdkCommonProperties = {
   /**
    * The path to a tarball file of the SDK. Use this option if, for example,
@@ -162,7 +167,12 @@ const otherSdkLanguages = z.object({
   ...sdkCommonProperties,
 });
 
-const codeSample = z.union([curl, typescript, python, otherSdkLanguages]);
+const codeSampleWithSdk = z.preprocess(
+  valueOrPromise,
+  z.union([typescript, python, otherSdkLanguages])
+);
+const codeSample = z.union([curl, codeSampleWithSdk]);
+export type CodeSampleWithSDK = z.infer<typeof codeSampleWithSdk>;
 
 export const settingsSchema = z.strictObject({
   /**