diff --git a/website/components/remote-plugin-docs/utils/fetch-github-file.js b/website/components/remote-plugin-docs/utils/fetch-github-file.js deleted file mode 100644 index e12a2738d..000000000 --- a/website/components/remote-plugin-docs/utils/fetch-github-file.js +++ /dev/null @@ -1,71 +0,0 @@ -const fetch = require('isomorphic-unfetch') - -const GITHUB_API_TOKEN = process.env.GITHUB_API_TOKEN - -async function githubQuery(body, token) { - const result = await fetch('https://api.github.com/graphql', { - method: 'POST', - headers: { - Authorization: `bearer ${token}`, - ContentType: 'application/json', - }, - body: JSON.stringify(body), - }) - return await result.json() -} - -// Fetch a file from GitHub using the GraphQL API -async function getGithubFile({ repo, branch, filePath }) { - const [repo_owner, repo_name] = repo.split('/') - // Set up the GraphQL query - // (usually we can keep this in a separate file, and rely on a - // plaintext loader we've set up in our NextJS config, but we need - // to fetch remote content when indexing it, which happens outside - // NextJS, so unfortunately it seems this has to be inlined) - const query = ` -query($repo_name: String!, $repo_owner: String!, $object_expression: String!) { - repository(name: $repo_name, owner: $repo_owner) { - object(expression: $object_expression) { - ... on Blob { - text - } - } - } -} -` - // Set variables - const variables = { - repo_name, - repo_owner, - object_expression: `${branch}:${filePath}`, - } - // Query the GitHub API, and parse the navigation data - const result = await githubQuery({ query, variables }, GITHUB_API_TOKEN) - try { - const fileText = result.data.repository.object.text - return [null, fileText] - } catch (e) { - const errorMsg = `Could not fetch remote file text from "${ - variables.object_expression - }" in "${repo_owner}/${repo_name}". Received instead:\n\n${JSON.stringify( - result, - null, - 2 - )}` - return [errorMsg, null] - } -} - -function memoize(method) { - let cache = {} - - return async function () { - let args = JSON.stringify(arguments[0]) - if (!cache[args]) { - cache[args] = method.apply(this, arguments) - } - return cache[args] - } -} - -module.exports = memoize(getGithubFile) diff --git a/website/components/remote-plugin-docs/utils/merge-remote-plugins.js b/website/components/remote-plugin-docs/utils/merge-remote-plugins.js deleted file mode 100644 index 41d0d7139..000000000 --- a/website/components/remote-plugin-docs/utils/merge-remote-plugins.js +++ /dev/null @@ -1,166 +0,0 @@ -const path = require('path') -const fetchGithubFile = require('./fetch-github-file') - -const COMPONENT_TYPES = [ - 'builders', - 'datasources', - 'post-processors', - 'provisioners', -] - -async function gatherRemotePlugins(pluginsData, navData, isDev = true) { - const allPluginData = await Promise.all( - pluginsData.map(async (pluginEntry) => { - const componentEntries = await Promise.all( - COMPONENT_TYPES.map(async (type) => { - const routes = await gatherPluginBranch(pluginEntry, type) - if (!routes) return false - const isSingleLeaf = - routes.length === 1 && typeof routes[0].path !== 'undefined' - const navData = isSingleLeaf - ? { ...routes[0], path: path.join(type, pluginEntry.path) } - : { title: pluginEntry.title, routes } - return { type, navData } - }) - ) - const validComponents = componentEntries.filter(Boolean) - if (validComponents.length === 0) { - const errMsg = `Could not fetch any component documentation for remote plugin from ${pluginEntry.repo}. This may be a GitHub credential issue at build time, or it may be an issue with missing docs in the source repository. Please ensure you have a valid GITHUB_API_TOKEN set in .env.local at the root of the project.` - if (isDev) { - console.warn(errMsg) - } else { - throw new Error(errMsg) - } - } - return validComponents - }) - ) - - const allPluginsByType = allPluginData.reduce((acc, pluginData) => { - pluginData.forEach((p) => { - const { type, navData } = p - if (!acc[type]) acc[type] = [] - acc[type].push(navData) - }) - return acc - }, {}) - - const navDataWithPlugins = navData.slice().map((n) => { - // we only care about top-level NavBranch nodes - if (!n.routes) return n - // for each component type, check if this NavBranch - // is the parent route for that type - for (var i = 0; i < COMPONENT_TYPES.length; i++) { - const type = COMPONENT_TYPES[i] - const isTypeRoute = n.routes.filter((nn) => nn.path === type).length > 0 - if (isTypeRoute) { - const pluginsOfType = allPluginsByType[type] - if (!pluginsOfType || pluginsOfType.length == 0) return n - // if this NavBranch is the parent route for the type, - // then append all remote plugins of this type to the - // NavBranch's child routes - const routesWithPlugins = n.routes.slice().concat(pluginsOfType) - // console.log(JSON.stringify(routesWithPlugins, null, 2)) - // Also, sort the child routes so the order is alphabetical - routesWithPlugins.sort((a, b) => { - // (exception: "Overview" comes first) - if (a.title == 'Overview') return -1 - if (b.title === 'Overview') return 1 - // (exception: "Community-Supported" comes last) - if (a.title == 'Community-Supported') return 1 - if (b.title === 'Community-Supported') return -1 - // (exception: "Custom" comes second-last) - if (a.title == 'Custom') return 1 - if (b.title === 'Custom') return -1 - return a.title < b.title ? -1 : a.title > b.title ? 1 : 0 - }) - // return n - return { ...n, routes: routesWithPlugins } - } - } - return n - }) - - return navDataWithPlugins -} - -async function gatherPluginBranch(pluginEntry, component) { - const artifactDir = pluginEntry.artifactDir || '.docs-artifacts' - const branch = pluginEntry.branch || 'main' - const navDataFilePath = `${artifactDir}/${component}/nav-data.json` - const [err, fileResult] = await fetchGithubFile({ - repo: pluginEntry.repo, - branch, - filePath: navDataFilePath, - }) - // If one component errors, that's expected - we try all components. - // We'll check one level up to see if ALL components fail. - if (err) return false - const navData = JSON.parse(fileResult) - const withPrefixedPath = await prefixNavDataPath( - navData, - { - repo: pluginEntry.repo, - branch, - componentArtifactsDir: path.join('.docs-artifacts', component), - }, - path.join(component, pluginEntry.path) - ) - // Add plugin tier - // Parse the plugin tier - const pluginOwner = pluginEntry.repo.split('/')[0] - const pluginTier = pluginOwner === 'hashicorp' ? 'official' : 'community' - const withPluginTier = addPluginTier(withPrefixedPath, pluginTier) - // Return the augmented navData - return withPluginTier -} - -function addPluginTier(navData, pluginTier) { - return navData.slice().map((navNode) => { - if (typeof navNode.path !== 'undefined') { - return { ...navNode, pluginTier } - } - if (navNode.routes) { - return { ...navNode, routes: addPluginTier(navNode.routes, pluginTier) } - } - return navNode - }) -} - -async function prefixNavDataPath( - navData, - { repo, branch, componentArtifactsDir }, - parentPath -) { - return await Promise.all( - navData.slice().map(async (navNode) => { - if (typeof navNode.path !== 'undefined') { - const prefixedPath = path.join(parentPath, navNode.path) - const remoteFile = { - repo, - branch, - filePath: path.join(componentArtifactsDir, navNode.filePath), - } - const withPrefixedRoute = { - ...navNode, - path: prefixedPath, - remoteFile: remoteFile, - } - delete withPrefixedRoute.filePath - return withPrefixedRoute - } - if (navNode.routes) { - const prefixedRoutes = await prefixNavDataPath( - navNode.routes, - { repo, branch, componentArtifactsDir }, - parentPath - ) - const withPrefixedRoutes = { ...navNode, routes: prefixedRoutes } - return withPrefixedRoutes - } - return navNode - }) - ) -} - -module.exports = gatherRemotePlugins