diff --git a/website/components/remote-plugin-docs/utils/fetch-plugin-docs.js b/website/components/remote-plugin-docs/utils/fetch-plugin-docs.js new file mode 100644 index 000000000..d02930ca2 --- /dev/null +++ b/website/components/remote-plugin-docs/utils/fetch-plugin-docs.js @@ -0,0 +1,61 @@ +const fetch = require('isomorphic-unfetch') +const parseSourceZip = require('./parse-source-zip') +const parseDocsZip = require('./parse-docs-zip') + +// Given a repo and tag, +// +// return [null, docsMdxFiles] if docs files +// are successfully fetched and valid, +// where docsMdxFiles is an array of { filePath, fileString } items. +// +// otherwise, return [err, null] +// where err is an error message describing whether the +// docs files were missing or invalid, with a path to resolution +async function fetchDocsFiles({ repo, tag }) { + // If there's a docs.zip asset, we'll prefer that + const docsZipUrl = `https://github.com/${repo}/releases/download/${tag}/docs.zip` + const docsZipResponse = await fetch(docsZipUrl, { method: 'GET' }) + const hasDocsZip = docsZipResponse.status === 200 + // Note: early return! + if (hasDocsZip) return await parseDocsZip(docsZipResponse) + // Else if docs.zip is not present, and we only have the "latest" tag, + // then throw an error - we can't resolve the fallback source ZIP + // unless we resort to calling the GitHub API, which we do not want to do + if (tag === 'latest') { + const err = `Failed to fetch. Could not find "docs.zip" at ${docsZipUrl}. To fall back to parsing docs from "source", please provide a specific tag instead of "${tag}".` + return [err, null] + } + // Else if docs.zip is not present, and we have a specific tag, then + // fall back to parsing docs files from the source zip + const sourceZipUrl = `https://github.com/${repo}/archive/${tag}.zip` + const sourceZipResponse = await fetch(sourceZipUrl, { method: 'GET' }) + const missingSourceZip = sourceZipResponse.status !== 200 + if (missingSourceZip) { + const err = `Failed to fetch. Could not find "docs.zip" at ${docsZipUrl}, and could not find fallback source ZIP at ${sourceZipUrl}. Please ensure one of these assets is available.` + return [err, null] + } + // Handle parsing from plugin source zip + return await parseSourceZip(sourceZipResponse) +} + +async function fetchPluginDocs({ repo, tag }) { + const [err, docsMdxFiles] = await fetchDocsFiles({ repo, tag }) + if (err) { + const errMsg = `Invalid plugin docs ${repo}, on release ${tag}. ${err}` + throw new Error(errMsg) + } + return docsMdxFiles +} + +function memoize(method) { + let cache = {} + return async function () { + let args = JSON.stringify(arguments) + if (!cache[args]) { + cache[args] = method.apply(this, arguments) + } + return cache[args] + } +} + +module.exports = memoize(fetchPluginDocs) diff --git a/website/components/remote-plugin-docs/utils/parse-docs-zip.js b/website/components/remote-plugin-docs/utils/parse-docs-zip.js new file mode 100644 index 000000000..69563877e --- /dev/null +++ b/website/components/remote-plugin-docs/utils/parse-docs-zip.js @@ -0,0 +1,44 @@ +const path = require('path') +const AdmZip = require('adm-zip') +const validatePluginDocsFiles = require('./validate-plugin-docs-files') + +// Given a response from fetching a docs.zip file, +// which is a compressed "docs" folder, +// +// return [null, docsMdxFiles] if docs files +// are successfully fetched and valid, +// where docsMdxFiles is an array of { filePath, fileString } items. +// +// otherwise, return [err, null] +// where err is an error message describing whether the +// docs files were missing or invalid, with a path to resolution +async function parseDocsZip(response) { + // the file path from the repo root is the same as the zip entryName, + // which includes the docs directory as the first part of the path + const responseBuffer = Buffer.from(await response.arrayBuffer()) + const responseZip = new AdmZip(responseBuffer) + const docsEntries = responseZip.getEntries() + // Validate the file paths within the "docs" folder + const docsFilePaths = docsEntries.map((e) => e.entryName) + const validationError = validatePluginDocsFiles(docsFilePaths) + if (validationError) return [validationError, null] + // If valid, filter for MDX files only, and return + // a { filePath, fileString } object for each mdx file + const docsMdxFiles = docsEntries + .filter((e) => { + return path.extname(e.entryName) === '.mdx' + }) + .map((e) => { + const filePath = e.entryName + const fileString = e.getData().toString() + return { filePath, fileString } + }) + return [null, docsMdxFiles] +} + +/* + const dirs = path.dirname(e.entryName).split('/') + const pathFromDocsDir = dirs.slice(1).join('/') + */ + +module.exports = parseDocsZip diff --git a/website/components/remote-plugin-docs/utils/parse-source-zip.js b/website/components/remote-plugin-docs/utils/parse-source-zip.js new file mode 100644 index 000000000..07a26ad6a --- /dev/null +++ b/website/components/remote-plugin-docs/utils/parse-source-zip.js @@ -0,0 +1,49 @@ +const path = require('path') +const AdmZip = require('adm-zip') +const validatePluginDocsFiles = require('./validate-plugin-docs-files') + +// Given a response from fetching a source .zip file, +// which contains a "docs" folder, +// +// return [null, docsMdxFiles] if docs files +// are successfully fetched and valid, +// where docsMdxFiles is an array of { filePath, fileString } items. +// +// otherwise, return [err, null] +// where err is an error message describing whether the +// docs files were missing or invalid, with a path to resolution +async function parseSourceZip(response) { + const responseBuffer = Buffer.from(await response.arrayBuffer()) + const responseZip = new AdmZip(responseBuffer) + const sourceEntries = responseZip.getEntries() + const docsEntries = sourceEntries.filter((entry) => { + // filter for zip entries in the docs subfolder only + const dirs = path.dirname(entry.entryName).split('/') + return dirs.length > 1 && dirs[1] === 'docs' + }) + // Validate the file paths within the "docs" folder + const docsFilePaths = docsEntries.map((e) => { + // We need to remove the leading directory, + // which will be something like packer-plugin-docs-0.0.5 + const filePath = e.entryName.split('/').slice(1).join('/') + return filePath + }) + const validationError = validatePluginDocsFiles(docsFilePaths) + if (validationError) return [validationError, null] + // If valid, filter for MDX files only, and return + // a { filePath, fileString } object for each mdx file + const docsMdxFiles = docsEntries + .filter((e) => { + return path.extname(e.entryName) === '.mdx' + }) + .map((e) => { + // We need to remove the leading directory, + // which will be something like packer-plugin-docs-0.0.5 + const filePath = e.entryName.split('/').slice(1).join('/') + const fileString = e.getData().toString() + return { filePath, fileString } + }) + return [null, docsMdxFiles] +} + +module.exports = parseSourceZip diff --git a/website/components/remote-plugin-docs/utils/resolve-nav-data.js b/website/components/remote-plugin-docs/utils/resolve-nav-data.js new file mode 100644 index 000000000..837feb6b4 --- /dev/null +++ b/website/components/remote-plugin-docs/utils/resolve-nav-data.js @@ -0,0 +1,218 @@ +const fs = require('fs') +const path = require('path') +const grayMatter = require('gray-matter') +const fetchPluginDocs = require('./fetch-plugin-docs') +const validateFilePaths = require('@hashicorp/react-docs-sidenav/utils/validate-file-paths') +const validateRouteStructure = require('@hashicorp/react-docs-sidenav/utils/validate-route-structure') + +/** + * Resolves nav-data from file, including optional + * resolution of remote plugin docs entries + * + * @param {string} navDataFile path to the nav-data.json file, relative to the cwd. Example: "data/docs-nav-data.json". + * @param {string} localContentDir path to the content root, relative to the cwd. Example: "content/docs". + * @param {object} options optional configuration object + * @param {string} options.remotePluginsFile path to a remote-plugins.json file, relative to the cwd. Example: "data/docs-remote-plugins.json". + * @returns {array} the resolved navData. This includes NavBranch nodes pulled from remote plugin repositories, as well as filePath properties on all local NavLeaf nodes, and remoteFile properties on all NavLeafRemote nodes. + */ +async function resolveNavData(navDataFile, localContentDir, options = {}) { + const { remotePluginsFile } = options + // Read in files + const navDataPath = path.join(process.cwd(), navDataFile) + const navData = JSON.parse(fs.readFileSync(navDataPath, 'utf8')) + // Fetch remote plugin docs, if applicable + let withPlugins = navData + if (remotePluginsFile) { + // Resolve plugins, this yields branches with NavLeafRemote nodes + withPlugins = await mergeRemotePlugins(remotePluginsFile, navData) + } + // Resolve local filePaths for NavLeaf nodes + const withFilePaths = await validateFilePaths(withPlugins, localContentDir) + validateRouteStructure(withFilePaths) + // Return the nav data with: + // 1. Plugins merged, transformed into navData structures with NavLeafRemote nodes + // 2. filePaths added to all local NavLeaf nodes + return withFilePaths +} + +// Given a remote plugins config file, and the full tree of docs navData which +// contains top-level branch routes that match plugin component types, +// fetch and parse all remote plugin docs, merge them into the +// broader tree of docs navData, and return the docs navData +// with the merged plugin docs +async function mergeRemotePlugins(remotePluginsFile, navData) { + // Read in and parse the plugin configuration JSON + const remotePluginsPath = path.join(process.cwd(), remotePluginsFile) + const pluginEntries = JSON.parse(fs.readFileSync(remotePluginsPath, 'utf-8')) + // Add navData for each plugin's component. + // Note that leaf nodes include a remoteFile property object with the full MDX fileString + const pluginEntriesWithDocs = await Promise.all( + pluginEntries.map(resolvePluginEntryDocs) + ) + // group navData by component type, to prepare to merge plugin docs + // into the broader tree of navData. + const pluginDocsByComponent = pluginEntriesWithDocs.reduce( + (acc, pluginEntry) => { + const { components } = pluginEntry + Object.keys(components).forEach((type) => { + const navData = components[type] + if (!navData) return + if (!acc[type]) acc[type] = [] + acc[type].push(navData[0]) + }) + return acc + }, + {} + ) + // merge plugin docs, by plugin component type, + // into the corresponding top-level component NavBranch + const navDataWithPlugins = navData.slice().map((n) => { + // we only care about top-level NavBranch nodes + if (!n.routes) return n + // for each component type, check if this NavBranch + // is the parent route for that type + const componentTypes = Object.keys(pluginDocsByComponent) + let typeMatch = false + for (var i = 0; i < componentTypes.length; i++) { + const componentType = componentTypes[i] + const routeMatches = n.routes.filter((r) => r.path === componentType) + if (routeMatches.length > 0) { + typeMatch = componentType + break + } + } + // if this NavBranch does not match a component type slug, + // then return it unmodified + if (!typeMatch) return n + // if there are no matching remote plugin components, + // then return the navBranch unmodified + const pluginsOfType = pluginDocsByComponent[typeMatch] + if (!pluginsOfType || pluginsOfType.length == 0) return n + // if this NavBranch is the parent route for the type, + // then append all remote plugins of this type to the + // NavBranch's child routes + const routesWithPlugins = n.routes.slice().concat(pluginsOfType) + // console.log(JSON.stringify(routesWithPlugins, null, 2)) + // Also, sort the child routes so the order is alphabetical + routesWithPlugins.sort((a, b) => { + // (exception: "Overview" comes first) + if (a.title == 'Overview') return -1 + if (b.title === 'Overview') return 1 + // (exception: "Community-Supported" comes last) + if (a.title == 'Community-Supported') return 1 + if (b.title === 'Community-Supported') return -1 + // (exception: "Custom" comes second-last) + if (a.title == 'Custom') return 1 + if (b.title === 'Custom') return -1 + return a.title < b.title ? -1 : a.title > b.title ? 1 : 0 + }) + // return n + return { ...n, routes: routesWithPlugins } + }) + // return the merged navData, which now includes special NavLeaf nodes + // for plugin docs with { filePath, fileString } remoteFile properties + return navDataWithPlugins +} + +// Fetch remote plugin docs .mdx files, and +// transform each plugin's array of .mdx files into navData. +// Organize this navData by component, add it to the plugin config entry, +// and return the modified entry. +// +// Note that navData leaf nodes have a special remoteFile property, +// which contains { filePath, fileString } data for the remote +// plugin doc .mdx file +async function resolvePluginEntryDocs(pluginConfigEntry) { + const { title, path: slug, repo, version } = pluginConfigEntry + const docsMdxFiles = await fetchPluginDocs({ repo, tag: version }) + // We construct a special kind of "NavLeaf" node, with a remoteFile property, + // consisting of a { filePath, fileString, sourceUrl }, where: + // - filePath is the path to the source file in the source repo + // - fileString is a string representing the file source + // - sourceUrl is a link to the original file in the source repo + // We also add a pluginTier attribute + const navNodes = docsMdxFiles.map((mdxFile) => { + const { filePath, fileString } = mdxFile + // Process into a NavLeaf, with a remoteFile attribute + const dirs = path.dirname(filePath).split('/') + const dirUrl = dirs.slice(2).join('/') + const basename = path.basename(filePath) + // build urlPath + // note that this will be prefixed to get to our final path + const isIndexFile = basename === 'index' + const urlPath = isIndexFile ? dirUrl : path.join(dirUrl, basename) + // parse title, either from frontmatter or file name + const { data: frontmatter } = grayMatter(fileString) + const { nav_title, sidebar_title } = frontmatter + const title = nav_title || sidebar_title || basename + // construct sourceUrl + const sourceUrl = `https://github.com/${repo}/blob/${version}/${filePath}` + // determine pluginTier + const pluginOwner = repo.split('/')[0] + const pluginTier = pluginOwner === 'hashicorp' ? 'official' : 'community' + // Construct and return a NavLeafRemote node + return { + title, + path: urlPath, + remoteFile: { filePath, fileString, sourceUrl }, + pluginTier, + } + }) + // + const navNodesByComponent = navNodes.reduce((acc, navLeaf) => { + const componentType = navLeaf.remoteFile.filePath.split('/')[1] + if (!acc[componentType]) acc[componentType] = [] + acc[componentType].push(navLeaf) + return acc + }, {}) + // + const components = Object.keys(navNodesByComponent).map((type) => { + // Plugins many not contain every component type, + // we return null if this is the case + const rawNavNodes = navNodesByComponent[type] + if (!rawNavNodes) return null + // Avoid unnecessary nesting if there's only a single doc file + const navData = normalizeNavNodes(title, rawNavNodes) + // Prefix paths to fit into broader docs nav-data + const pathPrefix = path.join(type, slug) + const withPrefixedPaths = visitNavLeaves(navData, (n) => { + const prefixedPath = path.join(pathPrefix, n.path) + return { ...n, path: prefixedPath } + }) + // + return { type, navData: withPrefixedPaths } + }) + const componentsObj = components.reduce((acc, component) => { + if (!component) return acc + acc[component.type] = component.navData + return acc + }, {}) + return { ...pluginConfigEntry, components: componentsObj } +} + +// For components with a single doc file, transform so that +// a single leaf node renders, rather than a nav branch +function normalizeNavNodes(pluginName, routes) { + const isSingleLeaf = + routes.length === 1 && typeof routes[0].path !== 'undefined' + const navData = isSingleLeaf + ? [{ ...routes[0], path: '' }] + : [{ title: pluginName, routes }] + return navData +} + +// Traverse a clone of the given navData, +// modifying any NavLeaf nodes with the provided visitFn +function visitNavLeaves(navData, visitFn) { + return navData.slice().map((navNode) => { + if (typeof navNode.path !== 'undefined') { + return visitFn(navNode) + } + if (navNode.routes) { + return { ...navNode, routes: visitNavLeaves(navNode.routes, visitFn) } + } + return navNode + }) +} + +module.exports = resolveNavData diff --git a/website/components/remote-plugin-docs/utils/validate-plugin-docs-files.js b/website/components/remote-plugin-docs/utils/validate-plugin-docs-files.js new file mode 100644 index 000000000..e846092a9 --- /dev/null +++ b/website/components/remote-plugin-docs/utils/validate-plugin-docs-files.js @@ -0,0 +1,46 @@ +const path = require('path') + +const COMPONENT_TYPES = [ + 'builders', + 'datasources', + 'post-processors', + 'provisioners', +] + +// Given an array of file paths within the "docs" folder, +// validate that no unexpected files are being included, +// and that there is at least one component subfolder +// with at least one .mdx file within it. +function validatePluginDocsFiles(filePaths) { + function isValidPath(filePath) { + const isDocsRoot = filePath === 'docs/' + const isComponentRoot = COMPONENT_TYPES.reduce((acc, type) => { + return acc || filePath === `docs/${type}/` + }, false) + const isComponentMdx = COMPONENT_TYPES.reduce((acc, type) => { + const mdxPathRegex = new RegExp(`^docs/${type}/(.*).mdx$`) + return acc || mdxPathRegex.test(filePath) + }, false) + const isValidPath = isDocsRoot || isComponentRoot || isComponentMdx + return isValidPath + } + const invalidPaths = filePaths.filter((f) => !isValidPath(f)) + if (invalidPaths.length > 0) { + return `Found invalid files or folders in the docs directory: ${JSON.stringify( + invalidPaths + )}. Please ensure the docs folder contains only component subfolders and .mdx files within those subfolders. Valid component types are: ${JSON.stringify( + COMPONENT_TYPES + )}.` + } + const validPaths = filePaths.filter(isValidPath) + const mdxFiles = validPaths.filter((fp) => path.extname(fp) === '.mdx') + const isMissingDocs = mdxFiles.length == 0 + if (isMissingDocs) { + return `Could not find valid .mdx files. Please ensure there is at least one component subfolder in the docs directory, which contains at least one .mdx file. Valid component types are: ${JSON.stringify( + COMPONENT_TYPES + )}.` + } + return null +} + +module.exports = validatePluginDocsFiles