website: check for unlinked content, rm split-out vagrant content (#10958)

This commit is contained in:
Zachary Shilton 2021-04-23 11:04:20 -04:00 committed by GitHub
parent 0bd7b20bb8
commit 95ed4443bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 160 additions and 483 deletions

View File

@ -0,0 +1,134 @@
var fs = require("fs");
var path = require("path");
const COLOR_RESET = "\x1b[0m";
const COLOR_GREEN = "\x1b[32m";
const COLOR_RED = "\x1b[31m";
runCheck([
{
contentDir: "website/content/docs",
navDataFile: "website/data/docs-nav-data.json",
},
{
contentDir: "website/content/guides",
navDataFile: "website/data/guides-nav-data.json",
},
{
contentDir: "website/content/intro",
navDataFile: "website/data/intro-nav-data.json",
},
]);
async function runCheck(baseRoutes) {
const validatedBaseRoutes = await Promise.all(
baseRoutes.map(async ({ contentDir, navDataFile }) => {
const missingRoutes = await validateMissingRoutes(
contentDir,
navDataFile
);
return { contentDir, navDataFile, missingRoutes };
})
);
const allMissingRoutes = validatedBaseRoutes.reduce((acc, baseRoute) => {
return acc.concat(baseRoute.missingRoutes);
}, []);
if (allMissingRoutes.length == 0) {
console.log(
`\n${COLOR_GREEN}✓ All content files have routes, and are included in navigation data.${COLOR_RESET}\n`
);
} else {
validatedBaseRoutes.forEach(
({ contentDir, navDataFile, missingRoutes }) => {
if (missingRoutes.length == 0) return true;
console.log(
`\n${COLOR_RED}Error: Missing pages found in the ${contentDir} directory.\n\nPlease add these paths to ${navDataFile}, or remove the .mdx files.\n\n${JSON.stringify(
missingRoutes,
null,
2
)}${COLOR_RESET}\n\n`
);
}
);
process.exit(1);
}
}
async function validateMissingRoutes(contentDir, navDataFile) {
// Read in nav-data.json, and make a flattened array of nodes
const navDataPath = path.join(process.cwd(), navDataFile);
const navData = JSON.parse(fs.readFileSync(navDataPath));
const navDataFlat = flattenNodes(navData);
// Read all files in the content directory
const files = await walkAsync(contentDir);
// Filter out content files that are already
// included in nav-data.json
const missingPages = files
// Ignore non-.mdx files
.filter((filePath) => {
return path.extname(filePath) == ".mdx";
})
// Transform the filePath into an expected route
.map((filePath) => {
// Get the relative filepath, that's what we'll see in the route
const contentDirPath = path.join(process.cwd(), contentDir);
const relativePath = path.relative(contentDirPath, filePath);
// Remove extensions, these will not be in routes
const pathNoExt = relativePath.replace(/\.mdx$/, "");
// Resolve /index routes, these will not have /index in their path
const routePath = pathNoExt.replace(/\/?index$/, "");
return routePath;
})
// Determine if there is a match in nav-data.
// If there is no match, then this is an unlinked content file.
.filter((pathToMatch) => {
// If it's the root path index page, we know
// it'll be rendered (hard-coded into docs-page/server.js)
const isIndexPage = pathToMatch === "";
if (isIndexPage) return false;
// Otherwise, needs a path match in nav-data
const matches = navDataFlat.filter(({ path }) => path == pathToMatch);
return matches.length == 0;
});
return missingPages;
}
function flattenNodes(nodes) {
return nodes.reduce((acc, n) => {
if (!n.routes) return acc.concat(n);
return acc.concat(flattenNodes(n.routes));
}, []);
}
function walkAsync(relativeDir) {
const dirPath = path.join(process.cwd(), relativeDir);
return new Promise((resolve, reject) => {
walk(dirPath, function (err, result) {
if (err) reject(err);
resolve(result);
});
});
}
function walk(dir, done) {
var results = [];
fs.readdir(dir, function (err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function (file) {
file = path.resolve(dir, file);
fs.stat(file, function (err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function (err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
results.push(file);
if (!--pending) done(null, results);
}
});
});
});
}

View File

@ -0,0 +1,26 @@
#
# This GitHub action checks that all .mdx files in the
# the website/content directory are being published.
# It fails if any of these files are not included
# in the expected nav-data.json file.
#
# To resolve failed checks, add the listed paths
# to the corresponding nav-data.json file
# in website/data.
name: "website: Check unlinked content"
on:
pull_request:
paths:
- "website/**"
jobs:
check-unlinked-content:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node
uses: actions/setup-node@v1
- name: Check that all content files are included in navigation
run: node .github/workflows/check-unlinked-content.js

View File

@ -1,211 +0,0 @@
---
description: |
The Vagrant Cloud post-processor enables the upload of Vagrant boxes to
Vagrant Cloud.
page_title: Vagrant Cloud - Post-Processors
---
# Vagrant Cloud Post-Processor
Type: `vagrant-cloud`
Artifact BuilderId: `pearkes.post-processor.vagrant-cloud`
[Vagrant Cloud](https://app.vagrantup.com/boxes/search) hosts and serves boxes
to Vagrant, allowing you to version and distribute boxes to an organization in a
simple way.
The Vagrant Cloud post-processor enables the upload of Vagrant boxes to Vagrant
Cloud. Currently, the Vagrant Cloud post-processor will accept and upload boxes
supplied to it from the [Vagrant](/docs/post-processors/vagrant) or
[Artifice](/docs/post-processors/artifice) post-processors and the
[Vagrant](/docs/builders/vagrant) builder.
You'll need to be familiar with Vagrant Cloud, have an upgraded account to
enable box hosting, and be distributing your box via the [shorthand
name](https://docs.vagrantup.com/v2/cli/box.html) configuration.
## Workflow
It's important to understand the workflow that using this post-processor
enforces in order to take full advantage of Vagrant and Vagrant Cloud.
The use of this processor assume that you currently distribute, or plan to
distribute, boxes via Vagrant Cloud. It also assumes you create Vagrant Boxes
and deliver them to your team in some fashion.
Here is an example workflow:
1. You use Packer to build a Vagrant Box for the `virtualbox` provider
2. The `vagrant-cloud` post-processor is configured to point to the box
`hashicorp/foobar` on Vagrant Cloud via the `box_tag` configuration
3. The post-processor receives the box from the `vagrant` post-processor
4. It then creates the configured version, or verifies the existence of it, on
Vagrant Cloud
5. A provider matching the name of the Vagrant provider is then created
6. The box is uploaded to Vagrant Cloud
7. The upload is verified
8. The version is released and available to users of the box
~> The Vagrant Cloud box (`hashicorp/foobar` in this example) must already
exist. Packer will not create the box automatically. If running Packer in
automation, consider using the
[Vagrant Cloud API](https://www.vagrantup.com/docs/vagrant-cloud/api.html)
to create the Vagrant Cloud box if it doesn't already exist.
## Configuration
The configuration allows you to specify the target box that you have access to
on Vagrant Cloud, as well as authentication and version information.
### Required:
- `box_tag` (string) - The shorthand tag for your box that maps to Vagrant
Cloud, for example `hashicorp/precise64`, which is short for
`vagrantcloud.com/hashicorp/precise64`. This box must already exist in
Vagrant Cloud. Packer will not create the box automatically.
- `version` (string) - The version number, typically incrementing a previous
version. The version string is validated based on [Semantic
Versioning](http://semver.org/). The string must match a pattern that could
be semver, and doesn't validate that the version comes after your previous
versions.
- `access_token` (string) - Your access token for the Vagrant Cloud API. This
can be generated on your [tokens
page](https://app.vagrantup.com/settings/security). If not specified, the
environment will be searched. First, `VAGRANT_CLOUD_TOKEN` is checked, and
if nothing is found, finally `ATLAS_TOKEN` will be used. This is required
unless you are using a private hosting solution (i.e. `vagrant_cloud_url`
has been populated).
**or**
- `vagrant_cloud_url` (string) - Override the base URL for Vagrant Cloud.
This is useful if you're using Vagrant Private Cloud in your own network.
Defaults to `https://vagrantcloud.com/api/v1`. If this value is set to something
other than the default then `access_token` can be left blank and no
`Authorization` header will be added to requests sent by this post-processor.
### Optional:
- `no_release` (string) - If set to true, does not release the version on
Vagrant Cloud, making it active. You can manually release the version via
the API or Web UI. Defaults to `false`.
- `insecure_skip_tls_verify` (boolean) - If set to true _and_ `vagrant_cloud_url`
is set to something different than its default, it will set TLS InsecureSkipVerify
to true. In other words, this will disable security checks of SSL. You may need
to set this option to true if your host at `vagrant_cloud_url` is using a
self-signed certificate.
- `keep_input_artifact` (boolean) - When true, preserve the local box
after uploading to Vagrant cloud. Defaults to `true`.
- `version_description` (string) - Optional Markdown text used as a
full-length and in-depth description of the version, typically for denoting
changes introduced
- `box_download_url` (string) - Optional URL for a self-hosted box. If this
is set the box will not be uploaded to the Vagrant Cloud.
This is a [template engine](/docs/templates/legacy_json_templates/engine). Therefore, you
may use user variables and template functions in this field.
The following extra variables are also available in this engine:
- `Provider`: The Vagrant provider the box is for
- `ArtifactId`: The ID of the input artifact.
- `no_direct_upload` (boolean) - When `true`, upload the box artifact through
Vagrant Cloud instead of directly to the backend storage.
## Use with the Vagrant Post-Processor
An example configuration is shown below. Note the use of the nested array that
wraps both the Vagrant and Vagrant Cloud post-processors within the
post-processor section. Chaining the post-processors together in this way tells
Packer that the artifact produced by the Vagrant post-processor should be passed
directly to the Vagrant Cloud Post-Processor. It also sets the order in which
the post-processors should run.
Failure to chain the post-processors together in this way will result in the
wrong artifact being supplied to the Vagrant Cloud post-processor. This will
likely cause the Vagrant Cloud post-processor to error and fail.
```json
{
"variables": {
"cloud_token": "{{ env `VAGRANT_CLOUD_TOKEN` }}",
"version": "1.0.{{timestamp}}"
},
"post-processors": [
{
"type": "shell-local",
"inline": ["echo Doing stuff..."]
},
[
{
"type": "vagrant",
"include": ["image.iso"],
"vagrantfile_template": "vagrantfile.tpl",
"output": "proxycore_{{.Provider}}.box"
},
{
"type": "vagrant-cloud",
"box_tag": "hashicorp/precise64",
"access_token": "{{user `cloud_token`}}",
"version": "{{user `version`}}"
}
]
]
}
```
## Use with the Artifice Post-Processor
An example configuration is shown below. Note the use of the nested array that
wraps both the Artifice and Vagrant Cloud post-processors within the
post-processor section. Chaining the post-processors together in this way tells
Packer that the artifact produced by the Artifice post-processor should be
passed directly to the Vagrant Cloud Post-Processor. It also sets the order in
which the post-processors should run.
Failure to chain the post-processors together in this way will result in the
wrong artifact being supplied to the Vagrant Cloud post-processor. This will
likely cause the Vagrant Cloud post-processor to error and fail.
Note that the Vagrant box specified in the Artifice post-processor `files` array
must end in the `.box` extension. It must also be the first file in the array.
Additional files bundled by the Artifice post-processor will be ignored.
```json
{
"variables": {
"cloud_token": "{{ env `VAGRANT_CLOUD_TOKEN` }}"
},
"builders": [
{
"type": "null",
"communicator": "none"
}
],
"post-processors": [
{
"type": "shell-local",
"inline": ["echo Doing stuff..."]
},
[
{
"type": "artifice",
"files": ["./path/to/my.box"]
},
{
"type": "vagrant-cloud",
"box_tag": "myorganisation/mybox",
"access_token": "{{user `cloud_token`}}",
"version": "0.1.0"
}
]
]
}
```

View File

@ -1,272 +0,0 @@
---
description: >
The Packer Vagrant post-processor takes a build and converts the artifact into
a valid Vagrant box, if it can. This lets you use Packer to automatically
create arbitrarily complex Vagrant boxes, and is in fact how the official
boxes
distributed by Vagrant are created.
page_title: Vagrant - Post-Processors
---
# Vagrant Post-Processor
Type: `vagrant`
Artifact BuilderId: `mitchellh.post-processor.vagrant`
The Packer Vagrant post-processor takes a build and converts the artifact into
a valid [Vagrant](https://www.vagrantup.com) box, if it can. This lets you use
Packer to automatically create arbitrarily complex Vagrant boxes, and is in
fact how the official boxes distributed by Vagrant are created.
If you've never used a post-processor before, please read the documentation on
[using post-processors](/docs/templates/legacy_json_templates/post-processors) in templates.
This knowledge will be expected for the remainder of this document.
Because Vagrant boxes are
[provider-specific](https://www.vagrantup.com/docs/boxes/format), the
Vagrant post-processor is hardcoded to understand how to convert the artifacts
of certain builders into proper boxes for their respective providers.
Currently, the Vagrant post-processor can create boxes for the following
providers.
- AWS
- Azure
- DigitalOcean
- Docker
- Hyper-V
- LXC
- Parallels
- QEMU
- VirtualBox
- VMware
-> **Support for additional providers** is planned. If the Vagrant
post-processor doesn't support creating boxes for a provider you care about,
please help by contributing to Packer and adding support for it.
Please note that if you are using the Vagrant builder, then the Vagrant
post-processor is unnecessary because the output of the Vagrant builder is
already a Vagrant box; using this post-processor with the Vagrant builder will
cause your build to fail.
## Configuration
The simplest way to use the post-processor is to just enable it. No
configuration is required by default. This will mostly do what you expect and
will build functioning boxes for many of the built-in builders of Packer.
However, if you want to configure things a bit more, the post-processor does
expose some configuration options. The available options are listed below, with
more details about certain options in following sections.
- `compression_level` (number) - An integer representing the compression
level to use when creating the Vagrant box. Valid values range from 0 to 9,
with 0 being no compression and 9 being the best compression. By default,
compression is enabled at level 6.
- `include` (array of strings) - Paths to files to include in the Vagrant
box. These files will each be copied into the top level directory of the
Vagrant box (regardless of their paths). They can then be used from the
Vagrantfile.
- `keep_input_artifact` (boolean) - When true, preserve the artifact we use to
create the vagrant box. Defaults to `false`, except when you set a cloud
provider (e.g. aws, azure, google, digitalocean). In these cases deleting
the input artifact would render the vagrant box useless, so we always keep
these artifacts -- even if you specifically set
`"keep_input_artifact":false`
- `output` (string) - The full path to the box file that will be created by
this post-processor. This is a
[template engine](/docs/templates/legacy_json_templates/engine). Therefore, you may use user
variables and template functions in this field. The following extra
variables are also available in this engine:
- `Provider`: The Vagrant provider the box is for
- `ArtifactId`: The ID of the input artifact.
- `BuildName`: The name of the build.
By default, the value of this config is
`packer_{{.BuildName}}_{{.Provider}}.box`.
- `provider_override` (string) - this option will override the internal logic
that decides which Vagrant provider to set for a particular Packer builder's
or post-processor's artifact. It is required when the artifact comes from the
Artifice post-processor, but is otherwise optional. Valid options are:
`digitalocean`, `virtualbox`, `azure`, `vmware`, `libvirt`, `docker`,
`lxc`, `scaleway`, `hyperv`, `parallels`, `aws`, or `google`.
- `vagrantfile_template` (string) - Path to a template to use for the
Vagrantfile that is packaged with the box. This option supports the usage of the [template engine](/docs/templates/legacy_json_templates/engine)
for JSON and the [contextual variables](/docs/templates/hcl_templates/contextual-variables) for HCL2.
- `vagrantfile_template_generated` (boolean) - By default, Packer will
exit with an error if the file specified using the
`vagrantfile_template` variable is not found. However, under certain
circumstances, it may be desirable to dynamically generate the
Vagrantfile during the course of the build. Setting this variable to
`true` skips the start up check and allows the user to script the
creation of the Vagrantfile at some previous point in the build.
Defaults to `false`.
## Using together with the Artifice post-processor
Sometimes you may want to run several builds in a pipeline rather than running
this post-processor inside a long-running Packer build. Here is an example of
how to do this:
<Tabs>
<Tab heading="JSON">
```json
{
"builders": [
{
"type": "null",
"communicator": "none"
}
],
"post-processors": [
[
{
"type": "artifice",
"files": [
"output-virtualbox-iso/vbox-example-disk001.vmdk",
"output-virtualbox-iso/vbox-example.ovf"
]
},
{
"type": "vagrant",
"keep_input_artifact": true,
"provider_override": "virtualbox"
}
]
]
}
```
</Tab>
<Tab heading="HCL2">
```hcl
source "null" "example" {
communicator = "none"
}
build {
sources = [
"source.null.example"
]
post-processor "artifice" {
files = ["output-virtualbox-iso/vbox-example-disk001.vmdk",
"output-virtualbox-iso/vbox-example.ovf"]
}
post-processor "vagrant" {
keep_input_artifact = true
provider_override = "virtualbox"
}
}
```
</Tab>
</Tabs>
## Provider-Specific Overrides
If you have a Packer template with multiple builder types within it, you may
want to configure the box creation for each type a little differently. For
example, the contents of the Vagrantfile for a Vagrant box for AWS might be
different from the contents of the Vagrantfile you want for VMware. The
post-processor lets you do this.
Specify overrides within the `override` configuration by provider name:
<Tabs>
<Tab heading="JSON">
```json
{
"type": "vagrant",
"compression_level": 1,
"override": {
"vmware": {
"compression_level": 0
}
}
}
```
</Tab>
<Tab heading="HCL2">
```hcl
## This feature is not implemented in HCL.
```
</Tab>
</Tabs>
In the example above, the compression level will be set to 1 except for VMware,
where it will be set to 0.
The available provider names are:
- `aws`
- `azure`
- `digitalocean`
- `google`
- `hyperv`
- `parallels`
- `libvirt`
- `lxc`
- `scaleway`
- `virtualbox`
- `vmware`
- `docker`
## Input Artifacts
By default, Packer will delete the original input artifact, assuming you only
want the final Vagrant box as the result. If you wish to keep the input
artifact (the raw virtual machine, for example), then you must configure Packer
to keep it.
Please see the [documentation on input
artifacts](/docs/templates/legacy_json_templates/post-processors#input-artifacts) for more information.
### Docker
Using a Docker input artifact will include a reference to the image in the
`Vagrantfile`. If the image tag is not specified in the post-processor, the
sha256 hash will be used.
The following Docker input artifacts are supported:
- `docker` builder with `commit: true`, always uses the sha256 hash
- `docker-import`
- `docker-tag`
- `docker-push`
### QEMU/libvirt
The `libvirt` provider supports QEMU artifacts built using any these
accelerators: none, kvm, tcg, or hvf.
### VMWare
If you are using the Vagrant post-processor with the `vmware-esxi` builder, you
must export the builder artifact locally; the Vagrant post-processor will
not work on remote artifacts.
### Artifice
If you are using this post-processor after defining an artifact using the
Artifice post-processor, then you must set the "provider_override" template
option so that the Vagrant post-processor knows what provider to use to create
the Vagrant box.