Add Algolia indexing to CI
This commit is contained in:
Kevin Pruett
parent
3dc7a53125
commit
4f59693be8
|
|
@ -172,6 +172,21 @@ jobs:
|
|||
docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS
|
||||
docker push hashicorp/packer-website
|
||||
fi
|
||||
algolia-index:
|
||||
docker:
|
||||
- image: node:12
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Push content to Algolia Index
|
||||
command: |
|
||||
if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/packer.git" ]; then
|
||||
echo "Not Packer OSS Repo, not indexing Algolia"
|
||||
exit 0
|
||||
fi
|
||||
cd website/
|
||||
npm install
|
||||
node scripts/index_search_content.js
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
|
|
@ -202,10 +217,15 @@ workflows:
|
|||
- build_freebsd
|
||||
- build_openbsd
|
||||
- build_solaris
|
||||
build_website_docker_image:
|
||||
website:
|
||||
jobs:
|
||||
- build-website-docker-image:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- algolia-index:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- stable-website
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
NEXT_PUBLIC_ALGOLIA_APP_ID=YY0FFNI7MF
|
||||
NEXT_PUBLIC_ALGOLIA_INDEX=product_PACKER
|
||||
NEXT_PUBLIC_ALGOLIA_SEARCH_ONLY_API_KEY=5037da4824714676226913c65e961ca0
|
||||
|
|
@ -3,3 +3,7 @@ node_modules
|
|||
.next
|
||||
out
|
||||
.mdx-data
|
||||
|
||||
# As per Next.js conventions (https://nextjs.org/docs/basic-features/environment-variables#default-environment-variables)
|
||||
!.env
|
||||
.env*.local
|
||||
|
|
|
|||
|
|
@ -4,6 +4,121 @@
|
|||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@algolia/cache-browser-local-storage": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.3.0.tgz",
|
||||
"integrity": "sha512-91Cf3IPUk84PF2wvR8ys8XO42FqaJEtIh/dyR0WvwMdv0x13GORkAvoBJgkFI2wofZqUY86jNimvHWfsWzPQ+g==",
|
||||
"requires": {
|
||||
"@algolia/cache-common": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/cache-common": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.3.0.tgz",
|
||||
"integrity": "sha512-AHTbOn9lk0f5IkjssXXmDgnaZfsUJVZ61sqOH1W3LyJdAscDzCj0KtwijELn8FHlLXQak7+K93/O3Oct0uHncQ=="
|
||||
},
|
||||
"@algolia/cache-in-memory": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.3.0.tgz",
|
||||
"integrity": "sha512-8BZS5IFEtiSFkA6vNQUXJXIWABDbSanQdkGX5LArlhbCjuykZqF68yaCjXWG10EZTySnkZLmKc+5ozYVOktJaQ==",
|
||||
"requires": {
|
||||
"@algolia/cache-common": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/client-account": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.3.0.tgz",
|
||||
"integrity": "sha512-8LJSvWooc+fe+XZXeu+h4dhpo9lsu3sb7rV9cpPhymYSHgEJAHaDkZEcPM1u/PBMvFe0mZXaW6nabeb3jeIRcw==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.3.0",
|
||||
"@algolia/client-search": "4.3.0",
|
||||
"@algolia/transporter": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/client-analytics": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.3.0.tgz",
|
||||
"integrity": "sha512-BFH4ddyrqI2pE3bUctn5KtJgYqgvO0Ap9vJEHBNj6mjSKqFbTnZeVEPG3yWrOuWRCqPHR3ewcWRisNwJHG3+Mw==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.3.0",
|
||||
"@algolia/client-search": "4.3.0",
|
||||
"@algolia/requester-common": "4.3.0",
|
||||
"@algolia/transporter": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/client-common": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.3.0.tgz",
|
||||
"integrity": "sha512-8Ohj6zXZkpwDKc8ZWVTZo2wPO4+LT5D258suGg/C6nh4UxOrFOp6QaqeQo8JZ1eqMqtfb3zv5SHgW4fZ00NCLQ==",
|
||||
"requires": {
|
||||
"@algolia/requester-common": "4.3.0",
|
||||
"@algolia/transporter": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/client-recommendation": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-recommendation/-/client-recommendation-4.3.0.tgz",
|
||||
"integrity": "sha512-jCMIAWPA2hsxc5CCtoTtQAcohaG+10CxXK122Tc47t4w1K8qzSJnCjC2cHvM4UNJO+k7NrmjOYW0EXp9RKc7SQ==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.3.0",
|
||||
"@algolia/requester-common": "4.3.0",
|
||||
"@algolia/transporter": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/client-search": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.3.0.tgz",
|
||||
"integrity": "sha512-KCgcIsNMW1/0F5OILiFTddbTAKduJHRvXQS4NxY1H9gQWMTVeWJS7VZQ/ukKBiUMLatwUQHJz2qpYm9fmqOjkQ==",
|
||||
"requires": {
|
||||
"@algolia/client-common": "4.3.0",
|
||||
"@algolia/requester-common": "4.3.0",
|
||||
"@algolia/transporter": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/logger-common": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.3.0.tgz",
|
||||
"integrity": "sha512-vQ+aukjZkRAyO9iyINBefT366UtF/B9QoA1Kw8PlY67T6fYmklFgYp3LNH/e7h/gz0py5LYY/HIwSsaTKk8/VQ=="
|
||||
},
|
||||
"@algolia/logger-console": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.3.0.tgz",
|
||||
"integrity": "sha512-7pWtcv1cSSa7F48gRBOZLcEWN073+WbnKjbpRrIGej+abZppw/h+22jtVZZORC8EIjFffGqz2/2e6bZiX+Jg7A==",
|
||||
"requires": {
|
||||
"@algolia/logger-common": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/requester-browser-xhr": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.3.0.tgz",
|
||||
"integrity": "sha512-CpUwgQhXZsnZmjEd5DTwQv1BKQNCt83bzyVdUqvljsFxZOsNQacS6lOYs0B1eD18tKHCwVMuwbYqTaLPGBXTKQ==",
|
||||
"requires": {
|
||||
"@algolia/requester-common": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/requester-common": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.3.0.tgz",
|
||||
"integrity": "sha512-1v73KyspJBiTzfyXupjHxikxTYjh5MoxI6mOIvAtQxRqc4ehUPAEdPCNHEvvLiCK96iKWzZaULmV0U7pj3yvTw=="
|
||||
},
|
||||
"@algolia/requester-node-http": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.3.0.tgz",
|
||||
"integrity": "sha512-Hg9Y8sUeSGQgoO1FpoL5jbkDzCtXI/8HXHybU6bimsX93DAz3HZWaoQFKmIpQDNhQ8G9FLgAtzDAxS6eckDxzg==",
|
||||
"requires": {
|
||||
"@algolia/requester-common": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@algolia/transporter": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.3.0.tgz",
|
||||
"integrity": "sha512-BTKHAtdQdfOJ0xzZkiyEK/2QVQJTiVgBZlOBfXp2gBtztjV26OqfW4n6Xz0o7eBRzLEwY1ot3mHF5QIVUjAsMg==",
|
||||
"requires": {
|
||||
"@algolia/cache-common": "4.3.0",
|
||||
"@algolia/logger-common": "4.3.0",
|
||||
"@algolia/requester-common": "4.3.0"
|
||||
}
|
||||
},
|
||||
"@ampproject/toolbox-core": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@ampproject/toolbox-core/-/toolbox-core-2.5.0.tgz",
|
||||
|
|
@ -2546,6 +2661,27 @@
|
|||
"resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.4.1.tgz",
|
||||
"integrity": "sha512-RO1ibKvd27e6FEShVFfPALuHI3WjSVNeK5FIsmme/LYRNxjKuNj+Dt7bucLa6NdSv3JcVTyMlm9kGR84z1XpaQ=="
|
||||
},
|
||||
"algoliasearch": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.3.0.tgz",
|
||||
"integrity": "sha512-H2woXyqmd1nFYDrQKLZXgghNkLBTcBXJ7Q/bxQ+F9WWS4H0Kb7IlQvNi7bDzHyldhDhIthImaUwcKqr5iiyMFQ==",
|
||||
"requires": {
|
||||
"@algolia/cache-browser-local-storage": "4.3.0",
|
||||
"@algolia/cache-common": "4.3.0",
|
||||
"@algolia/cache-in-memory": "4.3.0",
|
||||
"@algolia/client-account": "4.3.0",
|
||||
"@algolia/client-analytics": "4.3.0",
|
||||
"@algolia/client-common": "4.3.0",
|
||||
"@algolia/client-recommendation": "4.3.0",
|
||||
"@algolia/client-search": "4.3.0",
|
||||
"@algolia/logger-common": "4.3.0",
|
||||
"@algolia/logger-console": "4.3.0",
|
||||
"@algolia/requester-browser-xhr": "4.3.0",
|
||||
"@algolia/requester-common": "4.3.0",
|
||||
"@algolia/requester-node-http": "4.3.0",
|
||||
"@algolia/transporter": "4.3.0"
|
||||
}
|
||||
},
|
||||
"ally.js": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/ally.js/-/ally.js-1.4.1.tgz",
|
||||
|
|
@ -5643,6 +5779,11 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"dotenv": {
|
||||
"version": "8.2.0",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
|
||||
"integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
|
||||
},
|
||||
"download": {
|
||||
"version": "6.2.5",
|
||||
"resolved": "https://registry.npmjs.org/download/-/download-6.2.5.tgz",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,10 @@
|
|||
"@hashicorp/react-section-header": "^2.0.0",
|
||||
"@hashicorp/react-subnav": "^3.2.2",
|
||||
"@hashicorp/react-vertical-text-block-list": "^2.0.1",
|
||||
"algoliasearch": "^4.3.0",
|
||||
"babel-plugin-import-glob-array": "^0.2.0",
|
||||
"dotenv": "^8.2.0",
|
||||
"gray-matter": "^4.0.2",
|
||||
"imagemin-mozjpeg": "^9.0.0",
|
||||
"imagemin-optipng": "^8.0.0",
|
||||
"imagemin-svgo": "^8.0.0",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
require('dotenv').config()
|
||||
|
||||
const algoliasearch = require('algoliasearch')
|
||||
const glob = require('glob')
|
||||
const matter = require('gray-matter')
|
||||
const path = require('path')
|
||||
|
||||
// In addition to the content of the page,
|
||||
// define additional front matter attributes that will be search-indexable
|
||||
const SEARCH_DIMENSIONS = ['page_title', 'description']
|
||||
|
||||
main()
|
||||
|
||||
async function main() {
|
||||
const pagesFolder = path.join(__dirname, '../pages')
|
||||
|
||||
// Grab all search-indexable content and format for Algolia
|
||||
const searchObjects = glob
|
||||
.sync(path.join(pagesFolder, '**/*.mdx'))
|
||||
.map((fullPath) => {
|
||||
const { content, data } = matter.read(fullPath)
|
||||
|
||||
// Get path relative to `pages`
|
||||
const __resourcePath = fullPath.replace(`${pagesFolder}/`, '')
|
||||
|
||||
// Use clean URL for Algolia id
|
||||
const objectID = __resourcePath.replace('.mdx', '')
|
||||
|
||||
const searchableDimensions = Object.keys(data)
|
||||
.filter((key) => SEARCH_DIMENSIONS.includes(key))
|
||||
.map((dimension) => ({
|
||||
[dimension]: data[dimension],
|
||||
}))
|
||||
|
||||
return {
|
||||
...searchableDimensions,
|
||||
content,
|
||||
__resourcePath,
|
||||
objectID,
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
await indexSearchContent(searchObjects)
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
async function indexSearchContent(objects) {
|
||||
const {
|
||||
NEXT_PUBLIC_ALGOLIA_APP_ID: appId,
|
||||
NEXT_PUBLIC_ALGOLIA_INDEX: index,
|
||||
ALGOLIA_API_KEY: apiKey,
|
||||
} = process.env
|
||||
|
||||
if (!apiKey || !appId || !index) {
|
||||
throw new Error(
|
||||
`[*** Algolia Search Indexing Error ***] Received: ALGOLIA_API_KEY=${apiKey} ALGOLIA_APP_ID=${appId} ALGOLIA_INDEX=${index} \n Please ensure all Algolia Search-related environment vars are set in CI settings.`
|
||||
)
|
||||
}
|
||||
|
||||
console.log(`updating ${objects.length} indices...`)
|
||||
|
||||
try {
|
||||
const searchClient = algoliasearch(appId, apiKey)
|
||||
const searchIndex = searchClient.initIndex(index)
|
||||
|
||||
await searchIndex.partialUpdateObjects(objects, {
|
||||
createIfNotExists: true,
|
||||
})
|
||||
|
||||
// Remove indices for items that aren't included in the new batch
|
||||
const newObjectIds = objects.map(({ objectID }) => objectID)
|
||||
let staleObjects = []
|
||||
|
||||
await searchIndex.browseObjects({
|
||||
query: '',
|
||||
batch: (batch) => {
|
||||
staleObjects = staleObjects.concat(
|
||||
batch.filter(({ objectID }) => !newObjectIds.includes(objectID))
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
const staleIds = staleObjects.map(({ objectID }) => objectID)
|
||||
|
||||
if (staleIds.length > 0) {
|
||||
console.log(`deleting ${staleIds.length} stale indices:`)
|
||||
console.log(staleIds)
|
||||
|
||||
await searchIndex.deleteObjects(staleIds)
|
||||
}
|
||||
|
||||
console.log('done')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
throw new Error(error)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue