Skip to content

Commit 486c362

Browse files
nickcerneranickcernera
authored
nickcernera/sitemap-cleanup (#411)
* chore: add proper sitemap generation, add lastmod to sitemap, prepare for algolia crawl * chore: add generate-pages script to yarn dev * test: testing script to detect file moves via git * enhancement: auto-detect reversions of redirects * chore: cleaning deprecated redirects * chore: lastmod changed from test
1 parent f67d967 commit 486c362

File tree

11 files changed

+402
-856
lines changed

11 files changed

+402
-856
lines changed

‎docsearch.json‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
{
2+
"index_name": "plural_docs",
3+
"start_urls": [
4+
"https://docs.plural.sh/"
5+
],
6+
"stop_urls": [
7+
"/pages-archive/",
8+
"/__components/"
9+
],
10+
"selectors": {
11+
"lvl0": {
12+
"selector": "h1",
13+
"global": true,
14+
"default_value": "Documentation"
15+
},
16+
"lvl1": "article h2",
17+
"lvl2": "article h3",
18+
"lvl3": "article h4",
19+
"lvl4": "article h5",
20+
"lvl5": "article h6",
21+
"text": "article p, article li, article td",
22+
"language": {
23+
"selector": "/html/@lang",
24+
"type": "xpath",
25+
"global": true
26+
}
27+
},
28+
"strip_chars": "0123456789-",
29+
"custom_settings": {
30+
"separatorsToIndex": "_",
31+
"attributesForFaceting": ["language", "version"],
32+
"attributesToRetrieve": [
33+
"hierarchy",
34+
"content",
35+
"anchor",
36+
"url",
37+
"url_without_anchor",
38+
"type"
39+
]
40+
},
41+
"js_render": false,
42+
"use_anchors": true,
43+
"selectors_exclude": [
44+
".DocSearch-content"
45+
],
46+
"scrape_start_urls": true,
47+
"min_indexed_level": 0
48+
}

‎index-pages.mjs‎

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Dirent } from 'fs'
1+
import { execSync } from 'child_process'
22
import { readdir, writeFile } from 'fs/promises'
33
import path from 'path'
44
import { fileURLToPath } from 'url'
@@ -7,23 +7,6 @@ const __filename = fileURLToPath(import.meta.url)
77

88
const __dirname = path.dirname(__filename)
99

10-
function writeUrl({ location, lastMod, priority = '0.5' }) {
11-
return ` <url>
12-
<loc>${process.env.NEXT_PUBLIC_ROOT_URL}/${location}</loc>
13-
<lastmod>${lastMod}</lastmod>
14-
<changefreq>weekly</changefreq>
15-
<priority>${priority || '0.5'}</priority>
16-
</url>`
17-
}
18-
19-
function wrapSiteMap(content) {
20-
return `<?xml version="1.0" encoding="UTF-8"?>
21-
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
22-
${content}
23-
</urlset>
24-
`
25-
}
26-
2710
const ignore = [
2811
// sitemaps
2912
/^sitemap\.xml.*/,
@@ -32,7 +15,7 @@ const ignore = [
3215
// [nextjs_dynamic_pages]
3316
// 404 and 500 error pages
3417
/^([_.[]|404|500).*$/,
35-
/^__components\/.*/
18+
/^__components\/.*/,
3619
]
3720

3821
const pageFilter = (file) => {
@@ -41,10 +24,41 @@ const pageFilter = (file) => {
4124
return false
4225
}
4326
}
27+
4428
if (file?.isDirectory()) {
4529
return true
4630
}
47-
return file.name.match(/\.(ts|tsx|js|jsx|md|mdoc)$/)
31+
32+
// Only include markdown files
33+
return file.name.match(/\.(md|mdoc)$/)
34+
}
35+
36+
// Function to strip numeric prefixes from path segments
37+
function stripNumericPrefixes(pathStr) {
38+
return pathStr
39+
.split('/')
40+
.map((segment) => segment.replace(/^\d+-/, ''))
41+
.join('/')
42+
}
43+
44+
// Function to get the last git modification date of a file
45+
function getGitLastModified(filePath) {
46+
try {
47+
// Get the last commit date that modified this file in UTC
48+
const date = execSync(`git log -1 --format="%cI" -- "${filePath}"`, {
49+
encoding: 'utf-8',
50+
}).trim()
51+
52+
if (!date) {
53+
return new Date().toISOString()
54+
}
55+
56+
// Convert to UTC and format as ISO string
57+
return new Date(date).toISOString()
58+
} catch (error) {
59+
// Fallback to current date if git command fails
60+
return new Date().toISOString()
61+
}
4862
}
4963

5064
const rootDir = __dirname
@@ -58,13 +72,25 @@ async function crawlPages(filePath = '/') {
5872
const filteredFiles = files.filter(pageFilter)
5973

6074
const pages = []
75+
6176
for (const file of filteredFiles) {
6277
if (file.isDirectory()) {
6378
pages.push(...(await crawlPages(path.join(filePath, file.name))))
6479
} else {
6580
let pathname = file.name.split('.').slice(0, -1).join('.')
81+
6682
pathname = path.join(filePath, pathname.replace(/(^|\/)index$/g, ''))
67-
pages.push({ path: pathname })
83+
// Strip numeric prefixes from the entire path
84+
pathname = stripNumericPrefixes(pathname)
85+
86+
// Get git modification time
87+
const fullFilePath = path.join(fullPath, file.name)
88+
const lastmod = getGitLastModified(fullFilePath)
89+
90+
pages.push({
91+
path: pathname,
92+
lastmod,
93+
})
6894
}
6995
}
7096

‎next.config.js‎

Lines changed: 7 additions & 187 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ const nextConfig = {
2727
},
2828
async redirects() {
2929
return [
30+
{
31+
source: '/getting-started/agent-api-reference',
32+
destination: '/overview/agent-api-reference',
33+
permanent: true,
34+
},
3035
{
3136
source: '/getting-started/readme',
3237
destination: '/',
@@ -43,200 +48,15 @@ const nextConfig = {
4348
permanent: true,
4449
},
4550
{
46-
source: '/reference/architecture-1',
47-
destination: '/reference/architecture',
48-
permanent: true,
49-
},
50-
{
51-
source: '/reference/workspaces/workspace-structure',
52-
destination: '/reference/workspaces',
53-
permanent: true,
54-
},
55-
{
56-
source: '/getting-started/getting-started-with-runbooks/runbook-yaml',
57-
destination:
58-
'/adding-new-application/getting-started-with-runbooks/runbook-yaml',
59-
permanent: true,
60-
},
61-
{
62-
source: '/basic-setup-and-deployment/setting-up-gitops',
63-
destination:
64-
'/getting-started/managing-git-repository/setting-up-gitops',
65-
permanent: true,
66-
},
67-
{
68-
source: '/basic-setup-and-deployment/openid-connect',
69-
destination: '/getting-started/openid-connect',
70-
permanent: true,
71-
},
72-
{
73-
source: '/basic-setup-and-deployment/admin-console',
74-
destination: '/getting-started/admin-console',
75-
permanent: true,
76-
},
77-
{
78-
source: '/basic-setup-and-deployment/cloud-shell-quickstart',
79-
destination: '/getting-started/cloud-shell-quickstart',
80-
permanent: true,
81-
},
82-
{
83-
source: '/basic-setup-and-deployment/uninstall',
84-
destination: '/operations/uninstall',
85-
permanent: true,
86-
},
87-
{
88-
source: '/reference/operator-guides',
89-
destination: '/repositories',
90-
permanent: true,
91-
},
92-
{
93-
source: '/reference/operator-guides/adding-kubecost-for-cost-analysis',
94-
destination: '/repositories/kubecost',
51+
source: '/deployments/architecture',
52+
destination: '/overview/architecture',
9553
permanent: true,
9654
},
9755
{
9856
source: '/reference/architecture',
9957
destination: '/',
10058
permanent: true,
10159
},
102-
{
103-
source: '/repositories',
104-
destination: '/applications',
105-
permanent: true,
106-
},
107-
{
108-
source: '/repositories/airbyte',
109-
destination: '/applications/airbyte',
110-
permanent: true,
111-
},
112-
{
113-
source: '/repositories/airflow',
114-
destination: '/applications/airflow',
115-
permanent: true,
116-
},
117-
{
118-
source: '/repositories/console',
119-
destination: '/applications/console',
120-
permanent: true,
121-
},
122-
{
123-
source: '/advanced-topics',
124-
destination: '/operations',
125-
permanent: true,
126-
},
127-
{
128-
source: '/advanced-topics/network-configuration',
129-
destination: '/operations/network-configuration',
130-
permanent: true,
131-
},
132-
{
133-
source: '/advanced-topics/dns-setup',
134-
destination: '/operations/dns-setup',
135-
permanent: true,
136-
},
137-
{
138-
source:
139-
'/advanced-topics/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
140-
destination:
141-
'/operations/dns-setup/creating-dns-zone-in-your-cloud-provider-console',
142-
permanent: true,
143-
},
144-
{
145-
source: '/advanced-topics/security',
146-
destination: '/operations/security',
147-
permanent: true,
148-
},
149-
{
150-
source: '/advanced-topics/security/secret-management',
151-
destination:
152-
'/getting-started/manage-git-repositories/sharing-git-repositories',
153-
permanent: true,
154-
},
155-
{
156-
source: '/advanced-topics/debugging',
157-
destination: '/debugging',
158-
permanent: true,
159-
},
160-
{
161-
source: '/advanced-topics/debugging/health-checks',
162-
destination: '/debugging/health-checks',
163-
permanent: true,
164-
},
165-
{
166-
source: '/advanced-topics/debugging/logs',
167-
destination: '/debugging/logs',
168-
permanent: true,
169-
},
170-
{
171-
source: '/advanced-topics/debugging/proxies',
172-
destination: '/debugging/proxies',
173-
permanent: true,
174-
},
175-
{
176-
source: '/advanced-topics/identity-and-access-management',
177-
destination: '/operations/auth-access-control',
178-
permanent: true,
179-
},
180-
{
181-
source: '/advanced-topics/identity-and-access-management/introduction',
182-
destination: '/operations/auth-access-control',
183-
permanent: true,
184-
},
185-
{
186-
source:
187-
'/advanced-topics/identity-and-access-management/openid-connect',
188-
destination: '/operations/auth-access-control/openid-connect',
189-
permanent: true,
190-
},
191-
{
192-
source: '/advanced-topics/identity-and-access-management/api-tokens',
193-
destination: '/operations/auth-access-control/api-tokens',
194-
permanent: true,
195-
},
196-
{
197-
source:
198-
'/advanced-topics/identity-and-access-management/identity-and-installations',
199-
destination:
200-
'/operations/auth-access-control/identity-and-installations',
201-
permanent: true,
202-
},
203-
{
204-
source:
205-
'/advanced-topics/identity-and-access-management/identity-and-installations/audit-logging',
206-
destination:
207-
'/operations/auth-access-control/identity-and-installations/audit-logging',
208-
permanent: true,
209-
},
210-
{
211-
source:
212-
'/advanced-topics/identity-and-access-management/identity-and-installations/service-accounts',
213-
destination:
214-
'/operations/auth-access-control/identity-and-installations/service-accounts',
215-
permanent: true,
216-
},
217-
{
218-
source:
219-
'/advanced-topics/identity-and-access-management/identity-and-installations/sharing-existing-repos',
220-
destination:
221-
'/getting-started/manage-git-repositories/sharing-git-repository',
222-
permanent: true,
223-
},
224-
{
225-
source: '/reference/workspaces',
226-
destination:
227-
'/getting-started/manage-git-repositories/your-plural-workspace',
228-
permanent: true,
229-
},
230-
{
231-
source: '/introduction',
232-
destination: '/overview/introduction',
233-
permanent: true,
234-
},
235-
{
236-
source: '/deployments/architecture',
237-
destination: '/overview/architecture',
238-
permanent: true,
239-
},
24060
{
24161
source: '/deployments/operator/api',
24262
destination: '/overview/api-reference',

‎package.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44
"node": "22.12.0"
55
},
66
"scripts": {
7-
"dev": "run-s generate:routes && concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
7+
"dev": "run-s generate:routes generate:pages track:redirects && concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
88
"dev:watch": "concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
9-
"build": "run-s generate:routes build:next",
9+
"build": "run-s generate:routes generate:pages build:next",
1010
"build:next": "next build",
1111
"start": "next start",
1212
"lint": "run-p lint:format lint:js lint:css",
@@ -19,7 +19,9 @@
1919
"fix:css": "stylelint src/**/*.css --fix",
2020
"graphql:codegen": "graphql-codegen --config codegen.yml",
2121
"graphql:codegen:watch": "graphql-codegen --watch --config codegen.yml",
22-
"generate:routes": "ts-node scripts/generate-unified-routes.ts"
22+
"generate:routes": "ts-node scripts/generate-unified-routes.ts",
23+
"generate:pages": "node index-pages.mjs",
24+
"track:redirects": "ts-node scripts/track-moves.ts"
2325
},
2426
"dependencies": {
2527
"@apollo/client": "3.7.15",

0 commit comments

Comments
 (0)