99 lines
2.7 KiB
TypeScript
99 lines
2.7 KiB
TypeScript
import ts from 'typescript'
|
|
import { writeFile } from 'node:fs/promises'
|
|
import robotsJson from '../src/data/robots.json' with { type: 'json' }
|
|
import { escape } from '../src/escape.ts'
|
|
|
|
interface Bot {
|
|
operator?: string
|
|
respect?: string
|
|
function?: string
|
|
frequency?: string
|
|
description?: string
|
|
}
|
|
|
|
const bots = new Map<string, Bot>(Object.entries(robotsJson))
|
|
|
|
// Robots.txt string
|
|
const userAgentDirectives = []
|
|
|
|
// Regexes constructors
|
|
const allBots = []
|
|
const nonRespectingBots = []
|
|
const allBotsExpressions = []
|
|
const nonRespectingBotsExpressions = []
|
|
|
|
for (const [key, value] of bots) {
|
|
userAgentDirectives.push(`User-agent: ${key}`)
|
|
const expression = ts.factory.createStringLiteral(key)
|
|
const item = escape(key.toUpperCase())
|
|
allBots.push(item)
|
|
allBotsExpressions.push(expression)
|
|
|
|
if (!value?.respect?.includes('Yes')) {
|
|
nonRespectingBots.push(item)
|
|
nonRespectingBotsExpressions.push(expression)
|
|
}
|
|
}
|
|
|
|
const statements = [
|
|
// export const ROBOTS_TXT = '...'
|
|
createExportConstStatement(
|
|
'ROBOTS_TXT',
|
|
ts.factory.createNoSubstitutionTemplateLiteral(
|
|
`${userAgentDirectives.join('\n')}\nDisallow: /\n`
|
|
)
|
|
),
|
|
// export const ALL_BOTS = [...]
|
|
createExportConstStatement(
|
|
'ALL_BOTS',
|
|
ts.factory.createArrayLiteralExpression(allBotsExpressions)
|
|
),
|
|
// export const NON_RESPECTING_BOTS = [...]
|
|
createExportConstStatement(
|
|
'NON_RESPECTING_BOTS',
|
|
ts.factory.createArrayLiteralExpression(nonRespectingBotsExpressions)
|
|
),
|
|
// export const ALL_BOTS_REGEX = /(...)/;
|
|
createExportConstStatement(
|
|
'ALL_BOTS_REGEX',
|
|
ts.factory.createRegularExpressionLiteral(`/(${allBots.join('|')})/`)
|
|
),
|
|
// export const NON_RESPECTING_BOTS_REGEX = /(...)/;
|
|
createExportConstStatement(
|
|
'NON_RESPECTING_BOTS_REGEX',
|
|
ts.factory.createRegularExpressionLiteral(`/(${nonRespectingBots.join('|')})/`)
|
|
),
|
|
]
|
|
|
|
ts.addSyntheticLeadingComment(
|
|
statements[0],
|
|
ts.SyntaxKind.SingleLineCommentTrivia,
|
|
' This file is generated by scripts/get-robots-txt.ts. Do not edit manually.'
|
|
)
|
|
|
|
const generatedFile = ts.factory.createSourceFile(
|
|
statements,
|
|
ts.factory.createToken(ts.SyntaxKind.EndOfFileToken),
|
|
ts.NodeFlags.None
|
|
)
|
|
|
|
const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed })
|
|
|
|
await writeFile(
|
|
new URL('../src/generated.ts', import.meta.url),
|
|
printer.printFile(generatedFile),
|
|
'utf-8'
|
|
)
|
|
|
|
function createExportConstStatement(name: string, initializer: ts.Expression) {
|
|
return ts.factory.createVariableStatement(
|
|
[ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)],
|
|
ts.factory.createVariableDeclarationList(
|
|
[ts.factory.createVariableDeclaration(name, undefined, undefined, initializer)],
|
|
ts.NodeFlags.Const
|
|
)
|
|
)
|
|
}
|
|
|
|
console.log('☑︎ Generated bots file successfully.')
|