import ts from 'typescript' import { writeFile } from 'node:fs/promises' import robotsJson from '../src/data/robots.json' with { type: 'json' } import { escape } from '../src/escape.ts' interface Bot { operator?: string respect?: string function?: string frequency?: string description?: string } const bots = new Map(Object.entries(robotsJson)) // Robots.txt string const userAgentDirectives = [] // Regexes constructors const allBots = [] const nonRespectingBots = [] const allBotsExpressions = [] const nonRespectingBotsExpressions = [] for (const [key, value] of bots) { userAgentDirectives.push(`User-agent: ${key}`) const expression = ts.factory.createStringLiteral(key) const item = escape(key.toUpperCase()) allBots.push(item) allBotsExpressions.push(expression) if (!value?.respect?.includes('Yes')) { nonRespectingBots.push(item) nonRespectingBotsExpressions.push(expression) } } const statements = [ // export const ROBOTS_TXT = '...' createExportConstStatement( 'ROBOTS_TXT', ts.factory.createNoSubstitutionTemplateLiteral( `${userAgentDirectives.join('\n')}\nDisallow: /\n` ) ), // export const ALL_BOTS = [...] createExportConstStatement( 'ALL_BOTS', ts.factory.createArrayLiteralExpression(allBotsExpressions) ), // export const NON_RESPECTING_BOTS = [...] createExportConstStatement( 'NON_RESPECTING_BOTS', ts.factory.createArrayLiteralExpression(nonRespectingBotsExpressions) ), // export const ALL_BOTS_REGEX = /(...)/; createExportConstStatement( 'ALL_BOTS_REGEX', ts.factory.createRegularExpressionLiteral(`/(${allBots.join('|')})/`) ), // export const NON_RESPECTING_BOTS_REGEX = /(...)/; createExportConstStatement( 'NON_RESPECTING_BOTS_REGEX', ts.factory.createRegularExpressionLiteral(`/(${nonRespectingBots.join('|')})/`) ), ] ts.addSyntheticLeadingComment( statements[0], ts.SyntaxKind.SingleLineCommentTrivia, ' This file is generated by scripts/get-robots-txt.ts. Do not edit manually.' ) const generatedFile = ts.factory.createSourceFile( statements, ts.factory.createToken(ts.SyntaxKind.EndOfFileToken), ts.NodeFlags.None ) const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed }) await writeFile( new URL('../src/generated.ts', import.meta.url), printer.printFile(generatedFile), 'utf-8' ) function createExportConstStatement(name: string, initializer: ts.Expression) { return ts.factory.createVariableStatement( [ts.factory.createModifier(ts.SyntaxKind.ExportKeyword)], ts.factory.createVariableDeclarationList( [ts.factory.createVariableDeclaration(name, undefined, undefined, initializer)], ts.NodeFlags.Const ) ) } console.log('☑︎ Generated bots file successfully.')