From a9ef9a35159a6fa26143663e3f70e49e3b7dc0cc Mon Sep 17 00:00:00 2001 From: Hattan Shobokshi Date: Sun, 28 Aug 2022 23:18:52 -0700 Subject: [PATCH 1/3] Adding support for ReWrite Rules in repo yaml. Co-authored-by: Hadwa Gaber --- generators/repo/src/commands/generate.ts | 85 ++++++++++--------- generators/repo/src/models/index.ts | 4 +- .../todo/projects/csharp-cosmos-sql/repo.yaml | 23 +++-- .../todo/projects/csharp-mongo/repo.yaml | 23 +++-- templates/todo/projects/csharp-sql/repo.yaml | 23 +++-- .../todo/projects/nodejs-mongo-aca/repo.yaml | 23 +++-- .../projects/nodejs-mongo-swa-func/repo.yaml | 23 +++-- .../nodejs-mongo/.repo/bicep/repo.yaml | 35 ++++---- .../todo/projects/python-mongo-aca/repo.yaml | 24 ++++-- .../projects/python-mongo-swa-func/repo.yaml | 23 +++-- .../python-mongo/.repo/bicep/repo.yaml | 39 ++++----- 11 files changed, 184 insertions(+), 141 deletions(-) diff --git a/generators/repo/src/commands/generate.ts b/generators/repo/src/commands/generate.ts index b96335b101b..364d849063b 100644 --- a/generators/repo/src/commands/generate.ts +++ b/generators/repo/src/commands/generate.ts @@ -6,7 +6,7 @@ import fs from "fs/promises"; import ansiEscapes from "ansi-escapes"; import chalk from "chalk"; import { cleanDirectoryPath, copyFile, createRepoUrlFromRemote, ensureDirectoryPath, getGlobFiles, getRepoPropsFromRemote, isStringNullOrEmpty, RepoProps, writeHeader,isFilePath } from "../common/util"; -import { AssetRule, GitRemote, RepomanCommand, RepomanCommandOptions, RepoManifest } from "../models"; +import { AssetRule, RewriteRule, GitRemote, RepomanCommand, RepomanCommandOptions, RepoManifest } from "../models"; import { GitRepo } from "../tools/git"; export interface GenerateCommandOptions extends RepomanCommandOptions { @@ -39,7 +39,7 @@ export class GenerateCommand implements RepomanCommand { private outputPath: string; private generatePath: string; private assetRules: AssetRule[]; - private rewritePatterns: string[]; + private rewriteRules: RewriteRule[]; constructor(private options: GenerateCommandOptions) { this.sourcePath = path.resolve(path.normalize(options.source)); @@ -61,7 +61,7 @@ export class GenerateCommand implements RepomanCommand { }); } - this.rewritePatterns = this.manifest.repo.rewrite?.patterns || ["**/*.@(yml|yaml)"]; + this.rewriteRules =(this.manifest.repo.rewrite) ? [...this.manifest.repo.rewrite?.rules] : []; } catch (err) { console.error(chalk.red(`Repo template manifest not found at '${this.templateFile}'`)); @@ -86,7 +86,10 @@ export class GenerateCommand implements RepomanCommand { } console.info(); - await this.rewritePaths(); + + for (const rule of this.rewriteRules) { + await this.processRewriteRule(rule); + } console.info(chalk.cyan('Repo generation completed.')); console.info(); @@ -375,53 +378,51 @@ export class GenerateCommand implements RepomanCommand { } } - private rewritePaths = async () => { - console.info(chalk.cyan("Rewriting relative paths found in files")); - - const globOptions: IOptions = { cwd: this.generatePath, matchBase: true }; - const tasks = this.rewritePatterns.map(pattern => getGlobFiles(pattern, globOptions)); - const files = (await Promise.all(tasks)).flat(); - - const pathRegex = new RegExp(/((?:\.{1,2}[\/\\]{1,2})+[^'"\s]*)/gm); - - for (const filePath of files) { - const destFilePath = path.join(this.generatePath, filePath); - const destFolderPath = path.dirname(destFilePath) - - if (this.options.debug) { - console.debug(chalk.whiteBright(`Processing file: ${destFilePath}`)); - } - - const buffer = await fs.readFile(destFilePath); - let contents = buffer.toString('utf8'); - - // Replace relative path updates - for (const rule of this.assetRules) { - if (this.options.debug) { - console.debug(chalk.white(`- Processing ${rule.from} => ${rule.to}`)); - } - contents = contents.replaceAll(rule.from, rule.to); + private processRewriteRule = async(rule: RewriteRule) => { + const globOptions: IOptions = { + cwd: this.generatePath, + ignore: rule.ignore, + matchBase: true + }; + const patterns = rule.patterns ?? []; + for (const pattern of patterns) { + const files = await getGlobFiles(pattern, globOptions); + for (const filePath of files) { + await this.rewritePath(rule, filePath); } + } + } + private rewritePath = async(rule: RewriteRule, filePath: string) => { + const destFilePath = path.join(this.generatePath, filePath); + const destFolderPath = path.dirname(destFilePath); + const buffer = await fs.readFile(destFilePath); + let contents = buffer.toString('utf8'); + if(contents.indexOf(rule.from) > -1) { + console.info(chalk.cyan(` -> Rewriting relative paths ${rule.from} => ${rule.to} for file "${filePath}"`)); + contents = contents.replaceAll(rule.from, rule.to); + // Normalize transformed paths + const pathRegex = new RegExp(/((?:\.{1,2}[\/\\]{1,2})+[^'"\s]*)/gm); const matches = contents.match(pathRegex); if (matches && matches.length > 0) { for (const match of matches) { - // Generate the absolute path to the referenced match - let refPath = path.resolve(destFolderPath, path.normalize(match)) - // Generate the relative path between the current processed file dir path & the referenced match path - let relativePath = path.relative(destFolderPath, refPath) - // Finally convert the path back to a POSIX compatible path - relativePath = relativePath.split(path.sep).join(path.posix.sep) - - contents = contents.replaceAll(match, relativePath); - - if (this.options.debug) { - console.log(chalk.grey(` -> Rewriting relative path ${match} => ${relativePath}`)); + if(match.indexOf(rule.to) > -1){ + // Generate the absolute path to the referenced match + let refPath = path.resolve(destFolderPath, path.normalize(match)) + // Generate the relative path between the current processed file dir path & the referenced match path + let relativePath = path.relative(destFolderPath, refPath) + // Finally convert the path back to a POSIX compatible path + relativePath = relativePath.split(path.sep).join(path.posix.sep) + + contents = contents.replaceAll(match, relativePath); + + if (this.options.debug) { + console.log(chalk.grey(` -> Rewriting relative path ${match} => ${relativePath} in ${destFilePath}`)); + } } } } - await fs.writeFile(destFilePath, contents); } } diff --git a/generators/repo/src/models/index.ts b/generators/repo/src/models/index.ts index 07460911ba7..17d5a624ca5 100644 --- a/generators/repo/src/models/index.ts +++ b/generators/repo/src/models/index.ts @@ -10,7 +10,7 @@ export interface RepoManifest { assets: AssetRule[] rewrite?: { - patterns: string[] + rules: RewriteRule[] } } } @@ -28,6 +28,8 @@ export interface AssetRule { ignore?: string[] } +export type RewriteRule = AssetRule; + export interface RepomanCommandOptions { debug: true [key: string]: any diff --git a/templates/todo/projects/csharp-cosmos-sql/repo.yaml b/templates/todo/projects/csharp-cosmos-sql/repo.yaml index 1b661737906..de0a3c2738e 100644 --- a/templates/todo/projects/csharp-cosmos-sql/repo.yaml +++ b/templates/todo/projects/csharp-cosmos-sql/repo.yaml @@ -15,17 +15,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/csharp-cosmos-sql + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/csharp-mongo/repo.yaml b/templates/todo/projects/csharp-mongo/repo.yaml index 66c0708cdb1..12115cda8b3 100644 --- a/templates/todo/projects/csharp-mongo/repo.yaml +++ b/templates/todo/projects/csharp-mongo/repo.yaml @@ -15,17 +15,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/csharp + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/csharp-sql/repo.yaml b/templates/todo/projects/csharp-sql/repo.yaml index d5afe8ca78e..c4f6d081666 100644 --- a/templates/todo/projects/csharp-sql/repo.yaml +++ b/templates/todo/projects/csharp-sql/repo.yaml @@ -16,17 +16,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/csharp-sql + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/nodejs-mongo-aca/repo.yaml b/templates/todo/projects/nodejs-mongo-aca/repo.yaml index b72ec4eb452..b1a05b7e892 100644 --- a/templates/todo/projects/nodejs-mongo-aca/repo.yaml +++ b/templates/todo/projects/nodejs-mongo-aca/repo.yaml @@ -15,17 +15,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/js + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/nodejs-mongo-swa-func/repo.yaml b/templates/todo/projects/nodejs-mongo-swa-func/repo.yaml index 8e7dc245a95..c9e34de6f76 100644 --- a/templates/todo/projects/nodejs-mongo-swa-func/repo.yaml +++ b/templates/todo/projects/nodejs-mongo-swa-func/repo.yaml @@ -15,17 +15,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/js + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/nodejs-mongo/.repo/bicep/repo.yaml b/templates/todo/projects/nodejs-mongo/.repo/bicep/repo.yaml index 69cee14455e..b03d6137036 100644 --- a/templates/todo/projects/nodejs-mongo/.repo/bicep/repo.yaml +++ b/templates/todo/projects/nodejs-mongo/.repo/bicep/repo.yaml @@ -15,29 +15,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/js + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rules for Path Rewrite Only - - from: ../../../../../../common/infra/bicep - to: ./ - ignore: - - ".repo/**/*" - - - from: ../../api/js - to: ./src/api - ignore: - - ".repo/**/*" - - - from: ../../web/react-fluent - to: ./src/web - ignore: - - ".repo/**/*" - # End Path Rewrite Rules - - from: ./../../ to: ./ ignore: diff --git a/templates/todo/projects/python-mongo-aca/repo.yaml b/templates/todo/projects/python-mongo-aca/repo.yaml index d2fb83ac09d..c4754c61bed 100644 --- a/templates/todo/projects/python-mongo-aca/repo.yaml +++ b/templates/todo/projects/python-mongo-aca/repo.yaml @@ -15,17 +15,25 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/python + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/python-mongo-swa-func/repo.yaml b/templates/todo/projects/python-mongo-swa-func/repo.yaml index e3427cc2d38..ae4af8e60d4 100644 --- a/templates/todo/projects/python-mongo-swa-func/repo.yaml +++ b/templates/todo/projects/python-mongo-swa-func/repo.yaml @@ -15,17 +15,24 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" + rules: + - from: ../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/python + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" assets: # Common assets - # Rule for Path Rewrite Only - - from: ../../../../common/infra/bicep - to: ./ - ignore: - - "**/*" # openapi.yaml to root - from: ../../api/common diff --git a/templates/todo/projects/python-mongo/.repo/bicep/repo.yaml b/templates/todo/projects/python-mongo/.repo/bicep/repo.yaml index 43703ae8a19..033be72fb72 100644 --- a/templates/todo/projects/python-mongo/.repo/bicep/repo.yaml +++ b/templates/todo/projects/python-mongo/.repo/bicep/repo.yaml @@ -15,30 +15,25 @@ repo: branch: staging rewrite: - patterns: - - "**/azure.@(yml|yaml)" - - "**/*.bicep" - + rules: + - from: ../../../../../../common/infra/bicep + to: ./ + patterns: + - "**/*.bicep" + + - from: ../../api/python + to: ./src/api + patterns: + - "**/azure.@(yml|yaml)" + + - from: ../../web/react-fluent + to: ./src/web + patterns: + - "**/azure.@(yml|yaml)" + assets: # Common assets - - # Rules for Path Rewrite Only - - from: ../../../../../../common/infra/bicep - to: ./ - ignore: - - ".repo/**/*" - - - from: ../../api/python - to: ./src/api - ignore: - - ".repo/**/*" - - - from: ../../web/react-fluent - to: ./src/web - ignore: - - ".repo/**/*" - # End Path Rewrite Rules - + - from: ./../../ to: ./ ignore: From ba9539e88f3fdc38d24ef329552b8130858be5ef Mon Sep 17 00:00:00 2001 From: Hattan Shobokshi Date: Tue, 30 Aug 2022 14:32:53 -0700 Subject: [PATCH 2/3] Repo.yaml cleanup. Removing extra line. Co-authored-by: Hadwa Gaber --- templates/todo/projects/python-mongo-aca/repo.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/todo/projects/python-mongo-aca/repo.yaml b/templates/todo/projects/python-mongo-aca/repo.yaml index c4754c61bed..8ad37b3bfd8 100644 --- a/templates/todo/projects/python-mongo-aca/repo.yaml +++ b/templates/todo/projects/python-mongo-aca/repo.yaml @@ -26,7 +26,6 @@ repo: patterns: - "**/azure.@(yml|yaml)" - - from: ../../web/react-fluent to: ./src/web patterns: From ab786f4b88821088cb120f7ccbaaa1b23b7fb03c Mon Sep 17 00:00:00 2001 From: Hattan Shobokshi Date: Tue, 30 Aug 2022 14:33:32 -0700 Subject: [PATCH 3/3] Modifying default Rewrite rule behavior so not including a pattern results in all files being inspected. Co-authored-by: Hadwa Gaber --- generators/repo/src/commands/generate.ts | 52 ++++++++++++------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/generators/repo/src/commands/generate.ts b/generators/repo/src/commands/generate.ts index 364d849063b..a3163a6ddfc 100644 --- a/generators/repo/src/commands/generate.ts +++ b/generators/repo/src/commands/generate.ts @@ -382,9 +382,10 @@ export class GenerateCommand implements RepomanCommand { const globOptions: IOptions = { cwd: this.generatePath, ignore: rule.ignore, - matchBase: true + matchBase: true, + nodir: true }; - const patterns = rule.patterns ?? []; + const patterns = rule.patterns ?? ["**/**"]; for (const pattern of patterns) { const files = await getGlobFiles(pattern, globOptions); for (const filePath of files) { @@ -398,32 +399,33 @@ export class GenerateCommand implements RepomanCommand { const destFolderPath = path.dirname(destFilePath); const buffer = await fs.readFile(destFilePath); let contents = buffer.toString('utf8'); - if(contents.indexOf(rule.from) > -1) { - console.info(chalk.cyan(` -> Rewriting relative paths ${rule.from} => ${rule.to} for file "${filePath}"`)); - contents = contents.replaceAll(rule.from, rule.to); - - // Normalize transformed paths - const pathRegex = new RegExp(/((?:\.{1,2}[\/\\]{1,2})+[^'"\s]*)/gm); - const matches = contents.match(pathRegex); - if (matches && matches.length > 0) { - for (const match of matches) { - if(match.indexOf(rule.to) > -1){ - // Generate the absolute path to the referenced match - let refPath = path.resolve(destFolderPath, path.normalize(match)) - // Generate the relative path between the current processed file dir path & the referenced match path - let relativePath = path.relative(destFolderPath, refPath) - // Finally convert the path back to a POSIX compatible path - relativePath = relativePath.split(path.sep).join(path.posix.sep) - - contents = contents.replaceAll(match, relativePath); - - if (this.options.debug) { - console.log(chalk.grey(` -> Rewriting relative path ${match} => ${relativePath} in ${destFilePath}`)); - } + if(contents.indexOf(rule.from) == -1) return; + + console.info(chalk.cyan(` -> Rewriting relative paths ${rule.from} => ${rule.to} for file "${filePath}"`)); + contents = contents.replaceAll(rule.from, rule.to); + + // Normalize transformed paths + const pathRegex = new RegExp(/((?:\.{1,2}[\/\\]{1,2})+[^'"\s]*)/gm); + const matches = contents.match(pathRegex); + if (matches && matches.length > 0) { + for (const match of matches) { + if(match.indexOf(rule.to) > -1){ + // Generate the absolute path to the referenced match + let refPath = path.resolve(destFolderPath, path.normalize(match)) + // Generate the relative path between the current processed file dir path & the referenced match path + let relativePath = path.relative(destFolderPath, refPath) + // Finally convert the path back to a POSIX compatible path + relativePath = relativePath.split(path.sep).join(path.posix.sep) + + contents = contents.replaceAll(match, relativePath); + + if (this.options.debug) { + console.log(chalk.grey(` -> Rewriting relative path ${match} => ${relativePath} in ${destFilePath}`)); } } } - await fs.writeFile(destFilePath, contents); } + await fs.writeFile(destFilePath, contents); + } } \ No newline at end of file