Vavera (#42)

* Fixing issues

* Remove comment

* Fix node version

* Fix deepscan issue

* Fix wrong data type

* Prettier formatting

* Fix excluding pattern

---------

Co-authored-by: Jakub <jakub.vavera@profiq.com>

.gitignore

@@ -211,6 +211,7 @@ cmake-build-*/
 
 # IntelliJ
 out/
+.idea
 
 # mpeltonen/sbt-idea plugin
 .idea_modules/

README.md

@@ -1,6 +1,7 @@
 # Site Spectre CLI
-![npm](https://img.shields.io/npm/v/site-spectre)
-![npm type definitions](https://img.shields.io/npm/types/site-spectre)
+
+[![npm](https://img.shields.io/npm/v/site-spectre)](https://www.npmjs.com/package/site-spectre)
+[![npm type definitions](https://img.shields.io/npm/types/site-spectre)](https://www.typescriptlang.org/)
 [![DeepScan grade](https://deepscan.io/api/teams/22045/projects/25384/branches/794017/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=22045&pid=25384&bid=794017)
 [![codecov](https://codecov.io/gh/profiq/site-spectre-cli/graph/badge.svg?token=AIW4AXPQ4R)](https://codecov.io/gh/profiq/site-spectre-cli)
 ![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/profiq/site-spectre-cli/publish-rc.yaml)

package.json

@@ -17,6 +17,20 @@
     "reinstall": "rm -fr node_modules && npm install",
     "prepare": "husky install"
   },
+  "engines": {
+    "node": ">18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/profiq/site-spectre-cli.git"
+  },
+  "keywords": [
+    "cli",
+    "tool",
+    "console",
+    "terminal",
+    "command"
+  ],
   "author": "profiq",
   "license": "MIT",
   "bin": {

src/cli-inputs.ts

@@ -36,6 +36,7 @@ const createConfig = (options: any) => {
     // custom headesr --> array
     parallelBlockSize: Number(options.parallel),
     customHeaders: {},
+    excludePattern: options.exclude,
     dryRun: options.dry,
     debugMode: options.debug,
     silentRun: options.silent,
@@ -54,7 +55,7 @@ const pasteConfigFile = (config: configType, customConfigFile: any) => {
   // custom headesr --> array
   config.parallelBlockSize = Number(customConfigFile.parallelBlockSize);
   config.customHeaders = {};
-  config.dryRun = customConfigFile.dryRun;
+  (config.excludePattern = customConfigFile.exclude), (config.dryRun = customConfigFile.dryRun);
   config.debugMode = customConfigFile.debugMode;
   config.silentRun = customConfigFile.silentRun;
   config.configFilePath = customConfigFile.configFilePath;
@@ -80,7 +81,8 @@ const updateConfig = (config: configType, options: OptionValues, customConfigFil
     config.pageLoadType = "network";
   }
 
-  config.dryRun = options.dry ? true : customConfigFile.dryRun;
+  (config.excludePattern = options.exclude ? true : customConfigFile.exclude),
+    (config.dryRun = options.dry ? true : customConfigFile.dryRun);
   config.debugMode = options.debug ? true : customConfigFile.debugMode;
   config.silentRun = options.silent ? true : customConfigFile.silentRun;
   config.configFilePath = options.configFile
@@ -105,4 +107,11 @@ const checkConfigFile = (config: any, options: OptionValues) => {
   }
 };
 
-export { sitesInput, createConfig, checkConfigFile, defaultParallel, defaultTimeout };
+export {
+  sitesInput,
+  createConfig,
+  pasteConfigFile,
+  checkConfigFile,
+  defaultParallel,
+  defaultTimeout,
+};

src/index.ts

@@ -1,20 +1,14 @@
-import { processSitemap, _readSitemap, _parseSitemap } from "./sitemap-parsers";
+import { processSitemap } from "./sitemap-parsers";
 import { visitConfigPrint, visitSitesWinston } from "./link-visit";
-import yargs, { option } from "yargs";
-import { Interface } from "readline";
 import { Command, Option } from "commander";
 import packageJSON from "../package.json";
-import { logger } from "./logger";
 import {
   checkConfigFile,
   createConfig,
   defaultParallel,
   defaultTimeout,
   sitesInput,
 } from "./cli-inputs";
-import chalk from "chalk";
-import { configType } from "./types";
-import { existsSync, readFileSync } from "fs";
 
 const program = new Command();
 
@@ -41,10 +35,11 @@ program
       .default("document"),
   )
   .option("-w, --no-wait-page-load", "Disable waiting for page to be loaded.")
-  .option("-h, --custom-headers", "Pass a custom header.") //
-  // .option("-d, --debug", "Sets the prinout level to debug.") //
+  .option("-h, --custom-headers", "Pass a custom header.")
+  .option("-d, --debug", "Sets the prinout level to debug.")
   .option("-D, --dry", "Just prints the links it would visit without visiting.")
   .option("-s, --silent", "Log only errors.")
+  .option("-e, --exclude <pattern>", "Regex expression for links that should be excluded.")
   .option(
     "-c, --config-file <filePath>",
     "JSON config file, if you specify any other parameters, they take priority over the config file.",
@@ -56,22 +51,28 @@ const options = program.opts();
 
 const config = createConfig(options);
 
+const regexPattern = new RegExp(options.exclude);
+config.excludePattern = regexPattern;
+
 checkConfigFile(config, options);
 
 let sites: string[] = [
   // "https://www.profiq.com/wp-sitemap.xml",
   // "https://www.profiq.com/wp-sitemap-posts-post-1.xml",
   // "https://www.profiq.com/wp-sitemap-posts-page-1.xml",
   // "https://www.profiq.com/wp-sitemap-posts-job-1.xml",
-  // "https://movingfast.tech/post-sitemap.xml"
+  // "https://movingfast.tech/post-sitemap.xml",
   // "https://www.advancedhtml.co.uk/sitemap.txt"
 ];
 
 visitConfigPrint(config);
 
 const runMain = async () => {
-  //prejmenovat linksToarary ProcessSitemap
-  const linksToVisit = await processSitemap(program.args[0], sitesInput(options.inputFile));
+  const linksToVisit = await processSitemap(
+    program.args[0],
+    sitesInput(options.inputFile),
+    config,
+  );
 
   await visitSitesWinston(linksToVisit, config);
 };

src/link-visit.ts

@@ -1,6 +1,5 @@
-import exp from "constants";
-import { BrowserContext, Page, chromium, devices } from "playwright-chromium";
-import { formatConnectionMessage, newLogger, logger, printPrefix } from "./logger";
+import { BrowserContext, chromium, devices } from "playwright-chromium";
+import { formatConnectionMessage, logger, printPrefix } from "./logger";
 import chalk from "chalk";
 import { totalNumberOfLinks } from "./sitemap-parsers";
 import { configType } from "./types";
@@ -139,6 +138,14 @@ const visitConfigPrint = (config: configType) => {
       ),
     );
   }
+  if (config.excludePattern) {
+    logger.log(
+      "info",
+      printPrefix(
+        `exclude: ${config.excludePattern} | Will exclude sites with ${config.excludePattern} regex. Setup using -e.`,
+      ),
+    );
+  }
   logger.log(
     "info",
     printPrefix(
@@ -182,7 +189,7 @@ async function visitSitesWinston(links: string[], config: configType) {
   let numOfOK = 0;
   let numOfErrors = 0;
 
-  logger.log("info", `expected total number of links: ${totalNumberOfLinks}\n`);
+  logger.log("info", `\nExpected total number of links: ${totalNumberOfLinks}\n`);
 
   const sTotalTime = performance.now();
 

src/logger.ts

@@ -1,5 +1,4 @@
 import { createLogger, format, transports } from "winston";
-import chalk from "chalk";
 
 const newLogger = () => {
   //@ts-ignore