Generate `sitemap.xml` and `robots.txt` files (#2502)

* base implementation of sitemap generator

* move generator to separate directory to optimize docker image size

* generate robots.txt file

.gitignore

@@ -17,6 +17,8 @@
 /public/icons/sprite.svg
 /public/icons/sprite.*.svg
 /public/icons/README.md
+/public/sitemap.xml
+/public/robots.txt
 /analyze
 
 # production

Dockerfile

@@ -33,6 +33,12 @@ WORKDIR /favicon-generator
 COPY ./deploy/tools/favicon-generator/package.json ./deploy/tools/favicon-generator/yarn.lock ./
 RUN yarn --frozen-lockfile --network-timeout 100000
 
+### SITEMAP GENERATOR
+# Install dependencies
+WORKDIR /sitemap-generator
+COPY ./deploy/tools/sitemap-generator/package.json ./deploy/tools/sitemap-generator/yarn.lock ./
+RUN yarn --frozen-lockfile --network-timeout 100000
+
 
 # *****************************
 # ****** STAGE 2: Build *******
@@ -88,6 +94,12 @@ RUN cd ./deploy/tools/envs-validator && yarn build
 # Copy dependencies and source code
 COPY --from=deps /favicon-generator/node_modules ./deploy/tools/favicon-generator/node_modules
 
+
+### SITEMAP GENERATOR
+# Copy dependencies and source code
+COPY --from=deps /sitemap-generator/node_modules ./deploy/tools/sitemap-generator/node_modules
+
+
 # *****************************
 # ******* STAGE 3: Run ********
 # *****************************
@@ -127,6 +139,9 @@ COPY --chmod=755 ./deploy/scripts/favicon_generator.sh .
 COPY --from=builder /app/deploy/tools/favicon-generator ./deploy/tools/favicon-generator
 RUN ["chmod", "-R", "777", "./deploy/tools/favicon-generator"]
 RUN ["chmod", "-R", "777", "./public"]
+## Sitemap generator
+COPY --chmod=755 ./deploy/scripts/sitemap_generator.sh .
+COPY --from=builder /app/deploy/tools/sitemap-generator ./deploy/tools/sitemap-generator
 
 # Copy ENVs files
 COPY --from=builder /app/.env.registry .

deploy/scripts/entrypoint.sh

@@ -61,6 +61,9 @@ echo
 # Create envs.js file with run-time environment variables for the client app
 ./make_envs_script.sh
 
+# Generate sitemap.xml and robots.txt files
+./sitemap_generator.sh
+
 # Print list of enabled features
 node ./feature-reporter.js
 

deploy/scripts/sitemap_generator.sh

+cd ./deploy/tools/sitemap-generator
+yarn next-sitemap
\ No newline at end of file

deploy/tools/sitemap-generator/.gitignore

+/node_modules
\ No newline at end of file

deploy/tools/sitemap-generator/next-sitemap.config.js

+/* eslint-disable no-console */
+const path = require('path');
+
+const stripTrailingSlash = (str) => str[str.length - 1] === '/' ? str.slice(0, -1) : str;
+
+const fetchResource = async(url, formatter) => {
+  console.log('🌀 [next-sitemap] Fetching resource:', url);
+  try {
+    const res = await fetch(url);
+    if (res.ok) {
+      const data = await res.json();
+      console.log('✅ [next-sitemap] Data fetched for resource:', url);
+      return formatter(data);
+    }
+  } catch (error) {
+    console.log('🚨 [next-sitemap] Error fetching resource:', url, error);
+  }
+};
+
+const siteUrl = [
+  process.env.NEXT_PUBLIC_APP_PROTOCOL || 'https',
+  '://',
+  process.env.NEXT_PUBLIC_APP_HOST,
+  process.env.NEXT_PUBLIC_APP_PORT && ':' + process.env.NEXT_PUBLIC_APP_PORT,
+].filter(Boolean).join('');
+
+const apiUrl = (() => {
+  const baseUrl = [
+    process.env.NEXT_PUBLIC_API_PROTOCOL || 'https',
+    '://',
+    process.env.NEXT_PUBLIC_API_HOST,
+    process.env.NEXT_PUBLIC_API_PORT && ':' + process.env.NEXT_PUBLIC_API_PORT,
+  ].filter(Boolean).join('');
+
+  const basePath = stripTrailingSlash(process.env.NEXT_PUBLIC_API_BASE_PATH || '');
+
+  return `${ baseUrl }${ basePath }/api/v2`;
+})();
+
+/** @type {import('next-sitemap').IConfig} */
+module.exports = {
+  siteUrl,
+  generateIndexSitemap: false,
+  generateRobotsTxt: true,
+  sourceDir: path.resolve(process.cwd(), '../../../.next'),
+  outDir: path.resolve(process.cwd(), '../../../public'),
+  exclude: [
+    '/account/*',
+    '/auth/*',
+    '/login',
+    '/sprite',
+  ],
+  transform: async(config, path) => {
+    switch (path) {
+      case '/mud-worlds':
+        if (process.env.NEXT_PUBLIC_HAS_MUD_FRAMEWORK !== 'true') {
+          return null;
+        }
+        break;
+      case '/batches':
+      case '/deposits':
+        if (!process.env.NEXT_PUBLIC_ROLLUP_TYPE) {
+          return null;
+        }
+        break;
+      case '/withdrawals':
+        if (!process.env.NEXT_PUBLIC_ROLLUP_TYPE && process.env.NEXT_PUBLIC_HAS_BEACON_CHAIN !== 'true') {
+          return null;
+        }
+        break;
+      case '/dispute-games':
+        if (process.env.NEXT_PUBLIC_ROLLUP_TYPE !== 'optimistic') {
+          return null;
+        }
+        break;
+      case '/blobs':
+        if (process.env.NEXT_PUBLIC_DATA_AVAILABILITY_ENABLED !== 'true') {
+          return null;
+        }
+        break;
+      case '/name-domains':
+        if (!process.env.NEXT_PUBLIC_NAME_SERVICE_API_HOST) {
+          return null;
+        }
+        break;
+      case '/ops':
+        if (process.env.NEXT_PUBLIC_HAS_USER_OPS !== 'true') {
+          return null;
+        }
+        break;
+      case '/output-roots':
+        if (process.env.NEXT_PUBLIC_ROLLUP_OUTPUT_ROOTS_ENABLED !== 'true') {
+          return null;
+        }
+        break;
+      case '/pools':
+        if (process.env.NEXT_PUBLIC_DEX_POOLS_ENABLED !== 'true') {
+          return null;
+        }
+        break;
+      case '/advanced-filter':
+        if (process.env.NEXT_PUBLIC_ADVANCED_FILTER_ENABLED === 'false') {
+          return null;
+        }
+        break;
+      case '/apps':
+        if (process.env.NEXT_PUBLIC_MARKETPLACE_ENABLED !== 'true') {
+          return null;
+        }
+        break;
+      case '/api-docs':
+        if (process.env.NEXT_PUBLIC_API_SPEC_URL === 'none') {
+          return null;
+        }
+        break;
+      case '/gas-tracker':
+        if (process.env.NEXT_PUBLIC_GAS_TRACKER_ENABLED === 'false') {
+          return null;
+        }
+        break;
+      case '/graphql':
+        if (process.env.NEXT_PUBLIC_GRAPHIQL_TRANSACTION === 'none') {
+          return null;
+        }
+        break;
+      case '/stats':
+        if (!process.env.NEXT_PUBLIC_STATS_API_HOST) {
+          return null;
+        }
+        break;
+      case '/validators':
+        if (!process.env.NEXT_PUBLIC_VALIDATORS_CHAIN_TYPE) {
+          return null;
+        }
+        break;
+    }
+
+    return {
+      loc: path,
+      changefreq: undefined,
+      priority: undefined,
+      lastmod: config.autoLastmod ? new Date().toISOString() : undefined,
+      alternateRefs: config.alternateRefs ?? [],
+    };
+  },
+  additionalPaths: async(config) => {
+    const addresses = fetchResource(
+      `${ apiUrl }/addresses`,
+      (data) => data.items.map(({ hash }) => `/address/${ hash }`),
+    );
+    const txs = fetchResource(
+      `${ apiUrl }/transactions?filter=validated`,
+      (data) => data.items.map(({ hash }) => `/tx/${ hash }`),
+    );
+    const blocks = fetchResource(
+      `${ apiUrl }/blocks?type=block`,
+      (data) => data.items.map(({ height }) => `/block/${ height }`),
+    );
+    const tokens = fetchResource(
+      `${ apiUrl }/tokens`,
+      (data) => data.items.map(({ address }) => `/token/${ address }`),
+    );
+    const contracts = fetchResource(
+      `${ apiUrl }/smart-contracts`,
+      (data) => data.items.map(({ address }) => `/address/${ address.hash }?tab=contract`),
+    );
+
+    return Promise.all([
+      ...(await addresses || []),
+      ...(await txs || []),
+      ...(await blocks || []),
+      ...(await tokens || []),
+      ...(await contracts || []),
+    ].map(path => config.transform(config, path)));
+  },
+};

deploy/tools/sitemap-generator/package.json

+{
+  "name": "sitemap-generator",
+  "version": "1.0.0",
+  "main": "index.js",
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "next-sitemap": "4.2.3"
+  }
+}

deploy/tools/sitemap-generator/yarn.lock

+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@corex/deepmerge@^4.0.43":
+  version "4.0.43"
+  resolved "https://registry.yarnpkg.com/@corex/deepmerge/-/deepmerge-4.0.43.tgz#9bd42559ebb41cc5a7fb7cfeea5f231c20977dca"
+  integrity sha512-N8uEMrMPL0cu/bdboEWpQYb/0i2K5Qn8eCsxzOmxSggJbbQte7ljMRoXm917AbntqTGOzdTu+vP3KOOzoC70HQ==
+
+"@next/env@^13.4.3":
+  version "13.5.8"
+  resolved "https://registry.yarnpkg.com/@next/env/-/env-13.5.8.tgz#404d3b3e5881b6a0510500c6cc97e3589a2e6371"
+  integrity sha512-YmiG58BqyZ2FjrF2+5uZExL2BrLr8RTQzLXNDJ8pJr0O+rPlOeDPXp1p1/4OrR3avDidzZo3D8QO2cuDv1KCkw==
+
+"@nodelib/fs.scandir@2.1.5":
+  version "2.1.5"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
+  integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==
+  dependencies:
+    "@nodelib/fs.stat" "2.0.5"
+    run-parallel "^1.1.9"
+
+"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b"
+  integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==
+
+"@nodelib/fs.walk@^1.2.3":
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a"
+  integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==
+  dependencies:
+    "@nodelib/fs.scandir" "2.1.5"
+    fastq "^1.6.0"
+
+braces@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
+  integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
+  dependencies:
+    fill-range "^7.1.1"
+
+fast-glob@^3.2.12:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129"
+  integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==
+  dependencies:
+    "@nodelib/fs.stat" "^2.0.2"
+    "@nodelib/fs.walk" "^1.2.3"
+    glob-parent "^5.1.2"
+    merge2 "^1.3.0"
+    micromatch "^4.0.4"
+
+fastq@^1.6.0:
+  version "1.18.0"
+  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.18.0.tgz#d631d7e25faffea81887fe5ea8c9010e1b36fee0"
+  integrity sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==
+  dependencies:
+    reusify "^1.0.4"
+
+fill-range@^7.1.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
+  integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
+  dependencies:
+    to-regex-range "^5.0.1"
+
+glob-parent@^5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
+  integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
+  dependencies:
+    is-glob "^4.0.1"
+
+is-extglob@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
+  integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==
+
+is-glob@^4.0.1:
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
+  integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==
+  dependencies:
+    is-extglob "^2.1.1"
+
+is-number@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
+  integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==
+
+merge2@^1.3.0:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
+  integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
+
+micromatch@^4.0.4:
+  version "4.0.8"
+  resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202"
+  integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==
+  dependencies:
+    braces "^3.0.3"
+    picomatch "^2.3.1"
+
+minimist@^1.2.8:
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
+  integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
+
+next-sitemap@4.2.3:
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/next-sitemap/-/next-sitemap-4.2.3.tgz#5db3f650351a51e84b9fd6b58c5af2f9257b5058"
+  integrity sha512-vjdCxeDuWDzldhCnyFCQipw5bfpl4HmZA7uoo3GAaYGjGgfL4Cxb1CiztPuWGmS+auYs7/8OekRS8C2cjdAsjQ==
+  dependencies:
+    "@corex/deepmerge" "^4.0.43"
+    "@next/env" "^13.4.3"
+    fast-glob "^3.2.12"
+    minimist "^1.2.8"
+
+picomatch@^2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
+  integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
+
+queue-microtask@^1.2.2:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
+  integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
+
+reusify@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
+  integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
+
+run-parallel@^1.1.9:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
+  integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==
+  dependencies:
+    queue-microtask "^1.2.2"
+
+to-regex-range@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
+  integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==
+  dependencies:
+    is-number "^7.0.0"

package.json

@@ -33,6 +33,7 @@
     "test:jest": "jest",
     "test:jest:watch": "jest --watch",
     "favicon:generate:dev": "./tools/scripts/favicon-generator.dev.sh",
+    "sitemap:generate:dev": "./tools/scripts/sitemap-generator.dev.sh",
     "monitoring:prometheus:local": "docker run --name blockscout_prometheus -d -p 127.0.0.1:9090:9090 -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus",
     "monitoring:grafana:local": "docker run -d -p 4000:3000 --name=blockscout_grafana --user $(id -u) --volume $(pwd)/grafana:/var/lib/grafana grafana/grafana-enterprise"
   },
@@ -60,10 +61,10 @@
     "@opentelemetry/sdk-node": "0.49.1",
     "@opentelemetry/sdk-trace-node": "1.22.0",
     "@opentelemetry/semantic-conventions": "1.22.0",
-    "@rollbar/react": "0.12.0-beta",
-    "@scure/base": "1.1.9",
     "@reown/appkit": "1.6.0",
     "@reown/appkit-adapter-wagmi": "1.6.0",
+    "@rollbar/react": "0.12.0-beta",
+    "@scure/base": "1.1.9",
     "@slise/embed-react": "^2.2.0",
     "@tanstack/react-query": "5.55.4",
     "@tanstack/react-query-devtools": "5.55.4",

tools/scripts/sitemap-generator.dev.sh

+#!/bin/bash
+
+config_file="./configs/envs/.env.eth"
+
+if [ ! -f "$config_file" ]; then
+    echo "Error: File '$config_file' not found."
+    exit 1
+fi
+
+dotenv \
+  -e $config_file \
+  -- bash -c 'cd ./deploy/tools/sitemap-generator && yarn && yarn next-sitemap'
\ No newline at end of file