feat: add top token urls to sitemap and improve script (#7429)

* feat: add top token urls to sitemap and improve script

* fix: remove unnecessary header

* fix: test

package.json

@@ -256,6 +256,7 @@
     "multicodec": "^3.0.1",
     "multihashes": "^4.0.2",
     "nock": "^13.3.3",
+    "node-fetch": "^3.3.2",
     "node-vibrant": "^3.2.1-alpha.1",
     "numbro": "^2.3.6",
     "polished": "^3.3.2",

scripts/generate-sitemap.js

@@ -3,23 +3,65 @@
 const fs = require('fs')
 const { parseStringPromise, Builder } = require('xml2js')
 
+const weekMs = 7 * 24 * 60 * 60 * 1000
+const nowISO = new Date().toISOString()
+
+const getQuery = (chain) => `
+  query {
+    topTokens(pageSize: 100, page: 1, chain: ${chain}, orderBy: VOLUME) {
+      address
+    }
+  }
+`
+const chains = ['ETHEREUM', 'ARBITRUM', 'OPTIMISM', 'POLYGON', 'BASE', 'BNB', 'CELO']
+
 fs.readFile('./public/sitemap.xml', 'utf8', async (err, data) => {
+  const sitemapURLs = {}
   try {
     const sitemap = await parseStringPromise(data)
-
-    const lastmodDate = new Date().toISOString()
     if (sitemap.urlset.url) {
       sitemap.urlset.url.forEach((url) => {
-        url['$'].lastmod = lastmodDate
+        const lastMod = new Date(url['$'].lastmod).getTime()
+        if (lastMod < Date.now() - weekMs) {
+          url['$'].lastmod = nowISO
+        }
+        sitemapURLs[url['$']['loc']] = true
       })
     }
+
+    for (const chainName of chains) {
+      const response = await fetch('https://api.uniswap.org/v1/graphql', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Origin: 'https://app.uniswap.org',
+        },
+        body: JSON.stringify({ query: getQuery(chainName) }),
+      })
+      const tokensJSON = await response.json()
+      const tokenAddresses = tokensJSON.data.topTokens.map((token) => token.address.toLowerCase())
+
+      tokenAddresses.forEach((address) => {
+        const tokenURL = `https://app.uniswap.org/tokens/${chainName.toLowerCase()}/${address}`
+        if (!(tokenURL in sitemapURLs)) {
+          sitemap.urlset.url.push({
+            $: {
+              loc: [tokenURL],
+              lastmod: [nowISO],
+              priority: [0.8],
+            },
+          })
+        }
+      })
+    }
+
     const builder = new Builder()
     const xml = builder.buildObject(sitemap)
     fs.writeFile('./public/sitemap.xml', xml, (error) => {
       if (error) throw error
       console.log('Sitemap updated')
     })
-  } catch {
-    throw new Error('Error parsing sitemap.xml')
+  } catch (e) {
+    console.error(e)
   }
 })

src/pages/routes.test.ts

@@ -11,9 +11,11 @@ describe('Routes', () => {
 
     const sitemapPaths = sitemap.urlset.url.map((url: any) => new URL(url['$'].loc).pathname)
 
-    sitemapPaths.forEach((path: string) => {
-      expect(pathNames).toContain(path)
-    })
+    pathNames
+      .filter((p) => !p.includes(':') && !p.includes('*') && !p.includes('not-found'))
+      .forEach((path: string) => {
+        expect(sitemapPaths).toContain(path)
+      })
   })
 
   /**

yarn.lock

@@ -9985,6 +9985,11 @@ data-uri-to-buffer@^2.0.0:
   resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-2.0.2.tgz#d296973d5a4897a5dbe31716d118211921f04770"
   integrity sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==
 
+data-uri-to-buffer@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e"
+  integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==
+
 data-urls@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"
@@ -11690,6 +11695,14 @@ fd-slicer@~1.1.0:
   dependencies:
     pend "~1.2.0"
 
+fetch-blob@^3.1.2, fetch-blob@^3.1.4:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9"
+  integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==
+  dependencies:
+    node-domexception "^1.0.0"
+    web-streams-polyfill "^3.0.3"
+
 fflate@^0.7.3:
   version "0.7.4"
   resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.7.4.tgz#61587e5d958fdabb5a9368a302c25363f4f69f50"
@@ -11929,6 +11942,13 @@ formdata-node@^4.3.1:
     node-domexception "1.0.0"
     web-streams-polyfill "4.0.0-beta.3"
 
+formdata-polyfill@^4.0.10:
+  version "4.0.10"
+  resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423"
+  integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
+  dependencies:
+    fetch-blob "^3.1.2"
+
 forwarded@0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811"
@@ -15647,7 +15667,7 @@ node-cache@^5.1.2:
   dependencies:
     clone "2.x"
 
-node-domexception@1.0.0:
+node-domexception@1.0.0, node-domexception@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
@@ -15659,6 +15679,15 @@ node-fetch@2.6.7, node-fetch@^2.2.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.2.tgz#d1e889bacdf733b4ff3b2b243eb7a12866a0b78b"
+  integrity sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==
+  dependencies:
+    data-uri-to-buffer "^4.0.0"
+    fetch-blob "^3.1.4"
+    formdata-polyfill "^4.0.10"
+
 node-forge@^1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -20494,7 +20523,7 @@ web-streams-polyfill@4.0.0-beta.3:
   resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
   integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
 
-web-streams-polyfill@^3.2.0, web-streams-polyfill@^3.2.1:
+web-streams-polyfill@^3.0.3, web-streams-polyfill@^3.2.0, web-streams-polyfill@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6"
   integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==