Commit 325fa3c3 authored by duanjinfei's avatar duanjinfei

fix crawl data

parent 1aa8e5ca
...@@ -36,31 +36,31 @@ async function fetchAllData(supabase, table: string, pageSize: number = 1000) { ...@@ -36,31 +36,31 @@ async function fetchAllData(supabase, table: string, pageSize: number = 1000) {
// 缓存机制:减少重复数据库查询 // 缓存机制:减少重复数据库查询
class DatabaseCache { class DatabaseCache {
private categoryCache = new Map<number, { id: number }>(); private categoryCache = new Map<string, { id: string }>();
private appNameIdCache = new Map<number, string>(); private appNameIdCache = new Map<number, string>();
constructor(supabase, table: string) { async init(supabase, table: string) {
const appAllData = await fetchAllData(supabase, table) const appAllData = await fetchAllData(supabase, table)
for (const app of appAllData) { for (const app of appAllData) {
appNameIdCache[app.app_id] = app.name this.appNameIdCache.set(app.app_id, app.name);
} }
} }
async getCategoryByPrettyUrl(supabase, pretty_url: string) { async getCategoryByPrettyUrl(supabase, pretty_url: string) {
if (!this.categoryCache.has(categoryId)) { if (!this.categoryCache.has(pretty_url)) {
const { data, error } = await supabase const { data, error } = await supabase
.from("category") .from("category")
.select("id") .select("id")
.eq("pretty_url", pretty_url) .eq("pretty_url", pretty_url)
.single(); .single();
if (error) throw error; if (error) throw error;
this.categoryCache.set(categoryId, data); this.categoryCache.set(pretty_url, data);
} }
return this.categoryCache.get(categoryId); return this.categoryCache.get(pretty_url);
} }
async checkAppExists(supabase, appId: number, appName: string) { async checkAppExists(appId: number, appName: string) {
return this.appNameCache[appId] == appName; return this.appNameIdCache.get(appId) === appName; // 通过 this.appNameIdCache 获取缓存值
} }
async batchInsertApps(supabase, apps: any[]) { async batchInsertApps(supabase, apps: any[]) {
...@@ -113,7 +113,8 @@ async function getTonAppInfo(supabase) { ...@@ -113,7 +113,8 @@ async function getTonAppInfo(supabase) {
]; ];
const baseUrl = "https://ton.app/_next/data/xTw0Vxd8Pu6ky2epaEtZJ/en/"; const baseUrl = "https://ton.app/_next/data/xTw0Vxd8Pu6ky2epaEtZJ/en/";
const databaseCache = new DatabaseCache(supabase, 'app'); const databaseCache = new DatabaseCache();
await databaseCache.init(supabase, 'app');
const appsToInsert = []; const appsToInsert = [];
for (const category of categories) { for (const category of categories) {
...@@ -174,8 +175,7 @@ Deno.serve(async (req) => { ...@@ -174,8 +175,7 @@ Deno.serve(async (req) => {
return new Response( return new Response(
JSON.stringify({ JSON.stringify({
message: 'Data uploaded successfully', message: 'Data uploaded successfully'
insertedAppsCount
}), }),
{ {
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment