Commit 325fa3c3 authored by duanjinfei's avatar duanjinfei

fix crawl data

parent 1aa8e5ca
......@@ -36,31 +36,31 @@ async function fetchAllData(supabase, table: string, pageSize: number = 1000) {
// 缓存机制:减少重复数据库查询
class DatabaseCache {
private categoryCache = new Map<number, { id: number }>();
private categoryCache = new Map<string, { id: string }>();
private appNameIdCache = new Map<number, string>();
constructor(supabase, table: string) {
async init(supabase, table: string) {
const appAllData = await fetchAllData(supabase, table)
for (const app of appAllData) {
appNameIdCache[app.app_id] = app.name
this.appNameIdCache.set(app.app_id, app.name);
}
}
async getCategoryByPrettyUrl(supabase, pretty_url: string) {
if (!this.categoryCache.has(categoryId)) {
if (!this.categoryCache.has(pretty_url)) {
const { data, error } = await supabase
.from("category")
.select("id")
.eq("pretty_url", pretty_url)
.single();
if (error) throw error;
this.categoryCache.set(categoryId, data);
this.categoryCache.set(pretty_url, data);
}
return this.categoryCache.get(categoryId);
return this.categoryCache.get(pretty_url);
}
async checkAppExists(supabase, appId: number, appName: string) {
return this.appNameCache[appId] == appName;
async checkAppExists(appId: number, appName: string) {
return this.appNameIdCache.get(appId) === appName; // 通过 this.appNameIdCache 获取缓存值
}
async batchInsertApps(supabase, apps: any[]) {
......@@ -113,7 +113,8 @@ async function getTonAppInfo(supabase) {
];
const baseUrl = "https://ton.app/_next/data/xTw0Vxd8Pu6ky2epaEtZJ/en/";
const databaseCache = new DatabaseCache(supabase, 'app');
const databaseCache = new DatabaseCache();
await databaseCache.init(supabase, 'app');
const appsToInsert = [];
for (const category of categories) {
......@@ -174,8 +175,7 @@ Deno.serve(async (req) => {
return new Response(
JSON.stringify({
message: 'Data uploaded successfully',
insertedAppsCount
message: 'Data uploaded successfully'
}),
{
headers: { 'Content-Type': 'application/json' },
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment