feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)

2024-09-30 09:57:19 +08:00
parent fb49413a41
commit 369e1e6f58
38 changed files with 927 additions and 75 deletions
--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@@ -49,6 +49,7 @@ export type CrawlOptions = {
  excludes: string
  limit: number | string
  max_depth: number | string
+  use_sitemap: boolean
 }

 export type CrawlResultItem = {