Add crawl_entry job

src/bin/web.rs

@@ -114,7 +114,7 @@ async fn main() -> Result<()> {
         config.content_dir.clone(),
         crawls.clone(),
     );
-    let _ = crawl_scheduler.bootstrap().await;
+    // let _ = crawl_scheduler.bootstrap().await;
     let importer = ImporterHandle::new(db.clone(), crawl_scheduler.clone(), imports.clone());
 
     let ip_source_extension = config.ip_source.0.clone().into_extension();

src/bin/worker.rs

@@ -49,6 +49,8 @@ async fn main() -> Result<()> {
                 .data(http_client)
                 .data(db)
                 .data(domain_request_limiter)
+                .data(config)
+                .data(apalis_storage.clone())
                 .backend(apalis_storage)
                 .build_fn(handle_async_job)
         })