From 4ace2164f2db63f53d369240bc0cfb945bec27ba Mon Sep 17 00:00:00 2001 From: Lacerum Date: Mon, 18 Apr 2022 18:38:16 -0700 Subject: [PATCH] more todos --- spacetime-crawler4py-master/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacetime-crawler4py-master/scraper.py b/spacetime-crawler4py-master/scraper.py index fdf6d60..cba8c3b 100644 --- a/spacetime-crawler4py-master/scraper.py +++ b/spacetime-crawler4py-master/scraper.py @@ -141,7 +141,7 @@ def is_valid(url): # https://support.archive-it.org/hc/en-us/articles/208332963-Modify-crawl-scope-with-a-Regular-Expression # add lem check # add another dir check - # add extra dir check + # add extra dir check (we can add as we find) # add cal check else: return True