diff --git a/spacetime-crawler4py-master/scraper.py b/spacetime-crawler4py-master/scraper.py index dead1ea..cd78471 100644 --- a/spacetime-crawler4py-master/scraper.py +++ b/spacetime-crawler4py-master/scraper.py @@ -95,6 +95,7 @@ def is_valid(url): return False elif parsed.fragment: return False + # will add trap check here most likely else: return True