ditto part 2
This commit is contained in:
@@ -16,7 +16,7 @@ class helper():
|
||||
|
||||
def remove_folders(folders_list):
|
||||
for folder in folders_list:
|
||||
logging.info(f"Clearing folder {folder}")
|
||||
logging.warning(f"Clearing folder {folder}")
|
||||
for file in folder:
|
||||
os.remove(file)
|
||||
if len(folder) > 0:
|
||||
@@ -26,7 +26,7 @@ class helper():
|
||||
# helper method to download media
|
||||
def download_media(url, filename):
|
||||
# get file first
|
||||
logging.info(f"Downloading {url} info {filename}")
|
||||
logging.warning(f"Downloading {url} warning {filename}")
|
||||
resp = requests.get(url)
|
||||
if resp.ok:
|
||||
# make sure directory structure exists
|
||||
|
||||
@@ -29,7 +29,7 @@ class reddit_scraper:
|
||||
for p in posts[::-1]:
|
||||
if helper.ts_older(p.created, self.seent[sub]):
|
||||
break
|
||||
logging.info(f"Scraping post {p.id}")
|
||||
logging.warning(f"Scraping post {p.id}")
|
||||
post_list.append(p)
|
||||
self.seent[sub] = posts[0].created
|
||||
return post_list
|
||||
@@ -82,7 +82,7 @@ class reddit_scraper:
|
||||
for url in urls:
|
||||
i += 1
|
||||
name = f"temp/{post.id}/{i}"
|
||||
logging.info(f"Downloading {url} ({i}/{len(urls)})")
|
||||
logging.warning(f"Downloading {url} ({i}/{len(urls)})")
|
||||
helper.download_media(url, name)
|
||||
local_urls.append(name)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user