logburn/autotoot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ditto part 2

Browse Source
This commit is contained in:
Luke Ogburn
2022-04-20 20:14:22 -04:00
parent 93b1d0bfea
commit 58de863412
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
helper.py
View File

@@ -16,7 +16,7 @@ class helper():
def remove_folders(folders_list): def remove_folders(folders_list):
for folder in folders_list: for folder in folders_list:
logging.info(f"Clearing folder {folder}") logging.warning(f"Clearing folder {folder}")
for file in folder: for file in folder:
os.remove(file) os.remove(file)
if len(folder) > 0: if len(folder) > 0:
@@ -26,7 +26,7 @@ class helper():
# helper method to download media # helper method to download media
def download_media(url, filename): def download_media(url, filename):
# get file first # get file first
logging.info(f"Downloading {url} info {filename}") logging.warning(f"Downloading {url} warning {filename}")
resp = requests.get(url) resp = requests.get(url)
if resp.ok: if resp.ok:
# make sure directory structure exists # make sure directory structure exists

4
reddit.py
View File

@@ -29,7 +29,7 @@ class reddit_scraper:
for p in posts[::-1]: for p in posts[::-1]:
if helper.ts_older(p.created, self.seent[sub]): if helper.ts_older(p.created, self.seent[sub]):
break break
logging.info(f"Scraping post {p.id}") logging.warning(f"Scraping post {p.id}")
post_list.append(p) post_list.append(p)
self.seent[sub] = posts[0].created self.seent[sub] = posts[0].created
return post_list return post_list
@@ -82,7 +82,7 @@ class reddit_scraper:
for url in urls: for url in urls:
i += 1 i += 1
name = f"temp/{post.id}/{i}" name = f"temp/{post.id}/{i}"
logging.info(f"Downloading {url} ({i}/{len(urls)})") logging.warning(f"Downloading {url} ({i}/{len(urls)})")
helper.download_media(url, name) helper.download_media(url, name)
local_urls.append(name) local_urls.append(name)