From 3468143a0a0a64d4a721c82a16083559bc95ae0e Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Mon, 6 Sep 2021 14:45:27 +0430 Subject: [PATCH 01/10] Fixed request headers --- prntsc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/prntsc.py b/prntsc.py index 8a1216c..9716e93 100644 --- a/prntsc.py +++ b/prntsc.py @@ -4,13 +4,16 @@ import argparse as parser import os -# Standard headers to prevent problems while scraping. May not all be necessary, they are -# copied from what my browser sent to the page. +import faker +# Standard headers to prevent problems while scraping. They are necessary +# randomly generated using the faker library +fake = faker.Faker() +fake.add_provider(faker.providers.user_agent) headers = { 'authority': 'prnt.sc', 'cache-control': 'max-age=0', 'upgrade-insecure-requests': '1', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36', + 'user-agent': fake.chrome(), 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'accept-encoding': 'gzip, deflate', 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8' @@ -57,7 +60,7 @@ def get_img_url(code): # Saves image from URL def get_img(url, path): - response = requests.get(url) + response = requests.get(url , headers=headers) if response.status_code == 200: with open(f"{path}.png", 'wb') as f: f.write(response.content) From 2c97de7dfa4cb40cbbb8f090a480c7b38c0f9c99 Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Mon, 6 Sep 2021 14:52:56 +0430 Subject: [PATCH 02/10] Fix for multi-platform paths and error handling --- prntsc.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/prntsc.py b/prntsc.py index 9716e93..4996439 100644 --- a/prntsc.py +++ b/prntsc.py @@ -2,9 +2,10 @@ from bs4 import BeautifulSoup import string import argparse as parser -import os +from pathlib import Path import faker +import mimetypes # Standard headers to prevent problems while scraping. They are necessary # randomly generated using the faker library fake = faker.Faker() @@ -59,11 +60,12 @@ def get_img_url(code): # Saves image from URL -def get_img(url, path): - response = requests.get(url , headers=headers) - if response.status_code == 200: - with open(f"{path}.png", 'wb') as f: - f.write(response.content) +def get_img(path): + response = requests.get(get_img_url(path.stem), headers=headers) + response.raise_for_status() + mimetypes.guess_extension(response.headers["content-type"]) + with open(path.with_suffix(), 'wb') as f: + f.write(response.content) if __name__ == '__main__': @@ -77,20 +79,15 @@ def get_img(url, path): args = parser.parse_args() - if not os.path.exists(args.output_path): - os.makedirs(args.output_path) - + output_path = Path(args.output_path) + output_path.mkdir(exist_ok=True) code = args.start_code for i in range(int(args.count)): code = next_code(code) try: - url = get_img_url(code) - get_img(url, args.output_path + f"/{code}") + get_img(output_path.joinpath(code)) print(f"Saved image number {i}/{args.count} with code: {code}") - except: - print(f"Error with image: {code}") - - - - - + except KeyboardInterrupt: + break + except Exception as e: + print(f"{e} with image: {code}") From 89ef48ad32c09122c0d8d81005d5e7917de905dc Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Mon, 6 Sep 2021 14:53:17 +0430 Subject: [PATCH 03/10] Formatting --- prntsc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/prntsc.py b/prntsc.py index 4996439..42f945d 100644 --- a/prntsc.py +++ b/prntsc.py @@ -1,11 +1,12 @@ -import requests -from bs4 import BeautifulSoup -import string import argparse as parser +import mimetypes +import string from pathlib import Path import faker -import mimetypes +import requests +from bs4 import BeautifulSoup + # Standard headers to prevent problems while scraping. They are necessary # randomly generated using the faker library fake = faker.Faker() From 5ec87aed75aa3ba9efd6bafa3ace1e84cde65f27 Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Tue, 7 Sep 2021 12:40:05 +0430 Subject: [PATCH 04/10] Fixed extension guessing --- prntsc.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/prntsc.py b/prntsc.py index 42f945d..44ab90d 100644 --- a/prntsc.py +++ b/prntsc.py @@ -7,7 +7,9 @@ import requests from bs4 import BeautifulSoup -# Standard headers to prevent problems while scraping. They are necessary +mimetypes.add_type("image/webp", ".webp") + +# Standard headers to prevent problems while scraping. They are necessary # randomly generated using the faker library fake = faker.Faker() fake.add_provider(faker.providers.user_agent) @@ -25,10 +27,13 @@ # The idea is that we can work in base 36 (length of all lowercase + digits) to add # one to a code i.e. if we have abcdef, we can essentially write abcdef + 1 to get # abcdeg, which is the next code. -code_chars = list(string.ascii_lowercase) + ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] +code_chars = list(string.ascii_lowercase) + \ + ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] base = len(code_chars) # Converts digit to a letter based on character codes + + def digit_to_char(digit): if digit < 10: return str(digit) @@ -64,8 +69,7 @@ def get_img_url(code): def get_img(path): response = requests.get(get_img_url(path.stem), headers=headers) response.raise_for_status() - mimetypes.guess_extension(response.headers["content-type"]) - with open(path.with_suffix(), 'wb') as f: + with open(path.with_suffix(mimetypes.guess_extension(response.headers["content-type"])), 'wb') as f: f.write(response.content) @@ -74,9 +78,11 @@ def get_img(path): parser = parser.ArgumentParser() parser.add_argument('--start_code', help='6 character string made up of lowercase letters and numbers which is ' 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', - default='lj9me9') - parser.add_argument('--count', help='The number of images to scrape.', default='200') - parser.add_argument('--output_path', help='The path where images will be stored.', default='output/') + default='lj9me9') + parser.add_argument( + '--count', help='The number of images to scrape.', default='200') + parser.add_argument( + '--output_path', help='The path where images will be stored.', default='output/') args = parser.parse_args() From fb3ff10d0704175b74e8e140f656b3a81969a340 Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Tue, 7 Sep 2021 15:13:06 +0430 Subject: [PATCH 05/10] Added faker to README.md requirements --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 374ffaa..4f6f350 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # Scraper for prnt.sc ## Introduction -The website [LightShot or prnt.sc](https://prnt.sc/) is a public image sharing website which is most well known for its quick and easy + +The website [LightShot or prnt.sc](https://prnt.sc/) is a public image sharing website which is most well known for its quick and easy downloadable sharing utility activated by pressing the PrtScn key. It's a very useful tool, however I noticed that it stores images -based on a sequential 6 digit code, meaning the 1.3 billion or so images uploaded there can be indexed programmatically quite easily. +based on a sequential 6 digit code, meaning the 1.3 billion or so images uploaded there can be indexed programmatically quite easily. That is what this utility does. ## Pre-downloaded Dataset @@ -20,6 +21,7 @@ This script was tested on the following python modules, however earlier/later ve - requests 2.8.1 - beautifulsoup4 4.6.0 - lxml 3.8.0 +- faker 8.11.0 ``` ## Using the Script @@ -36,13 +38,13 @@ The script takes 3 arguments as follows: ## Uses/Explanation It can be very interesting to see what people upload to these sites, generally having sequential IDs of any type is bad, and the -same applies here. People might not be aware that what they are uploading is visible to others, however prnt.sc/lightshot have +same applies here. People might not be aware that what they are uploading is visible to others, however prnt.sc/lightshot have not shown any inclination in wanting to change their site design. As a result, this provides a useful way to create datasets of real world images. A useful/interesting use case is building a machine learning algorithm to classify images into categories, which requires some manual classification, but is nonetheless interesting and a good learning task. -# Licensing - -This project is released under the MIT license, see LICENSE.md for more details. \ No newline at end of file +## Licensing + +This project is released under the MIT license, see LICENSE.md for more details. From 3e4791ac9343111119e87f67b41998fccb5e86bd Mon Sep 17 00:00:00 2001 From: "Jeremy D. Gerdes" Date: Sat, 11 Sep 2021 01:50:35 -0400 Subject: [PATCH 06/10] Update prntsc.py re-ordered code_chars to match sequence of prnt.sc updates help to acknowledge 6 or 7 characters allowed for --start_code --- prntsc.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/prntsc.py b/prntsc.py index 44ab90d..bf65ee0 100644 --- a/prntsc.py +++ b/prntsc.py @@ -2,9 +2,9 @@ import mimetypes import string from pathlib import Path - import faker import requests +# may require a 'pip install lxml' from bs4 import BeautifulSoup mimetypes.add_type("image/webp", ".webp") @@ -27,8 +27,9 @@ # The idea is that we can work in base 36 (length of all lowercase + digits) to add # one to a code i.e. if we have abcdef, we can essentially write abcdef + 1 to get # abcdeg, which is the next code. -code_chars = list(string.ascii_lowercase) + \ - ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] +# order for prnt.sc appears to be numeric then alphabetic +code_chars = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + list(string.ascii_lowercase) + base = len(code_chars) # Converts digit to a letter based on character codes @@ -75,14 +76,36 @@ def get_img(path): if __name__ == '__main__': + # --start_code is sequential, so from image context... + # 14akf6 ~ Oct 2013 + # 999997 ~ Jan 2015 + # a9998j ~ Feb 2016 + # h4akgb ~ Oct 2017 + # sp2gna ~ May 2020 ;sp2nuo=2020-05-27;sp2v18=2020-05-27;sp2v7o=2020-05-28 + # z4akga ~ Feb 2021 + # 10000am ~ Feb 18, 2021; 100001g=2021-02-18;10000rt=2021-02-18-194850 parser = parser.ArgumentParser() - parser.add_argument('--start_code', help='6 character string made up of lowercase letters and numbers which is ' - 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', - default='lj9me9') + parser.add_argument('--start_code', + help='6 or 7 character string made up of lowercase letters and numbers which is ' + 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', + default='10000rt') + + # [TODO] add argument as an improvement, by getting the last modifed file + # if they allready exist in the output folder and starting one after that :) + # parser.add_argument( + # '--resume_from_last', + # help='(PLANNED-Not yet implemented) If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).', + # default=True) + + # Default is 9 billion, just go forever parser.add_argument( - '--count', help='The number of images to scrape.', default='200') + '--count', + help='The number of images to scrape.', + default='9000000000') parser.add_argument( - '--output_path', help='The path where images will be stored.', default='output/') + '--output_path', + help='The path where images will be stored.', + default='output/') args = parser.parse_args() From 1c779b37a1e8c09d5639da34ba3f0e9d0c5e2281 Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Sun, 12 Sep 2021 11:48:21 +0430 Subject: [PATCH 07/10] Changed default amount back to 200 --- prntsc.py | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/prntsc.py b/prntsc.py index bf65ee0..8f35598 100644 --- a/prntsc.py +++ b/prntsc.py @@ -4,7 +4,7 @@ from pathlib import Path import faker import requests -# may require a 'pip install lxml' +# may require a 'pip install lxml' from bs4 import BeautifulSoup mimetypes.add_type("image/webp", ".webp") @@ -28,7 +28,8 @@ # one to a code i.e. if we have abcdef, we can essentially write abcdef + 1 to get # abcdeg, which is the next code. # order for prnt.sc appears to be numeric then alphabetic -code_chars = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + list(string.ascii_lowercase) +code_chars = ["0", "1", "2", "3", "4", "5", "6", + "7", "8", "9"] + list(string.ascii_lowercase) base = len(code_chars) @@ -76,35 +77,26 @@ def get_img(path): if __name__ == '__main__': - # --start_code is sequential, so from image context... - # 14akf6 ~ Oct 2013 - # 999997 ~ Jan 2015 - # a9998j ~ Feb 2016 - # h4akgb ~ Oct 2017 - # sp2gna ~ May 2020 ;sp2nuo=2020-05-27;sp2v18=2020-05-27;sp2v7o=2020-05-28 - # z4akga ~ Feb 2021 - # 10000am ~ Feb 18, 2021; 100001g=2021-02-18;10000rt=2021-02-18-194850 parser = parser.ArgumentParser() - parser.add_argument('--start_code', - help='6 or 7 character string made up of lowercase letters and numbers which is ' - 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', - default='10000rt') + parser.add_argument('--start_code', + help='6 or 7 character string made up of lowercase letters and numbers which is ' + 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', + default='10000rt') - # [TODO] add argument as an improvement, by getting the last modifed file + # [TODO] add argument as an improvement, by getting the last modifed file # if they allready exist in the output folder and starting one after that :) # parser.add_argument( - # '--resume_from_last', + # '--resume_from_last', # help='(PLANNED-Not yet implemented) If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).', # default=True) - # Default is 9 billion, just go forever parser.add_argument( - '--count', - help='The number of images to scrape.', - default='9000000000') + '--count', + help='The number of images to scrape.', + default='200') parser.add_argument( - '--output_path', - help='The path where images will be stored.', + '--output_path', + help='The path where images will be stored.', default='output/') args = parser.parse_args() From b210c2eac8b3aaa45959061f3f13cc7361f3c222 Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Sun, 12 Sep 2021 14:01:01 +0430 Subject: [PATCH 08/10] Implemented --resume_from_last option --- prntsc.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/prntsc.py b/prntsc.py index 8f35598..1feade7 100644 --- a/prntsc.py +++ b/prntsc.py @@ -85,10 +85,10 @@ def get_img(path): # [TODO] add argument as an improvement, by getting the last modifed file # if they allready exist in the output folder and starting one after that :) - # parser.add_argument( - # '--resume_from_last', - # help='(PLANNED-Not yet implemented) If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).', - # default=True) + parser.add_argument( + '--resume_from_last', + help='If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).', + default=True) parser.add_argument( '--count', @@ -100,12 +100,17 @@ def get_img(path): default='output/') args = parser.parse_args() - output_path = Path(args.output_path) output_path.mkdir(exist_ok=True) - code = args.start_code + + if(args.resume_from_last): + try: + code = max(output_path.iterdir(), + key=lambda f: int(f.stem, base)).stem + except ValueError: + code = "0" + code = str_base(max(int(code, base)+1, int(args.start_code, base)), base) for i in range(int(args.count)): - code = next_code(code) try: get_img(output_path.joinpath(code)) print(f"Saved image number {i}/{args.count} with code: {code}") @@ -113,3 +118,4 @@ def get_img(path): break except Exception as e: print(f"{e} with image: {code}") + code = next_code(code) From bc4020dcbedd31fa5fbc3cfb8a36bc7f86ed3a07 Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Sun, 12 Sep 2021 14:32:54 +0430 Subject: [PATCH 09/10] Fix for some images that don't have a schema --- prntsc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/prntsc.py b/prntsc.py index 1feade7..9612825 100644 --- a/prntsc.py +++ b/prntsc.py @@ -4,6 +4,7 @@ from pathlib import Path import faker import requests +from urllib.parse import urljoin # may require a 'pip install lxml' from bs4 import BeautifulSoup @@ -64,7 +65,7 @@ def get_img_url(code): html = requests.get(f"http://prnt.sc/{code}", headers=headers).text soup = BeautifulSoup(html, 'lxml') img_url = soup.find_all('img', {'class': 'no-click screenshot-image'}) - return img_url[0]['src'] + return urljoin("https://",img_url[0]['src']) # Saves image from URL @@ -83,8 +84,6 @@ def get_img(path): 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', default='10000rt') - # [TODO] add argument as an improvement, by getting the last modifed file - # if they allready exist in the output folder and starting one after that :) parser.add_argument( '--resume_from_last', help='If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).', From 545c62e1e70a810bf9f93101fca82b12649c327b Mon Sep 17 00:00:00 2001 From: vandad1901 Date: Sun, 12 Sep 2021 14:33:22 +0430 Subject: [PATCH 10/10] file names now zfill --- prntsc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/prntsc.py b/prntsc.py index 9612825..7c58b0e 100644 --- a/prntsc.py +++ b/prntsc.py @@ -71,6 +71,7 @@ def get_img_url(code): # Saves image from URL def get_img(path): response = requests.get(get_img_url(path.stem), headers=headers) + path = path.with_stem(path.stem.zfill(7)) response.raise_for_status() with open(path.with_suffix(mimetypes.guess_extension(response.headers["content-type"])), 'wb') as f: f.write(response.content)