diff --git a/README.md b/README.md index 374ffaa..4f6f350 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # Scraper for prnt.sc ## Introduction -The website [LightShot or prnt.sc](https://prnt.sc/) is a public image sharing website which is most well known for its quick and easy + +The website [LightShot or prnt.sc](https://prnt.sc/) is a public image sharing website which is most well known for its quick and easy downloadable sharing utility activated by pressing the PrtScn key. It's a very useful tool, however I noticed that it stores images -based on a sequential 6 digit code, meaning the 1.3 billion or so images uploaded there can be indexed programmatically quite easily. +based on a sequential 6 digit code, meaning the 1.3 billion or so images uploaded there can be indexed programmatically quite easily. That is what this utility does. ## Pre-downloaded Dataset @@ -20,6 +21,7 @@ This script was tested on the following python modules, however earlier/later ve - requests 2.8.1 - beautifulsoup4 4.6.0 - lxml 3.8.0 +- faker 8.11.0 ``` ## Using the Script @@ -36,13 +38,13 @@ The script takes 3 arguments as follows: ## Uses/Explanation It can be very interesting to see what people upload to these sites, generally having sequential IDs of any type is bad, and the -same applies here. People might not be aware that what they are uploading is visible to others, however prnt.sc/lightshot have +same applies here. People might not be aware that what they are uploading is visible to others, however prnt.sc/lightshot have not shown any inclination in wanting to change their site design. As a result, this provides a useful way to create datasets of real world images. A useful/interesting use case is building a machine learning algorithm to classify images into categories, which requires some manual classification, but is nonetheless interesting and a good learning task. -# Licensing - -This project is released under the MIT license, see LICENSE.md for more details. \ No newline at end of file +## Licensing + +This project is released under the MIT license, see LICENSE.md for more details. diff --git a/prntsc.py b/prntsc.py index 8a1216c..7c58b0e 100644 --- a/prntsc.py +++ b/prntsc.py @@ -1,16 +1,24 @@ +import argparse as parser +import mimetypes +import string +from pathlib import Path +import faker import requests +from urllib.parse import urljoin +# may require a 'pip install lxml' from bs4 import BeautifulSoup -import string -import argparse as parser -import os -# Standard headers to prevent problems while scraping. May not all be necessary, they are -# copied from what my browser sent to the page. +mimetypes.add_type("image/webp", ".webp") + +# Standard headers to prevent problems while scraping. They are necessary +# randomly generated using the faker library +fake = faker.Faker() +fake.add_provider(faker.providers.user_agent) headers = { 'authority': 'prnt.sc', 'cache-control': 'max-age=0', 'upgrade-insecure-requests': '1', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36', + 'user-agent': fake.chrome(), 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'accept-encoding': 'gzip, deflate', 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8' @@ -20,10 +28,15 @@ # The idea is that we can work in base 36 (length of all lowercase + digits) to add # one to a code i.e. if we have abcdef, we can essentially write abcdef + 1 to get # abcdeg, which is the next code. -code_chars = list(string.ascii_lowercase) + ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] +# order for prnt.sc appears to be numeric then alphabetic +code_chars = ["0", "1", "2", "3", "4", "5", "6", + "7", "8", "9"] + list(string.ascii_lowercase) + base = len(code_chars) # Converts digit to a letter based on character codes + + def digit_to_char(digit): if digit < 10: return str(digit) @@ -52,42 +65,57 @@ def get_img_url(code): html = requests.get(f"http://prnt.sc/{code}", headers=headers).text soup = BeautifulSoup(html, 'lxml') img_url = soup.find_all('img', {'class': 'no-click screenshot-image'}) - return img_url[0]['src'] + return urljoin("https://",img_url[0]['src']) # Saves image from URL -def get_img(url, path): - response = requests.get(url) - if response.status_code == 200: - with open(f"{path}.png", 'wb') as f: - f.write(response.content) +def get_img(path): + response = requests.get(get_img_url(path.stem), headers=headers) + path = path.with_stem(path.stem.zfill(7)) + response.raise_for_status() + with open(path.with_suffix(mimetypes.guess_extension(response.headers["content-type"])), 'wb') as f: + f.write(response.content) if __name__ == '__main__': parser = parser.ArgumentParser() - parser.add_argument('--start_code', help='6 character string made up of lowercase letters and numbers which is ' - 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', - default='lj9me9') - parser.add_argument('--count', help='The number of images to scrape.', default='200') - parser.add_argument('--output_path', help='The path where images will be stored.', default='output/') + parser.add_argument('--start_code', + help='6 or 7 character string made up of lowercase letters and numbers which is ' + 'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh', + default='10000rt') + + parser.add_argument( + '--resume_from_last', + help='If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).', + default=True) + + parser.add_argument( + '--count', + help='The number of images to scrape.', + default='200') + parser.add_argument( + '--output_path', + help='The path where images will be stored.', + default='output/') args = parser.parse_args() + output_path = Path(args.output_path) + output_path.mkdir(exist_ok=True) - if not os.path.exists(args.output_path): - os.makedirs(args.output_path) - - code = args.start_code + if(args.resume_from_last): + try: + code = max(output_path.iterdir(), + key=lambda f: int(f.stem, base)).stem + except ValueError: + code = "0" + code = str_base(max(int(code, base)+1, int(args.start_code, base)), base) for i in range(int(args.count)): - code = next_code(code) try: - url = get_img_url(code) - get_img(url, args.output_path + f"/{code}") + get_img(output_path.joinpath(code)) print(f"Saved image number {i}/{args.count} with code: {code}") - except: - print(f"Error with image: {code}") - - - - - + except KeyboardInterrupt: + break + except Exception as e: + print(f"{e} with image: {code}") + code = next_code(code)