headers and Multi-platform fix

README.md

@@ -1,9 +1,10 @@
 # Scraper for prnt.sc
 
 ## Introduction
-The website [LightShot or prnt.sc](https://prnt.sc/) is a public image sharing website which is most well known for its quick and easy 
+
+The website [LightShot or prnt.sc](https://prnt.sc/) is a public image sharing website which is most well known for its quick and easy
 downloadable sharing utility activated by pressing the PrtScn key. It's a very useful tool, however I noticed that it stores images
-based on a sequential 6 digit code, meaning the 1.3 billion or so images uploaded there can be indexed programmatically quite easily. 
+based on a sequential 6 digit code, meaning the 1.3 billion or so images uploaded there can be indexed programmatically quite easily.
 That is what this utility does.
 
 ## Pre-downloaded Dataset
@@ -20,6 +21,7 @@ This script was tested on the following python modules, however earlier/later ve
 - requests 2.8.1
 - beautifulsoup4 4.6.0
 - lxml 3.8.0
+- faker 8.11.0
 ```
 
 ## Using the Script
@@ -36,13 +38,13 @@ The script takes 3 arguments as follows:
 ## Uses/Explanation
 
 It can be very interesting to see what people upload to these sites, generally having sequential IDs of any type is bad, and the
-same applies here. People might not be aware that what they are uploading is visible to others, however prnt.sc/lightshot have 
+same applies here. People might not be aware that what they are uploading is visible to others, however prnt.sc/lightshot have
 not shown any inclination in wanting to change their site design.
 
 As a result, this provides a useful way to create datasets of real world images. A useful/interesting use case is building a machine
 learning algorithm to classify images into categories, which requires some manual classification, but is nonetheless interesting
 and a good learning task.
 
-# Licensing
- 
-This project is released under the MIT license, see LICENSE.md for more details.
+## Licensing
+
+This project is released under the MIT license, see LICENSE.md for more details.

prntsc.py

@@ -1,16 +1,24 @@
+import argparse as parser
+import mimetypes
+import string
+from pathlib import Path
+import faker
 import requests
+from urllib.parse import urljoin
+# may require a 'pip install lxml'
 from bs4 import BeautifulSoup
-import string
-import argparse as parser
-import os
 
-# Standard headers to prevent problems while scraping. May not all be necessary, they are
-# copied from what my browser sent to the page.
+mimetypes.add_type("image/webp", ".webp")
+
+# Standard headers to prevent problems while scraping. They are necessary
+# randomly generated using the faker library
+fake = faker.Faker()
+fake.add_provider(faker.providers.user_agent)
 headers = {
     'authority': 'prnt.sc',
     'cache-control': 'max-age=0',
     'upgrade-insecure-requests': '1',
-    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
+    'user-agent': fake.chrome(),
     'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
     'accept-encoding': 'gzip, deflate',
     'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8'
@@ -20,10 +28,15 @@
 # The idea is that we can work in base 36 (length of all lowercase + digits) to add
 # one to a code i.e. if we have abcdef, we can essentially write abcdef + 1 to get
 # abcdeg, which is the next code.
-code_chars = list(string.ascii_lowercase) + ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
+# order for prnt.sc appears to be numeric then alphabetic
+code_chars = ["0", "1", "2", "3", "4", "5", "6",
+              "7", "8", "9"] + list(string.ascii_lowercase)
+
 base = len(code_chars)
 
 # Converts digit to a letter based on character codes
+
+
 def digit_to_char(digit):
     if digit < 10:
         return str(digit)
@@ -52,42 +65,57 @@ def get_img_url(code):
     html = requests.get(f"http://prnt.sc/{code}", headers=headers).text
     soup = BeautifulSoup(html, 'lxml')
     img_url = soup.find_all('img', {'class': 'no-click screenshot-image'})
-    return img_url[0]['src']
+    return urljoin("https://",img_url[0]['src'])
 
 
 # Saves image from URL
-def get_img(url, path):
-    response = requests.get(url)
-    if response.status_code == 200:
-        with open(f"{path}.png", 'wb') as f:
-            f.write(response.content)
+def get_img(path):
+    response = requests.get(get_img_url(path.stem), headers=headers)
+    path = path.with_stem(path.stem.zfill(7))
+    response.raise_for_status()
+    with open(path.with_suffix(mimetypes.guess_extension(response.headers["content-type"])), 'wb') as f:
+        f.write(response.content)
 
 
 if __name__ == '__main__':
 
     parser = parser.ArgumentParser()
-    parser.add_argument('--start_code', help='6 character string made up of lowercase letters and numbers which is '
-                                             'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh',
-                                        default='lj9me9')
-    parser.add_argument('--count', help='The number of images to scrape.', default='200')
-    parser.add_argument('--output_path', help='The path where images will be stored.', default='output/')
+    parser.add_argument('--start_code',
+                        help='6 or 7 character string made up of lowercase letters and numbers which is '
+                        'where the scraper will start. e.g. abcdef -> abcdeg -> abcdeh',
+                        default='10000rt')
+
+    parser.add_argument(
+        '--resume_from_last',
+        help='If files allready exist in the output get last created/modified and resume from there (if --start_code < lastFile).',
+        default=True)
+
+    parser.add_argument(
+        '--count',
+        help='The number of images to scrape.',
+        default='200')
+    parser.add_argument(
+        '--output_path',
+        help='The path where images will be stored.',
+        default='output/')
 
     args = parser.parse_args()
+    output_path = Path(args.output_path)
+    output_path.mkdir(exist_ok=True)
 
-    if not os.path.exists(args.output_path):
-        os.makedirs(args.output_path)
-
-    code = args.start_code
+    if(args.resume_from_last):
+        try:
+            code = max(output_path.iterdir(),
+                       key=lambda f: int(f.stem, base)).stem
+        except ValueError:
+            code = "0"
+    code = str_base(max(int(code, base)+1, int(args.start_code, base)), base)
     for i in range(int(args.count)):
-        code = next_code(code)
         try:
-            url = get_img_url(code)
-            get_img(url, args.output_path + f"/{code}")
+            get_img(output_path.joinpath(code))
             print(f"Saved image number {i}/{args.count} with code: {code}")
-        except:
-            print(f"Error with image: {code}")
-
-
-
-
-
+        except KeyboardInterrupt:
+            break
+        except Exception as e:
+            print(f"{e} with image: {code}")
+        code = next_code(code)