From a47b28e40c9f177f3ce71a93bd26540e72a665a7 Mon Sep 17 00:00:00 2001 From: Mateusz-7 <80526301+Mateusz-7@users.noreply.github.com> Date: Sun, 3 Dec 2023 15:42:35 +0100 Subject: [PATCH 01/12] Merge pull request #1 * Added places' Data, Photos, Type. Changed source of Name and Cords. * Code cleanup. --- GoogleMyMaps/GoogleMyMaps.py | 78 ++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/GoogleMyMaps/GoogleMyMaps.py b/GoogleMyMaps/GoogleMyMaps.py index 5aad251..7e011e8 100644 --- a/GoogleMyMaps/GoogleMyMaps.py +++ b/GoogleMyMaps/GoogleMyMaps.py @@ -1,66 +1,94 @@ -# import sys import requests from bs4 import BeautifulSoup from pyjsparser import PyJsParser -class GoogleMyMaps(): +class GoogleMyMaps: def __init__(self): self.parser = PyJsParser() - def getFromMyMap(self, mapID): + @staticmethod + def get_from_my_map(map_id): r = requests.get( - "https://www.google.com/maps/d/edit?hl=ja&mid=" + mapID) + "https://www.google.com/maps/d/edit?&mid=" + map_id) return r - def parseData(self, r): + def parse_data(self, r): soup = BeautifulSoup(r.text, "html.parser") script = soup.find_all("script")[1].text js = self.parser.parse(script) - pagedata = js["body"][1]["declarations"][0]["init"]["value"] + page_data = js["body"][1]["declarations"][0]["init"]["value"] - data = pagedata.replace("true", "True") + data = page_data.replace("true", "True") data = data.replace("false", "False") data = data.replace("null", "None") data = data.replace("\n", "") - # exec("data = " + data) + data = data.replace('\xa0', ' ') + data = eval(data) return data[1] - def parseLayerData(self, layerData): - # layerName = layerData[2] - - places = layerData[4] - # url = places[0][0] + @staticmethod + def get_place_type_and_cords(place): + if place[1] is not None: + return 'Point', place[1][0][0] + elif place[2] is not None: + return 'Line', [cord[0] for cord in place[2][0][0]] + elif place[3] is not None: + return 'Polygon', [cord[0] for cord in place[3][0][0][0][0]] + + @staticmethod + def extract_place_data(place_info): + place_data = {} + if len(place_info) > 1 and place_info[1] is not None: + place_data[place_info[1][0]] = place_info[1][1][place_info[1][2] - 1] + if len(place_info) > 3 and place_info[3] is not None: + for info in place_info[3]: + place_data[info[0]] = info[1][info[2] - 1] + return place_data if place_data else None + + def parse_layer_data(self, layer_data): + # layerName = layerData[2] # TODO: Use + + places = layer_data[12][0][13][0] + # places_icons = layerData[12][0][13][1] -> [0][0] parsed = [] for place in places: - placeName = place[5][0][0] + place_type, place_cords = self.get_place_type_and_cords(place) + + place_info = place[5] + place_name = place_info[0][1][0] - info = place[4] - point = info[4] + place_photos = [photo[1] for photo in place_info[2]] \ + if len(place_info) > 2 and place_info[2] is not None else None + + place_data = self.extract_place_data(place_info) parsed.append({ - "placeName": placeName, - "point": point, + "Cords": place_cords, + "Data": place_data, + "Name": place_name, + "Photos": place_photos, + "Type": place_type, }) return parsed - def get(self, mapID, layers=[0]): - r = self.getFromMyMap(mapID) + def get(self, map_id, layers=[0]): + r = self.get_from_my_map(map_id) if r.status_code != 200: print("status_code:", r.status_code) raise - data = self.parseData(r) - # mapID = data[1] - # mapName = data[2] + data = self.parse_data(r) + + # mapName = data[2] # TODO: Use parsed = [] for layer in layers: - layerData = data[6][layer] - parsed += self.parseLayerData(layerData) + layer_data = data[6][layer] + parsed += self.parse_layer_data(layer_data) return parsed From 3e170b6d2a5a811dd330ba1fd07fec163929a367 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 00:03:32 +0100 Subject: [PATCH 02/12] Init classes. --- GoogleMyMaps/__init__.py | 1 + GoogleMyMaps/models/Layer.py | 2 ++ GoogleMyMaps/models/Map.py | 2 ++ GoogleMyMaps/models/Place.py | 2 ++ GoogleMyMaps/models/__init__.py | 3 +++ GoogleMyMaps/parsers/GoogleMyMapsParser.py | 2 ++ GoogleMyMaps/parsers/__init__.py | 1 + 7 files changed, 13 insertions(+) create mode 100644 GoogleMyMaps/models/Layer.py create mode 100644 GoogleMyMaps/models/Map.py create mode 100644 GoogleMyMaps/models/Place.py create mode 100644 GoogleMyMaps/models/__init__.py create mode 100644 GoogleMyMaps/parsers/GoogleMyMapsParser.py create mode 100644 GoogleMyMaps/parsers/__init__.py diff --git a/GoogleMyMaps/__init__.py b/GoogleMyMaps/__init__.py index 1fc0806..dd0c3b5 100644 --- a/GoogleMyMaps/__init__.py +++ b/GoogleMyMaps/__init__.py @@ -1 +1,2 @@ from .GoogleMyMaps import GoogleMyMaps +from .models import Map, Layer, Place diff --git a/GoogleMyMaps/models/Layer.py b/GoogleMyMaps/models/Layer.py new file mode 100644 index 0000000..4e10260 --- /dev/null +++ b/GoogleMyMaps/models/Layer.py @@ -0,0 +1,2 @@ +class Layer: + pass diff --git a/GoogleMyMaps/models/Map.py b/GoogleMyMaps/models/Map.py new file mode 100644 index 0000000..520abf7 --- /dev/null +++ b/GoogleMyMaps/models/Map.py @@ -0,0 +1,2 @@ +class Map: + pass diff --git a/GoogleMyMaps/models/Place.py b/GoogleMyMaps/models/Place.py new file mode 100644 index 0000000..8e27b33 --- /dev/null +++ b/GoogleMyMaps/models/Place.py @@ -0,0 +1,2 @@ +class Place: + pass diff --git a/GoogleMyMaps/models/__init__.py b/GoogleMyMaps/models/__init__.py new file mode 100644 index 0000000..2f30b74 --- /dev/null +++ b/GoogleMyMaps/models/__init__.py @@ -0,0 +1,3 @@ +from .Map import Map +from .Layer import Layer +from .Place import Place diff --git a/GoogleMyMaps/parsers/GoogleMyMapsParser.py b/GoogleMyMaps/parsers/GoogleMyMapsParser.py new file mode 100644 index 0000000..63a3fb9 --- /dev/null +++ b/GoogleMyMaps/parsers/GoogleMyMapsParser.py @@ -0,0 +1,2 @@ +class GoogleMyMapsParser: + pass diff --git a/GoogleMyMaps/parsers/__init__.py b/GoogleMyMaps/parsers/__init__.py new file mode 100644 index 0000000..ea1e982 --- /dev/null +++ b/GoogleMyMaps/parsers/__init__.py @@ -0,0 +1 @@ +from .GoogleMyMapsParser import GoogleMyMapsParser From f0affd6be6bcfc06b9cc18e28dfcc811bd36d029 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 00:06:45 +0100 Subject: [PATCH 03/12] Create map models. --- GoogleMyMaps/models/Layer.py | 4 +++- GoogleMyMaps/models/Map.py | 8 +++++++- GoogleMyMaps/models/Place.py | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/GoogleMyMaps/models/Layer.py b/GoogleMyMaps/models/Layer.py index 4e10260..96a70a4 100644 --- a/GoogleMyMaps/models/Layer.py +++ b/GoogleMyMaps/models/Layer.py @@ -1,2 +1,4 @@ class Layer: - pass + def __init__(self, name, places): + self.name = name + self.places = places diff --git a/GoogleMyMaps/models/Map.py b/GoogleMyMaps/models/Map.py index 520abf7..c8e43d6 100644 --- a/GoogleMyMaps/models/Map.py +++ b/GoogleMyMaps/models/Map.py @@ -1,2 +1,8 @@ class Map: - pass + def __init__(self, link, name, layers): + self.link = link + self.name = name + self.layers = layers + + def add_layer(self, layer): + self.layers.append(layer) diff --git a/GoogleMyMaps/models/Place.py b/GoogleMyMaps/models/Place.py index 8e27b33..bbed7f8 100644 --- a/GoogleMyMaps/models/Place.py +++ b/GoogleMyMaps/models/Place.py @@ -1,2 +1,7 @@ class Place: - pass + def __init__(self, name, coords, photos, place_type, data): + self.name = name + self.coords = coords + self.photos = photos if photos else None + self.place_type = place_type + self.data = data From 62118c52095d309383ccd5ca9f19d4380bc14b04 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 00:42:39 +0100 Subject: [PATCH 04/12] Update Map model. --- GoogleMyMaps/models/Map.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/GoogleMyMaps/models/Map.py b/GoogleMyMaps/models/Map.py index c8e43d6..0efe75c 100644 --- a/GoogleMyMaps/models/Map.py +++ b/GoogleMyMaps/models/Map.py @@ -3,6 +3,3 @@ def __init__(self, link, name, layers): self.link = link self.name = name self.layers = layers - - def add_layer(self, layer): - self.layers.append(layer) From c1856a50a420e8c896fb39aa3f4b75ef4b900b24 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 01:03:09 +0100 Subject: [PATCH 05/12] Update GoogleMyMaps. --- GoogleMyMaps/GoogleMyMaps.py | 105 +++++++-------------- GoogleMyMaps/parsers/GoogleMyMapsParser.py | 35 ++++++- examples/example_simple.py | 12 ++- 3 files changed, 74 insertions(+), 78 deletions(-) diff --git a/GoogleMyMaps/GoogleMyMaps.py b/GoogleMyMaps/GoogleMyMaps.py index 7e011e8..e3d41dc 100644 --- a/GoogleMyMaps/GoogleMyMaps.py +++ b/GoogleMyMaps/GoogleMyMaps.py @@ -1,36 +1,40 @@ -import requests -from bs4 import BeautifulSoup -from pyjsparser import PyJsParser +from GoogleMyMaps.parsers import GoogleMyMapsParser +from .models import Map, Layer, Place class GoogleMyMaps: - def __init__(self): - self.parser = PyJsParser() - - @staticmethod - def get_from_my_map(map_id): - r = requests.get( - "https://www.google.com/maps/d/edit?&mid=" + map_id) - return r - - def parse_data(self, r): - soup = BeautifulSoup(r.text, "html.parser") - script = soup.find_all("script")[1].text - js = self.parser.parse(script) - page_data = js["body"][1]["declarations"][0]["init"]["value"] - - data = page_data.replace("true", "True") - data = data.replace("false", "False") - data = data.replace("null", "None") - data = data.replace("\n", "") - data = data.replace('\xa0', ' ') - - data = eval(data) - return data[1] + self.parser = GoogleMyMapsParser() + + def create_map(self, map_link, chosen_layers: list = None): + data = self.parser.get_map_data(map_link) + name = data[2] if len(data) > 2 else "Unnamed Map" + chosen_layers = self._parse_layers(data[6], chosen_layers) if len(data) > 6 else [] + return Map(map_link, name, chosen_layers) + + def _parse_layers(self, layers_data, chosen_layers=None): + layers = [] + for index, layer_data in enumerate(layers_data): + if chosen_layers is None or index in chosen_layers: + layer_name = layer_data[2] if len(layer_data) > 2 else f"Unnamed Layer {index + 1}" + places = self._parse_places(layer_data[12][0][13][0]) if len(layer_data) > 12 else [] + layers.append(Layer(layer_name, places)) + return layers + + def _parse_places(self, places_data): + places = [] + for place_data in places_data: + place_type, coords = self._get_place_type_and_coords(place_data) if len(place_data) > 5 else (None, None) + + place_info = place_data[5] if len(place_data) > 5 else None + name = place_info[0][1][0] if place_info is not None and len(place_info[0]) > 1 else "Unnamed Place" + photos = [photo[1] for photo in place_info[2]] if len(place_info) > 2 and place_info[2] else None + data = self._extract_place_data(place_info) if len(place_info) > 5 else None + places.append(Place(name, coords, photos, place_type, data)) + return places @staticmethod - def get_place_type_and_cords(place): + def _get_place_type_and_coords(place): if place[1] is not None: return 'Point', place[1][0][0] elif place[2] is not None: @@ -39,7 +43,7 @@ def get_place_type_and_cords(place): return 'Polygon', [cord[0] for cord in place[3][0][0][0][0]] @staticmethod - def extract_place_data(place_info): + def _extract_place_data(place_info): place_data = {} if len(place_info) > 1 and place_info[1] is not None: place_data[place_info[1][0]] = place_info[1][1][place_info[1][2] - 1] @@ -47,48 +51,3 @@ def extract_place_data(place_info): for info in place_info[3]: place_data[info[0]] = info[1][info[2] - 1] return place_data if place_data else None - - def parse_layer_data(self, layer_data): - # layerName = layerData[2] # TODO: Use - - places = layer_data[12][0][13][0] - # places_icons = layerData[12][0][13][1] -> [0][0] - - parsed = [] - for place in places: - place_type, place_cords = self.get_place_type_and_cords(place) - - place_info = place[5] - place_name = place_info[0][1][0] - - place_photos = [photo[1] for photo in place_info[2]] \ - if len(place_info) > 2 and place_info[2] is not None else None - - place_data = self.extract_place_data(place_info) - - parsed.append({ - "Cords": place_cords, - "Data": place_data, - "Name": place_name, - "Photos": place_photos, - "Type": place_type, - }) - - return parsed - - def get(self, map_id, layers=[0]): - r = self.get_from_my_map(map_id) - if r.status_code != 200: - print("status_code:", r.status_code) - raise - - data = self.parse_data(r) - - # mapName = data[2] # TODO: Use - - parsed = [] - for layer in layers: - layer_data = data[6][layer] - parsed += self.parse_layer_data(layer_data) - - return parsed diff --git a/GoogleMyMaps/parsers/GoogleMyMapsParser.py b/GoogleMyMaps/parsers/GoogleMyMapsParser.py index 63a3fb9..4b74df4 100644 --- a/GoogleMyMaps/parsers/GoogleMyMapsParser.py +++ b/GoogleMyMaps/parsers/GoogleMyMapsParser.py @@ -1,2 +1,35 @@ +import requests +from bs4 import BeautifulSoup +from pyjsparser import PyJsParser + + class GoogleMyMapsParser: - pass + def __init__(self): + self.parser = PyJsParser() + + def get_map_data(self, map_id): + raw_data = self._fetch_data(map_id) + parsed_data = self._parse_data(raw_data) + return parsed_data + + @staticmethod + def _fetch_data(map_link: str): + # TODO: map_link validation + url = map_link + response = requests.get(url) + + if response.status_code != 200: + raise Exception(f"Failed to fetch map data. Status code: {response.status_code}") + + return response.text + + def _parse_data(self, raw_data): + soup = BeautifulSoup(raw_data, "html.parser") + script = soup.find_all("script")[1].text + js = self.parser.parse(script) + page_data = js["body"][1]["declarations"][0]["init"]["value"] + + data = page_data.replace("true", "True").replace("false", "False").replace("null", "None") + data = data.replace("\n", "").replace('\xa0', ' ') + + return eval(data)[1] diff --git a/examples/example_simple.py b/examples/example_simple.py index 5c5785b..38134ad 100644 --- a/examples/example_simple.py +++ b/examples/example_simple.py @@ -1,10 +1,14 @@ from GoogleMyMaps import GoogleMyMaps -from pprint import pprint -mapID = "YOUR_MAP_ID" +map_link = "YOUR MAP LINK" if __name__ == "__main__": gmm = GoogleMyMaps() - data = gmm.get(mapID) - pprint(data) + my_map = gmm.create_map(map_link, [1]) + print(my_map.name, end='\n\n') + + for layer in my_map.layers: + print(f"Layer: {layer.name}") + for place in layer.places: + print(f" Place: {place.name}") From eeb6eaf8c5618add1bd93ed40c783978ced2ad96 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 01:33:42 +0100 Subject: [PATCH 06/12] Added str for models. --- GoogleMyMaps/models/Layer.py | 5 +++++ GoogleMyMaps/models/Map.py | 6 ++++++ GoogleMyMaps/models/Place.py | 10 +++++++++- examples/example_simple.py | 13 ++++--------- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/GoogleMyMaps/models/Layer.py b/GoogleMyMaps/models/Layer.py index 96a70a4..8e8d653 100644 --- a/GoogleMyMaps/models/Layer.py +++ b/GoogleMyMaps/models/Layer.py @@ -2,3 +2,8 @@ class Layer: def __init__(self, name, places): self.name = name self.places = places + + def __str__(self): + places_str = '\n'.join([f" {place}" for place in self.places]) if self.places else "No places" + return f'Layer: {self.name}\n' \ + f' Places:\n{places_str}\n' diff --git a/GoogleMyMaps/models/Map.py b/GoogleMyMaps/models/Map.py index 0efe75c..ab378e6 100644 --- a/GoogleMyMaps/models/Map.py +++ b/GoogleMyMaps/models/Map.py @@ -3,3 +3,9 @@ def __init__(self, link, name, layers): self.link = link self.name = name self.layers = layers + + def __str__(self): + layers_str = '\n'.join([f" {layer}" for layer in self.layers]) if self.layers else "No layers" + return f'Map: {self.name}\n' \ + f'Link: {self.link}\n\n' \ + f'Layers:\n{layers_str}\n' diff --git a/GoogleMyMaps/models/Place.py b/GoogleMyMaps/models/Place.py index bbed7f8..c338160 100644 --- a/GoogleMyMaps/models/Place.py +++ b/GoogleMyMaps/models/Place.py @@ -1,7 +1,15 @@ class Place: def __init__(self, name, coords, photos, place_type, data): self.name = name + self.place_type = place_type self.coords = coords self.photos = photos if photos else None - self.place_type = place_type self.data = data + + def __str__(self): + data_str = '\n'.join([f" {data}" for data in self.data]) if self.data else " No data" + return f'Place: {self.name}\n' \ + f' Place type: {self.place_type}\n' \ + f' Coordinates: {self.coords}\n' \ + f' Photos: {self.photos}\n' \ + f' Data:\n{data_str}\n' diff --git a/examples/example_simple.py b/examples/example_simple.py index 38134ad..7ef6acb 100644 --- a/examples/example_simple.py +++ b/examples/example_simple.py @@ -1,14 +1,9 @@ from GoogleMyMaps import GoogleMyMaps -map_link = "YOUR MAP LINK" +map_link = 'YOUR_MAP_LINK' -if __name__ == "__main__": +if __name__ == '__main__': gmm = GoogleMyMaps() - my_map = gmm.create_map(map_link, [1]) - print(my_map.name, end='\n\n') - - for layer in my_map.layers: - print(f"Layer: {layer.name}") - for place in layer.places: - print(f" Place: {place.name}") + my_map = gmm.create_map(map_link) + print(my_map) From 40198ad506bfe613332c00f2d94fef77edd43b88 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 01:34:28 +0100 Subject: [PATCH 07/12] Change " to '. --- GoogleMyMaps/GoogleMyMaps.py | 6 +++--- GoogleMyMaps/parsers/GoogleMyMapsParser.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/GoogleMyMaps/GoogleMyMaps.py b/GoogleMyMaps/GoogleMyMaps.py index e3d41dc..ecba2ee 100644 --- a/GoogleMyMaps/GoogleMyMaps.py +++ b/GoogleMyMaps/GoogleMyMaps.py @@ -8,7 +8,7 @@ def __init__(self): def create_map(self, map_link, chosen_layers: list = None): data = self.parser.get_map_data(map_link) - name = data[2] if len(data) > 2 else "Unnamed Map" + name = data[2] if len(data) > 2 else 'Unnamed Map' chosen_layers = self._parse_layers(data[6], chosen_layers) if len(data) > 6 else [] return Map(map_link, name, chosen_layers) @@ -16,7 +16,7 @@ def _parse_layers(self, layers_data, chosen_layers=None): layers = [] for index, layer_data in enumerate(layers_data): if chosen_layers is None or index in chosen_layers: - layer_name = layer_data[2] if len(layer_data) > 2 else f"Unnamed Layer {index + 1}" + layer_name = layer_data[2] if len(layer_data) > 2 else f'Unnamed Layer {index + 1}' places = self._parse_places(layer_data[12][0][13][0]) if len(layer_data) > 12 else [] layers.append(Layer(layer_name, places)) return layers @@ -27,7 +27,7 @@ def _parse_places(self, places_data): place_type, coords = self._get_place_type_and_coords(place_data) if len(place_data) > 5 else (None, None) place_info = place_data[5] if len(place_data) > 5 else None - name = place_info[0][1][0] if place_info is not None and len(place_info[0]) > 1 else "Unnamed Place" + name = place_info[0][1][0] if place_info is not None and len(place_info[0]) > 1 else 'Unnamed Place' photos = [photo[1] for photo in place_info[2]] if len(place_info) > 2 and place_info[2] else None data = self._extract_place_data(place_info) if len(place_info) > 5 else None places.append(Place(name, coords, photos, place_type, data)) diff --git a/GoogleMyMaps/parsers/GoogleMyMapsParser.py b/GoogleMyMaps/parsers/GoogleMyMapsParser.py index 4b74df4..c6eebc3 100644 --- a/GoogleMyMaps/parsers/GoogleMyMapsParser.py +++ b/GoogleMyMaps/parsers/GoogleMyMapsParser.py @@ -19,17 +19,17 @@ def _fetch_data(map_link: str): response = requests.get(url) if response.status_code != 200: - raise Exception(f"Failed to fetch map data. Status code: {response.status_code}") + raise Exception(f'Failed to fetch map data. Status code: {response.status_code}') return response.text def _parse_data(self, raw_data): - soup = BeautifulSoup(raw_data, "html.parser") - script = soup.find_all("script")[1].text + soup = BeautifulSoup(raw_data, 'html.parser') + script = soup.find_all('script')[1].text js = self.parser.parse(script) - page_data = js["body"][1]["declarations"][0]["init"]["value"] + page_data = js['body'][1]['declarations'][0]['init']['value'] - data = page_data.replace("true", "True").replace("false", "False").replace("null", "None") - data = data.replace("\n", "").replace('\xa0', ' ') + data = page_data.replace('true', 'True').replace('false', 'False').replace('null', 'None') + data = data.replace('\n', '').replace('\xa0', ' ') return eval(data)[1] From 3eb2a4de1a594a1a870ffbfd32b12fbc64351403 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 16:02:49 +0100 Subject: [PATCH 08/12] Updated models. --- GoogleMyMaps/models/Layer.py | 9 ++++++--- GoogleMyMaps/models/Map.py | 11 +++++++---- GoogleMyMaps/models/Place.py | 28 +++++++++++++++++++--------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/GoogleMyMaps/models/Layer.py b/GoogleMyMaps/models/Layer.py index 8e8d653..b65d879 100644 --- a/GoogleMyMaps/models/Layer.py +++ b/GoogleMyMaps/models/Layer.py @@ -1,9 +1,12 @@ +from . import Place + + class Layer: - def __init__(self, name, places): + def __init__(self, name: str, places: list[Place]): self.name = name self.places = places def __str__(self): - places_str = '\n'.join([f" {place}" for place in self.places]) if self.places else "No places" + places_str = '\n'.join([f" {place}" for place in self.places]) if self.places else "No places" return f'Layer: {self.name}\n' \ - f' Places:\n{places_str}\n' + f'{places_str}\n' diff --git a/GoogleMyMaps/models/Map.py b/GoogleMyMaps/models/Map.py index ab378e6..8a1342f 100644 --- a/GoogleMyMaps/models/Map.py +++ b/GoogleMyMaps/models/Map.py @@ -1,11 +1,14 @@ +from . import Layer + + class Map: - def __init__(self, link, name, layers): + def __init__(self, link: str, name: str, layers: list[Layer]): self.link = link self.name = name self.layers = layers def __str__(self): layers_str = '\n'.join([f" {layer}" for layer in self.layers]) if self.layers else "No layers" - return f'Map: {self.name}\n' \ - f'Link: {self.link}\n\n' \ - f'Layers:\n{layers_str}\n' + return f'Link: {self.link}\n' \ + f'Map: {self.name}\n' \ + f'{layers_str}\n' diff --git a/GoogleMyMaps/models/Place.py b/GoogleMyMaps/models/Place.py index c338160..f084806 100644 --- a/GoogleMyMaps/models/Place.py +++ b/GoogleMyMaps/models/Place.py @@ -1,15 +1,25 @@ class Place: - def __init__(self, name, coords, photos, place_type, data): - self.name = name + def __init__(self, + place_type: str, + name: str, + coords: list[float] or list[list[float]], + photos: list[str] or None, + data: dict or None): self.place_type = place_type + self.name = name self.coords = coords - self.photos = photos if photos else None + self.photos = photos self.data = data def __str__(self): - data_str = '\n'.join([f" {data}" for data in self.data]) if self.data else " No data" - return f'Place: {self.name}\n' \ - f' Place type: {self.place_type}\n' \ - f' Coordinates: {self.coords}\n' \ - f' Photos: {self.photos}\n' \ - f' Data:\n{data_str}\n' + photos_str = (' Photos:\n' + + ''.join([f" {photo}\n" for photo in self.photos])) \ + if self.photos else '' + data_str = (' Data:\n' + + ''.join([f" {data}: {self.data[data]}\n" for data in self.data])) \ + if self.data else '' + + return f'{self.place_type}: {self.name}\n' \ + f' Coordinates: {self.coords}\n' \ + f'{photos_str}' \ + f'{data_str}' From 225b10244f4f8560e5f20afe209084b666b7e2a2 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 16:06:12 +0100 Subject: [PATCH 09/12] Added map link simple validation. --- GoogleMyMaps/parsers/GoogleMyMapsParser.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/GoogleMyMaps/parsers/GoogleMyMapsParser.py b/GoogleMyMaps/parsers/GoogleMyMapsParser.py index c6eebc3..6be8d69 100644 --- a/GoogleMyMaps/parsers/GoogleMyMapsParser.py +++ b/GoogleMyMaps/parsers/GoogleMyMapsParser.py @@ -1,3 +1,5 @@ +import re + import requests from bs4 import BeautifulSoup from pyjsparser import PyJsParser @@ -7,23 +9,31 @@ class GoogleMyMapsParser: def __init__(self): self.parser = PyJsParser() - def get_map_data(self, map_id): - raw_data = self._fetch_data(map_id) + def get_map_data(self, map_link: str): + GoogleMyMapsParser._validate_map_link(map_link) + raw_data = GoogleMyMapsParser._fetch_data(map_link) parsed_data = self._parse_data(raw_data) return parsed_data + @staticmethod + def _validate_map_link(map_link: str): + map_link_pattern = re.compile( + r'https://www\.google\.com/maps/d/u/.*' + ) + + if not map_link_pattern.match(map_link): + raise ValueError('Invalid map link format.') + @staticmethod def _fetch_data(map_link: str): - # TODO: map_link validation - url = map_link - response = requests.get(url) + response = requests.get(map_link) if response.status_code != 200: raise Exception(f'Failed to fetch map data. Status code: {response.status_code}') return response.text - def _parse_data(self, raw_data): + def _parse_data(self, raw_data: str): soup = BeautifulSoup(raw_data, 'html.parser') script = soup.find_all('script')[1].text js = self.parser.parse(script) From 9c32608e09cbd3982a7e68dcd52bd889c9da1aa4 Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 16:19:25 +0100 Subject: [PATCH 10/12] Update GoogleMyMaps and add icon. --- GoogleMyMaps/GoogleMyMaps.py | 27 ++++++++++++++++----------- GoogleMyMaps/models/Place.py | 3 +++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/GoogleMyMaps/GoogleMyMaps.py b/GoogleMyMaps/GoogleMyMaps.py index ecba2ee..627ce33 100644 --- a/GoogleMyMaps/GoogleMyMaps.py +++ b/GoogleMyMaps/GoogleMyMaps.py @@ -9,28 +9,33 @@ def __init__(self): def create_map(self, map_link, chosen_layers: list = None): data = self.parser.get_map_data(map_link) name = data[2] if len(data) > 2 else 'Unnamed Map' - chosen_layers = self._parse_layers(data[6], chosen_layers) if len(data) > 6 else [] + chosen_layers = GoogleMyMaps._parse_layers(data[6], chosen_layers) if len(data) > 6 else [] return Map(map_link, name, chosen_layers) - def _parse_layers(self, layers_data, chosen_layers=None): + @staticmethod + def _parse_layers(layers_data, chosen_layers=None): layers = [] for index, layer_data in enumerate(layers_data): if chosen_layers is None or index in chosen_layers: layer_name = layer_data[2] if len(layer_data) > 2 else f'Unnamed Layer {index + 1}' - places = self._parse_places(layer_data[12][0][13][0]) if len(layer_data) > 12 else [] + places = GoogleMyMaps._parse_places(layer_data[12][0][13]) if len(layer_data) > 12 else [] layers.append(Layer(layer_name, places)) return layers - def _parse_places(self, places_data): + @staticmethod + def _parse_places(places_data): places = [] - for place_data in places_data: - place_type, coords = self._get_place_type_and_coords(place_data) if len(place_data) > 5 else (None, None) + for place_data, place_icon_data in zip(places_data[0], places_data[1]): + icon = place_icon_data[0][0] if place_icon_data and len(place_icon_data) > 0 else None + + place_type, coords = GoogleMyMaps._get_place_type_and_coords(place_data) if len(place_data) > 5 else (None, None) place_info = place_data[5] if len(place_data) > 5 else None - name = place_info[0][1][0] if place_info is not None and len(place_info[0]) > 1 else 'Unnamed Place' + name = place_info[0][1][0] if place_info and len(place_info[0]) > 1 else 'Unnamed Place' photos = [photo[1] for photo in place_info[2]] if len(place_info) > 2 and place_info[2] else None - data = self._extract_place_data(place_info) if len(place_info) > 5 else None - places.append(Place(name, coords, photos, place_type, data)) + data = GoogleMyMaps._extract_place_data(place_info) + + places.append(Place(place_type, name, icon, coords, photos, data)) return places @staticmethod @@ -45,9 +50,9 @@ def _get_place_type_and_coords(place): @staticmethod def _extract_place_data(place_info): place_data = {} - if len(place_info) > 1 and place_info[1] is not None: + if len(place_info) > 1 and place_info[1]: place_data[place_info[1][0]] = place_info[1][1][place_info[1][2] - 1] - if len(place_info) > 3 and place_info[3] is not None: + if len(place_info) > 3 and place_info[3]: for info in place_info[3]: place_data[info[0]] = info[1][info[2] - 1] return place_data if place_data else None diff --git a/GoogleMyMaps/models/Place.py b/GoogleMyMaps/models/Place.py index f084806..31d6385 100644 --- a/GoogleMyMaps/models/Place.py +++ b/GoogleMyMaps/models/Place.py @@ -2,11 +2,13 @@ class Place: def __init__(self, place_type: str, name: str, + icon: str or None, coords: list[float] or list[list[float]], photos: list[str] or None, data: dict or None): self.place_type = place_type self.name = name + self.icon = icon self.coords = coords self.photos = photos self.data = data @@ -20,6 +22,7 @@ def __str__(self): if self.data else '' return f'{self.place_type}: {self.name}\n' \ + f' Icon: {self.icon}\n' \ f' Coordinates: {self.coords}\n' \ f'{photos_str}' \ f'{data_str}' From 64cd8916b894308f148510cff3a3108d89718c5a Mon Sep 17 00:00:00 2001 From: Mateusz-7 Date: Fri, 8 Dec 2023 16:38:58 +0100 Subject: [PATCH 11/12] Update gitignore and setup. --- .gitignore | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++-- setup.py | 2 + 2 files changed, 215 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 62998c7..bfd2757 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,213 @@ -build -dist -*egg-info* -private.py -__pycache__ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + diff --git a/setup.py b/setup.py index a4ab9dd..713eea3 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ 'install_requires': [ 'beautifulsoup4', 'pyjsparser', + 'requests', + 're', ], } From b23573922c589f2a39f4912f094dc26dab41b35e Mon Sep 17 00:00:00 2001 From: Mateusz-7 <80526301+Mateusz-7@users.noreply.github.com> Date: Fri, 8 Dec 2023 16:49:53 +0100 Subject: [PATCH 12/12] Create README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..65081eb --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# GoogleMyMaps +Python parser for Google My Maps