OSM-POI: Include brand property [DRAFT]

kuwala/pipelines/osm-poi/src/Downloader.py

@@ -1,7 +1,10 @@
 import os
 from python_utils.src.FileDownloader import download_file
 from python_utils.src.FileSelector import select_osm_file
-
+import urllib.request as req
+import zipfile
+import json
+import pandas as pd
 
 class Downloader:
     @staticmethod
@@ -23,3 +26,38 @@ def start(args):
         file_path += '.osm.pbf'
 
         download_file(url=args.url or file['url'], path=file_path)
+
+    def names_downloader():
+        # here, instead of cloning the repository that recommended using extra library,
+        # we download the whole repo in zip, then extract it.
+        if not os.path.exists('../tmp/name-suggestion-index-main'):
+            download_link='https://github.com/osmlab/name-suggestion-index/archive/refs/heads/main.zip'
+            req.urlretrieve(download_link, "../tmp/main.zip")
+            with zipfile.ZipFile('../tmp/main.zip', 'r') as zip_ref:
+                zip_ref.extractall('../tmp/')
+            os.remove('../tmp/main.zip')
+
+        file_paths=['../tmp/name-suggestion-index-main/data/brands','../tmp/name-suggestion-index-main/data/operators']
+        data = {'id': [], 'display_name': [], 'wiki_data': []}
+        for file_path in file_paths:
+            for folders in os.listdir(file_path):
+                if os.path.isdir(os.path.join(file_path,folders)):
+                    for files in os.listdir(os.path.join(file_path,folders)):
+                        with open(os.path.join(file_path,folders,files)) as f:
+                            file_content=json.load(f)
+                        for a in file_content['items'] :
+                            wiki_data=id=display_name='-'
+                            if ('id' in a.keys()):
+                                id=(dict(a)['id'])
+                            if ('displayName' in a.keys()):
+                                display_name=(dict(a)['displayName'])
+                            if ("tags" in a.keys()):
+                                if ('brand:wikidata' in list(a['tags'].keys())):
+                                    wiki_data=(dict(a["tags"].items())['brand:wikidata'])
+
+                            data['id'].append(id)
+                            data['display_name'].append(display_name)
+                            data['wiki_data'].append(wiki_data)
+
+        df=pd.DataFrame(data)
+        df.to_csv('../tmp/names.csv',index=False)

kuwala/pipelines/osm-poi/src/main.py

@@ -25,5 +25,6 @@
 
     if action == 'download':
         Downloader.start(args)
+        Downloader.names_downloader()
     else:
         Processor.start(args)