Days 47 - 54

Author: Plamen Milenkov

.DS_Store

@@ -0,0 +1,38 @@
+from typing import List
+
+import logbook
+import requests
+import bs4
+import os
+
+from bs4 import Tag
+
+
+def file_exists(file_name):
+    return os.path.isfile(file_name)
+
+def download_site(url):
+    if file_exists('sportal.html'):
+        print("Not downloading...")
+        return
+
+    response = requests.get(url)
+    response.raise_for_status()
+    with open('sportal.html', 'w') as f:
+        f.write(response.text)
+
+
+def get_top_news() -> List[Tag]:
+    top_news = []
+    with open('sportal.html') as f:
+        soup = bs4.BeautifulSoup(f, 'html.parser')
+    for item in soup.select('.orange_link'):
+        top_news.append(item)
+
+    return top_news
+
+def look_for_pattern(news: List[Tag], pattern):
+    for item in news:
+        if pattern in item.text:
+            print(f'Title {item.text}: link https://www.sportal.bg/{item.get("href")}')
+

days/.DS_Store

@@ -0,0 +1,16 @@
+import logbook
+import api
+
+logbook.RotatingFileHandler('sportal_scraper.log').push_application()
+logger = logbook.Logger('Main')
+
+
+def main():
+    logger.info('Starting application...')
+    api.download_site('https://www.sportal.bg')
+    top_news = api.get_top_news()
+    api.look_for_pattern(top_news, 'Мадрид')
+    api.look_for_pattern(top_news, 'Барселона')
+
+if __name__ == '__main__':
+    main()

days/46-48-beautifulsoup4/scrap_sportal/api.py

@@ -0,0 +1,3 @@
+requests
+bs4
+logbook

days/46-48-beautifulsoup4/scrap_sportal/program.py

@@ -0,0 +1,28 @@
+import feedparser
+import logbook
+
+logbook.RotatingFileHandler('parser.log').push_application()
+logger = logbook.Logger('feed_parser')
+
+def print_feed(feed, selected_tag):
+    logger.debug("Trying to print 'published', 'title' and 'link'...")
+    selected_feeds = [entry for entry in feed.entries if selected_tag in entry.tags[0].term]
+    for entry in selected_feeds:
+        try:
+            print(f'[{entry.tags[0].term}] {entry.published} - {entry.title}: {entry.link}')
+        except AttributeError as error:
+            logger.error(f'Could not print some of the attributes of the feed.')
+            print(f'Cannot find some of the attributes: {error}')
+
+
+def main():
+    logger.info('Starting the application...')
+    selected_tag = input('Preferred tag name:')
+    # day1_file = 'my_file_day1.xml'
+    # feed = feedparser.parse(day1_file)
+    day2_file = 'my_file_day2.xml'
+    feed = feedparser.parse(day2_file)
+    print_feed(feed, selected_tag)
+
+if __name__ == '__main__':
+    main()

days/46-48-beautifulsoup4/scrap_sportal/requirements.txt

@@ -0,0 +1,29 @@
+import requests
+
+
+def day1():
+    URL = "http://store.steampowered.com/feeds/newreleases.xml"
+    r = requests.get(URL)
+    r.raise_for_status()
+
+    with open("my_file_day1.xml", "w") as f:
+        f.write(r.text)
+
+
+def day2():
+    URL = "https://www.sportal.bg/uploads/rss_category_0.xml"
+    r = requests.get(URL)
+    r.raise_for_status()
+
+    with open("my_file_day2.xml", "w", encoding='iso_8859_1') as f:
+        f.write(r.text)
+
+
+def main():
+    # day1()
+    day2()
+
+
+
+if __name__ == '__main__':
+    main()