bin/ircd: titlebot skips fetching titles from twitter.com

bin/ircd/script/bots/titlebot.py

@@ -4,6 +4,7 @@ import re
 import irc
 import requests
 from bs4 import BeautifulSoup
+from urllib.parse import urlparse
 
 ## IRC Config
 server = "127.0.0.1"
@@ -15,6 +16,9 @@ ircc.connect(server, port, channels, botnick)
 
 while True:
     text = ircc.get_response()
+    if not len(text) > 0:
+        continue
+    print(text)
     text_list = text.split(' ')
     if text_list[1] == "PRIVMSG":
         channel = text_list[2]
@@ -22,21 +26,23 @@ while True:
         url = re.findall(r'(https?://[^\s]+)', msg)
 
         for i in url:
+            parsed_url = urlparse(i)
+            if parsed_url.netloc.lower() in ['twitter.com','t.co'] and parsed_url.scheme != 'https':
+                continue
             reqs = requests.get(i)
             soup = BeautifulSoup(reqs.text, 'html.parser')
 
-            for title in soup.find_all('title'):
-                title_text = title.get_text()
-                if not len(title_text) > 0:
-                    print("Error: Title not found!")
+            title_text = soup.find('title').get_text()
+            if not len(title_text) > 0:
+                print("Error: Title not found!")
+                continue
+            title_text = title_text.split('\n')
+            title_msg = []
+            # remove empty lines from tweet body
+            for line in title_text:
+                if not line.strip():
                     continue
-                title_text = title_text.split('\n')
-                title_msg = []
-                # remove empty lines from tweet body
-                for line in title_text:
-                    if not line.strip():
-                        continue
-                    title_msg.append(line)
-                title_msg = " ".join(title_msg)
-                print(f"Title: {title_msg}")
-                ircc.send(channel, f"Title: {title_msg}")
+                title_msg.append(line)
+            title_msg = " ".join(title_msg)
+            print(f"Title: {title_msg}")
+            ircc.send(channel, f"Title: {title_msg}")