remove first element rather than last from sent list

.gitignore

@@ -1,3 +1,4 @@
 config.ini
 .devcontainer/
-.github/
+.github/
+cache.txt

main.py

@@ -6,13 +6,17 @@ import requests
 import json
 import datetime
 import smtplib
+import os
+import crcmod
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
 # Arguments parse
 parser = argparse.ArgumentParser(description='Reddit notify on keywords')
-parser.add_argument("--config", type=str, help="name of config file", 
+parser.add_argument("--config", type=str, help="location of config file", 
-                    default='config.ini.example')
+                    default='config.ini')
+parser.add_argument("--cache", type=str, help="location of cache file", 
+                    default='cache.txt')
 args = parser.parse_args()
 
 # Config parse
@@ -23,14 +27,46 @@ http_headers = {
     'User-Agent': 'Python_Reddit_Notif_Dtam/1.0'
 }
 
+# Keep track of sent
+crc32_func = crcmod.predefined.mkPredefinedCrcFun('crc-32')
+cache_file = ""
+sent = []
+
 # Etc
 nl = '\n'
 
 # Config error message
 def config_error(msg):
-    print(f'Error parsing config file: {msg}')
+    print_and_flush(f'Error parsing config file: {msg}')
     sys.exit(1)
 
+# Print and flush stdout
+def print_and_flush(msg):
+    print(msg)
+    sys.stdout.flush()
+
+# Check if message was sent previously
+def sent_previously(url):
+    # Simple hash to keep track of urls
+    encoded = url.encode('utf-8')
+    hashed_url = str(crc32_func(encoded))
+
+    # If not sent previously, add it to the sent list
+    if hashed_url not in sent:
+        sent.append(hashed_url)
+
+        # Prune sent list
+        while len(sent) > 100:
+            sent.pop(0)
+
+        # Write cache to disk
+        with open(cache_file, "w") as file:
+            file.write(', '.join(str(value) for value in sent))
+
+        return False
+    else:
+        return True
+
 # Read config file
 def get_config(filename):
     config = {}
@@ -43,7 +79,7 @@ def get_config(filename):
             raise configparser.Error
         
     except configparser.Error as e:
-        print(f'Error reading config file {e}')
+        print_and_flush(f'Error reading config file {e}')
         sys.exit(1)
 
     # Parse config file
@@ -104,6 +140,26 @@ def get_config(filename):
 
     return config
 
+# Get/Setup cache file
+def setup_cache(filename):
+    global cache_file
+    cache_file = filename
+
+    # If cache exists, read it and update sent
+    if os.path.exists(cache_file):
+        with open(cache_file, "r") as file:
+            content = file.read()
+            global sent
+            sent = [token.strip() for token in content.split(',')]
+
+            # Remove random empty string
+            sent = [item for item in sent if item != ""]
+
+        print_and_flush(f'Cache file found at: {filename}')
+    else:
+        with open(cache_file, "w") as file:
+            pass
+
 # Constant loop to check subreddit
 def check_reddit(config):
     subreddits = [subreddit.strip() for subreddit in config.get('subreddit').split(',')]
@@ -111,6 +167,10 @@ def check_reddit(config):
 
     while(True):
         for subreddit in subreddits:
+            # Debug message
+            current_time = datetime.datetime.now()
+            print_and_flush(f'Starting search at: {current_time.strftime("%Y-%m-%d %H:%M:%S")}')
+            
             resp = requests.get(f'https://www.reddit.com/r/{subreddit}/new.json', headers=http_headers)
 
             if resp.status_code == 200:
@@ -138,24 +198,31 @@ def check_reddit(config):
                             # Send alert
                             send_alert(config, title, text, url, timestamp, keyword)
         
-        time.sleep(config.get('interval'))
+        time.sleep(config.get('interval') * 60)
 
 # Send alert out
 def send_alert(config, title, text, url, timestamp, keyword):
+    # Check if sent previously
+    if sent_previously(url):
+        return
+        
     # Setup
     smtp_from = config.get('smtp_from')
     smtp_to = config.get('smtp_to')
+    time_format = datetime.datetime.fromtimestamp(timestamp)
+
+    # Debug message
+    print_and_flush(f'Found match: {title} at {time_format.strftime("%Y-%m-%d %H:%M:%S")}')
 
     # Setup message
     message = MIMEMultipart()
     message["From"] = smtp_from
     message["To"] = smtp_to
     message["Subject"] = f'Reddit Notify: Found Match ({title})'
-    time_format = datetime.datetime.fromtimestamp(timestamp)
     body = f'Keyword: {keyword}{nl}{nl}{nl}\
         {title}{nl}{nl}{nl}\
         {text}{nl}{nl}{nl}\
-        URL: {url}{nl}\
+        URL: https://www.reddit.com{url}{nl}\
         Time: {time_format.strftime("%Y-%m-%d %H:%M:%S")}'
     
     message.attach(MIMEText(body, "plain"))
@@ -164,18 +231,20 @@ def send_alert(config, title, text, url, timestamp, keyword):
     try:
         with smtplib.SMTP(config.get('smtp_server'),
                         config.get('smtp_port')) as server:
-            server.login(config.get('smtp_username'),
-                        config.get('smtp_password'))
             text = message.as_string()
 
             server.sendmail(smtp_from, smtp_to, text)
     except Exception as e:
-        print(f'SMTP Send Error: {e}')
+        print_and_flush(f'SMTP Send Error: {e}')
 
 if __name__ == '__main__':
 
+    # Setup config
     config = get_config(args.config)
+    print_and_flush(f'Current config file: {config}')
 
-    print(f'Current config file: {config}')
+    # Setup cache
+    setup_cache(args.cache)
 
+    # Main loop, check reddit
     check_reddit(config)

requirements.txt

@@ -1,5 +1,6 @@
 certifi==2024.2.2
 charset-normalizer==3.3.2
+crcmod==1.7
 idna==3.7
 requests==2.32.2
 setuptools==69.0.2