remove first element rather than last from sent list

.gitignore

@@ -1,3 +1,4 @@
 config.ini
 .devcontainer/
 .github/
+cache.txt

Dockerfile

@@ -7,4 +7,4 @@ COPY . .
 
 RUN pip install -r requirements.txt
 
-ENTRYPOINT ["python", "main.py"]
+ENTRYPOINT ["python", "/app/main.py"]

main.py

@@ -6,13 +6,17 @@ import requests
 import json
 import datetime
 import smtplib
+import os
+import crcmod
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
 # Arguments parse
 parser = argparse.ArgumentParser(description='Reddit notify on keywords')
-parser.add_argument("--config", type=str, help="name of config file", 
-                    default='config.ini.example')
+parser.add_argument("--config", type=str, help="location of config file", 
+                    default='config.ini')
+parser.add_argument("--cache", type=str, help="location of cache file", 
+                    default='cache.txt')
 args = parser.parse_args()
 
 # Config parse
@@ -23,14 +27,46 @@ http_headers = {
     'User-Agent': 'Python_Reddit_Notif_Dtam/1.0'
 }
 
+# Keep track of sent
+crc32_func = crcmod.predefined.mkPredefinedCrcFun('crc-32')
+cache_file = ""
+sent = []
+
 # Etc
 nl = '\n'
 
 # Config error message
 def config_error(msg):
-    print(f'Error parsing config file: {msg}')
+    print_and_flush(f'Error parsing config file: {msg}')
     sys.exit(1)
 
+# Print and flush stdout
+def print_and_flush(msg):
+    print(msg)
+    sys.stdout.flush()
+
+# Check if message was sent previously
+def sent_previously(url):
+    # Simple hash to keep track of urls
+    encoded = url.encode('utf-8')
+    hashed_url = str(crc32_func(encoded))
+
+    # If not sent previously, add it to the sent list
+    if hashed_url not in sent:
+        sent.append(hashed_url)
+
+        # Prune sent list
+        while len(sent) > 100:
+            sent.pop(0)
+
+        # Write cache to disk
+        with open(cache_file, "w") as file:
+            file.write(', '.join(str(value) for value in sent))
+
+        return False
+    else:
+        return True
+
 # Read config file
 def get_config(filename):
     config = {}
@@ -43,7 +79,7 @@ def get_config(filename):
             raise configparser.Error
         
     except configparser.Error as e:
-        print(f'Error reading config file {e}')
+        print_and_flush(f'Error reading config file {e}')
         sys.exit(1)
 
     # Parse config file
@@ -55,7 +91,7 @@ def get_config(filename):
 
     if config_parser.has_option('app', 'interval') and \
             len(config_parser.get('app', 'interval').strip()) > 0:
-        config['interval'] = config_parser.get('app', 'interval').strip()
+        config['interval'] = int(config_parser.get('app', 'interval'))
     else:
         config_error("Missing 'interval'")
 
@@ -74,7 +110,7 @@ def get_config(filename):
 
     if config_parser.has_option('smtp', 'smtp_port') and \
             len(config_parser.get('smtp', 'smtp_port').strip()) > 0:
-        config['smtp_port'] = config_parser.get('smtp', 'smtp_port').strip()
+        config['smtp_port'] = int(config_parser.get('smtp', 'smtp_port'))
     else:
         config_error("Missing 'smtp_port'")
 
@@ -104,6 +140,26 @@ def get_config(filename):
 
     return config
 
+# Get/Setup cache file
+def setup_cache(filename):
+    global cache_file
+    cache_file = filename
+
+    # If cache exists, read it and update sent
+    if os.path.exists(cache_file):
+        with open(cache_file, "r") as file:
+            content = file.read()
+            global sent
+            sent = [token.strip() for token in content.split(',')]
+
+            # Remove random empty string
+            sent = [item for item in sent if item != ""]
+
+        print_and_flush(f'Cache file found at: {filename}')
+    else:
+        with open(cache_file, "w") as file:
+            pass
+
 # Constant loop to check subreddit
 def check_reddit(config):
     subreddits = [subreddit.strip() for subreddit in config.get('subreddit').split(',')]
@@ -111,6 +167,10 @@ def check_reddit(config):
 
     while(True):
         for subreddit in subreddits:
+            # Debug message
+            current_time = datetime.datetime.now()
+            print_and_flush(f'Starting search at: {current_time.strftime("%Y-%m-%d %H:%M:%S")}')
+            
             resp = requests.get(f'https://www.reddit.com/r/{subreddit}/new.json', headers=http_headers)
 
             if resp.status_code == 200:
@@ -138,24 +198,31 @@ def check_reddit(config):
                             # Send alert
                             send_alert(config, title, text, url, timestamp, keyword)
         
-        time.sleep(config.get('interval'))
+        time.sleep(config.get('interval') * 60)
 
 # Send alert out
 def send_alert(config, title, text, url, timestamp, keyword):
+    # Check if sent previously
+    if sent_previously(url):
+        return
+        
     # Setup
     smtp_from = config.get('smtp_from')
     smtp_to = config.get('smtp_to')
+    time_format = datetime.datetime.fromtimestamp(timestamp)
+
+    # Debug message
+    print_and_flush(f'Found match: {title} at {time_format.strftime("%Y-%m-%d %H:%M:%S")}')
 
     # Setup message
     message = MIMEMultipart()
     message["From"] = smtp_from
     message["To"] = smtp_to
     message["Subject"] = f'Reddit Notify: Found Match ({title})'
-    time_format = datetime.datetime.fromtimestamp(timestamp)
     body = f'Keyword: {keyword}{nl}{nl}{nl}\
         {title}{nl}{nl}{nl}\
         {text}{nl}{nl}{nl}\
-        URL: {url}{nl}\
+        URL: https://www.reddit.com{url}{nl}\
         Time: {time_format.strftime("%Y-%m-%d %H:%M:%S")}'
     
     message.attach(MIMEText(body, "plain"))
@@ -164,18 +231,20 @@ def send_alert(config, title, text, url, timestamp, keyword):
     try:
         with smtplib.SMTP(config.get('smtp_server'),
                         config.get('smtp_port')) as server:
-            server.login(config.get('smtp_username'),
-                        config.get('smtp_password'))
             text = message.as_string()
 
             server.sendmail(smtp_from, smtp_to, text)
     except Exception as e:
-        print(f'SMTP Send Error: {e}')
+        print_and_flush(f'SMTP Send Error: {e}')
 
 if __name__ == '__main__':
 
+    # Setup config
     config = get_config(args.config)
+    print_and_flush(f'Current config file: {config}')
 
-    print(f'Current config file: {config}')
+    # Setup cache
+    setup_cache(args.cache)
 
+    # Main loop, check reddit
     check_reddit(config)

requirements.txt

@@ -1,5 +1,6 @@
 certifi==2024.2.2
 charset-normalizer==3.3.2
+crcmod==1.7
 idna==3.7
 requests==2.32.2
 setuptools==69.0.2