Merge branch 'master' into feat/email-templates

Dockerfile

@@ -1,5 +1,5 @@
 
-FROM node:20 as webappbuild
+FROM node:20 AS webappbuild
 
 # Build the webapp
 COPY ./webapp/ /webapp/

README.md

@@ -17,6 +17,19 @@ $ utils/keygen.sh ./src/main/resources/sk ./src/main/resources/pk
 2. Create the Java app configuration:
 Copy the file `src/main/resources/config.sample.json` to `src/main/resources/config.json`.
 
+#### Redis
+For Redis the following environment variables need to be set:
+
+|Name | Description |
+|---|---|
+| REDIS_HOST | Host to reach the redis at |
+| REDIS_PORT | Port to reach the redis at |
+| REDIS_MASTER_NAME | The master name for the Redis Sentinel |
+| REDIS_USERNAME | Username for the Redis user |
+| REDIS_PASSWORD | The password for the Redis user |
+| REDIS_KEY_PREFIX | The prefix to use for all redis keys |
+| STORAGE_TYPE | The type of storage used: if you want to enable Redis, set it to "redis" |
+
 ### Run
 Use docker-compose up combined with your localhost IP address as environment variable to spin up the containers:
 ```bash

build.gradle

@@ -37,6 +37,7 @@ dependencies {
     implementation 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
 
     implementation 'io.jsonwebtoken:jjwt:0.12.5'
+    implementation 'redis.clients:jedis:5.1.5'
     implementation 'com.google.code.gson:gson:2.10.1'
     implementation 'org.apache.commons:commons-lang3:3.14.0'
     implementation 'org.bouncycastle:bcpkix-jdk18on:1.77'

src/main/java/foundation/privacybydesign/email/CleanupBackgroundJob.java

@@ -1,6 +1,7 @@
 package foundation.privacybydesign.email;
 
-import foundation.privacybydesign.email.ratelimit.MemoryRateLimit;
+import foundation.privacybydesign.email.ratelimit.RateLimitUtils;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,7 +28,7 @@ public void contextInitialized(ServletContextEvent event) {
         scheduler.scheduleAtFixedRate(new Runnable() {
             @Override public void run() {
                 try {
-                    MemoryRateLimit.getInstance().periodicCleanup();
+                    RateLimitUtils.getRateLimiter().periodicCleanup();
                 } catch (Exception e) {
                     logger.error("Failed to run periodic cleanup:");
                     e.printStackTrace();

src/main/java/foundation/privacybydesign/email/EmailRestApi.java

@@ -8,8 +8,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import foundation.privacybydesign.email.ratelimit.MemoryRateLimit;
 import foundation.privacybydesign.email.ratelimit.RateLimit;
+import foundation.privacybydesign.email.ratelimit.RateLimitUtils;
 import jakarta.mail.internet.AddressException;
 import jakarta.ws.rs.*;
 import jakarta.ws.rs.core.MediaType;
@@ -28,7 +28,7 @@
 @Path("")
 public class EmailRestApi {
     private static Logger logger = LoggerFactory.getLogger(EmailRestApi.class);
-    private static RateLimit rateLimiter = MemoryRateLimit.getInstance();
+    private static RateLimit rateLimiter = RateLimitUtils.getRateLimiter();
 
     private static final String ERR_ADDRESS_MALFORMED = "error:email-address-malformed";
     private static final String ERR_INVALID_TOKEN = "error:invalid-token";
@@ -84,6 +84,7 @@ public Response sendEmail(@FormParam("email") String email,
                     client.getEmail(lang),
                     client.getReplyToEmail(),
                     true,
+                    url,
                     url
 
             );

src/main/java/foundation/privacybydesign/email/ratelimit/MemoryRateLimit.java

@@ -9,7 +9,7 @@
  *
  * How it works:
  * How much budget a user has, is expressed in a timestamp. The timestamp is
- * initially some period in the past, but with every usage (countEmail) 
+ * initially some period in the past, but with every usage (countEmail)
  * this timestamp is incremented. For e-mail addresses this
  * amount is exponential.
  *
@@ -89,34 +89,26 @@ protected synchronized void countEmail(String email, long now) {
         if (nextTry > now) {
             throw new IllegalStateException("counting rate limit while over the limit");
         }
-        limit.tries = Math.min(limit.tries+1, 6); // add 1, max at 6
+        limit.tries = Math.min(limit.tries + 1, 6); // add 1, max at 6
         // If the last usage was e.g. ≥2 days ago, we should allow them 2
         // extra tries this day.
-        long lastTryDaysAgo = (now-limit.timestamp)/DAY;
+        long lastTryDaysAgo = (now - limit.timestamp) / DAY;
         long bonusTries = limit.tries - lastTryDaysAgo;
         if (bonusTries >= 1) {
-            limit.tries = (int)bonusTries;
+            limit.tries = (int) bonusTries;
         }
         limit.timestamp = now;
     }
 
+    @Override
     public void periodicCleanup() {
         long now = System.currentTimeMillis();
-        // Use enhanced for loop, because an iterator makes sure concurrency issues cannot occur.
+        // Use enhanced for loop, because an iterator makes sure concurrency issues
+        // cannot occur.
         for (Map.Entry<String, Limit> entry : emailLimits.entrySet()) {
-            if (entry.getValue().timestamp < now - 2*DAY) {
+            if (entry.getValue().timestamp < now - 2 * DAY) {
                 emailLimits.remove(entry.getKey());
             }
         }
     }
 }
-
-class Limit {
-    long timestamp;
-    int tries;
-
-    Limit(long now) {
-        tries = 0;
-        timestamp = now;
-    }
-}

src/main/java/foundation/privacybydesign/email/ratelimit/RateLimit.java

@@ -31,4 +31,20 @@ public long rateLimited(String email) {
 
     protected abstract long nextTryEmail(String email, long now);
     protected abstract void countEmail(String email, long now);
+    public abstract void periodicCleanup();
+}
+
+class Limit {
+    long timestamp;
+    int tries;
+
+    Limit(long timestamp, int tries) {
+        this.timestamp = timestamp;
+        this.tries = tries;
+    }
+
+    Limit(long now) {
+        tries = 0;
+        timestamp = now;
+    }
 }

src/main/java/foundation/privacybydesign/email/ratelimit/RateLimitUtils.java

@@ -0,0 +1,12 @@
+package foundation.privacybydesign.email.ratelimit;
+
+public class RateLimitUtils {
+    /// Returns the active rate limiter based on the configuration
+    public static RateLimit getRateLimiter() {
+        final String storageType = System.getenv("STORAGE_TYPE");
+        if (storageType.equals("redis")) {
+            return RedisRateLimit.getInstance();
+        }
+        return MemoryRateLimit.getInstance();
+    }
+}

src/main/java/foundation/privacybydesign/email/ratelimit/RedisRateLimit.java

@@ -0,0 +1,196 @@
+
+package foundation.privacybydesign.email.ratelimit;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import foundation.privacybydesign.email.redis.Redis;
+import redis.clients.jedis.*;
+import redis.clients.jedis.params.ScanParams;
+import redis.clients.jedis.resps.ScanResult;
+
+class RedisRateLimit extends RateLimit {
+    private static final long SECOND = 1000; // 1000ms = 1s
+    private static final long MINUTE = SECOND * 60;
+    private static final long HOUR = MINUTE * 60;
+    private static final long DAY = HOUR * 24;
+
+    private static Logger LOG = LoggerFactory.getLogger(RedisRateLimit.class);
+    final private static String NAMESPACE = "rate-limit";
+    final private static String TIMESTAMP_FIELD_NAME = "timestamp";
+    final private static String TRIES_FIELD_NAME = "tries";
+
+    private static RedisRateLimit instance;
+
+    private JedisSentinelPool pool;
+
+    RedisRateLimit() {
+        pool = Redis.createSentinelPoolFromEnv();
+    }
+
+    static RateLimit getInstance() {
+        if (instance == null) {
+            instance = new RedisRateLimit();
+        }
+        return instance;
+    }
+
+    @Override
+    protected long nextTryEmail(String email, long now) {
+        // Rate limiter durations (sort-of logarithmic):
+        // 1 10 second
+        // 2 5 minute
+        // 3 1 hour
+        // 4 24 hour
+        // 5+ 1 per day
+        // Keep log 5 days for proper limiting.
+
+        final String key = Redis.createKey(NAMESPACE, email);
+
+        Limit limit;
+
+        try (var jedis = pool.getResource()) {
+            limit = limitFromRedis(jedis, key);
+            if (limit == null) {
+                limit = new Limit(now);
+                limitToRedis(jedis, key, limit);
+            }
+        }
+        //
+        // Limit limit = phoneLimits.get(phone);
+        // if (limit == null) {
+        // limit = new Limit(now);
+        // phoneLimits.put(phone, limit);
+        // }
+        long nextTry; // timestamp when the next request is allowed
+        switch (limit.tries) {
+            case 0: // try 1: always succeeds
+                nextTry = limit.timestamp;
+                break;
+            case 1: // try 2: allowed after 10 seconds
+                nextTry = limit.timestamp + 10 * SECOND;
+                break;
+            case 2: // try 3: allowed after 5 minutes
+                nextTry = limit.timestamp + 5 * MINUTE;
+                break;
+            case 3: // try 4: allowed after 3 hours
+                nextTry = limit.timestamp + 3 * HOUR;
+                break;
+            case 4: // try 5: allowed after 24 hours
+                nextTry = limit.timestamp + 24 * HOUR;
+                break;
+            default:
+                throw new IllegalStateException("invalid tries count");
+        }
+        return nextTry;
+    }
+
+    @Override
+    protected void countEmail(String email, long now) {
+        long nextTry = nextTryEmail(email, now);
+        final String key = Redis.createKey(NAMESPACE, email);
+
+        try (var jedis = pool.getResource()) {
+            Limit limit = limitFromRedis(jedis, key);
+            if (limit == null) {
+                throw new IllegalStateException("limit is null where that should be impossible");
+            }
+            if (nextTry > now) {
+                throw new IllegalStateException("counting rate limit while over the limit");
+            }
+            limit.tries = Math.min(limit.tries + 1, 6); // add 1, max at 6
+            // If the last usage was e.g. ≥2 days ago, we should allow them 2
+            // extra tries this day.
+            long lastTryDaysAgo = (now - limit.timestamp) / DAY;
+            long bonusTries = limit.tries - lastTryDaysAgo;
+            if (bonusTries >= 1) {
+                limit.tries = (int) bonusTries;
+            }
+            limit.timestamp = now;
+            limitToRedis(jedis, key, limit);
+        }
+    }
+
+    // TODO: This is not the idiomatic way to delete expired items in Redis,
+    // use the built in `expire` command instead
+    @Override
+    public void periodicCleanup() {
+        long now = System.currentTimeMillis();
+
+        final String pattern = Redis.createNamespace(NAMESPACE) + "*";
+        ScanParams scanParams = new ScanParams().match(pattern);
+        String cursor = "0";
+
+        try (var jedis = pool.getResource()) {
+            do {
+                ScanResult<String> scanResult = jedis.scan(cursor, scanParams);
+                List<String> keys = scanResult.getResult();
+                cursor = scanResult.getCursor();
+
+                for (String key : keys) {
+                    Limit limit = limitFromRedis(jedis, key);
+                    if (limit != null && limit.timestamp < now - 5 * DAY) {
+                        jedis.del(key);
+                    }
+                }
+            } while (!cursor.equals("0")); // continue until the cursor wraps around
+        }
+    }
+
+    void limitToRedis(Jedis jedis, String key, Limit limit) {
+        final String ts = Long.toString(limit.timestamp);
+        final String tries = Long.toString(limit.tries);
+
+        jedis.watch(key);
+        Transaction transaction = jedis.multi();
+
+        transaction.hset(key, TIMESTAMP_FIELD_NAME, ts);
+        transaction.hset(key, TRIES_FIELD_NAME, tries);
+
+        final List<Object> results = transaction.exec();
+
+        if (results == null) {
+            LOG.error("failed to set limit to Redis: exec() returned null");
+            return;
+        }
+
+        for (var r : results) {
+            if (r instanceof Exception) {
+                LOG.error("failed to set limit to Redis: " + ((Exception) r).getMessage());
+            }
+        }
+    }
+
+    Limit limitFromRedis(Jedis jedis, String key) {
+        try {
+            jedis.watch(key);
+            Transaction transaction = jedis.multi();
+
+            final Response<String> timestampRes = transaction.hget(key, TIMESTAMP_FIELD_NAME);
+            final Response<String> triesRes = transaction.hget(key, TRIES_FIELD_NAME);
+
+            final List<Object> results = transaction.exec();
+
+            if (results == null) {
+                LOG.error("failed to get limit from Redis: exec() returned null");
+                return null;
+            }
+
+            for (var r : results) {
+                if (r instanceof Exception) {
+                    LOG.error("failed to get limit from Redis: " + ((Exception) r).getMessage());
+                    return null;
+                }
+            }
+
+            final long ts = Long.parseLong(timestampRes.get());
+            final int tries = Integer.parseInt(triesRes.get());
+            return new Limit(ts, tries);
+        } catch (NumberFormatException e) {
+            LOG.error("failed to parse int: " + e.getMessage());
+            return null;
+        }
+    }
+}