@@ -1932,7 +1932,7 @@ int main(int argc, char **argv)

             alarm(0);

 #endif

-            if (ret > 1) {

+            if (ret > FC_UPTODATE) {

                 if ((opt = optget(opts, "OnErrorExecute"))->enabled)

                     arg = opt->strarg;

@@ -1940,6 +1940,14 @@ int main(int argc, char **argv)

                     execute("OnErrorExecute", arg, optget(opts, "daemon")->enabled);

                 arg = NULL;

+

+                if (FC_EFORBIDDEN == ret) {

+                    /* We're being actively blocked, which is a fatal error. Exit. */

+                    logg("^Freshclam was forbidden from downloading a database.\n");

+                    logg("^This is fatal. Retrying later won't help. Exiting now.\n");

+                    status = ret;

+                    goto done;

+                }

             }

             logg("#--------------------------------------\n");

@@ -250,6 +250,17 @@ fc_error_t fc_initialize(fc_config *fcConfig)

     g_bCompressLocalDatabase = fcConfig->bCompressLocalDatabase;

+    /* Load or create mirrors.dat */

+    if (FC_SUCCESS != load_mirrors_dat()) {

+        logg("*Failed to load mirrors.dat; will create a new mirrors.dat\n");

+

+        if (FC_SUCCESS != new_mirrors_dat()) {

+            logg("^Failed to create a new mirrors.dat!\n");

+            status = FC_EINIT;

+            goto done;

+        }

+    }

+

     status = FC_SUCCESS;

 done:

@@ -667,6 +678,10 @@ fc_error_t fc_update_database(

                     break;

                 }

                 case FC_ERETRYLATER: {

+                    char retry_after_string[26];

+                    struct tm *tm_info;

+                    tm_info = localtime(&g_mirrorsDat->retry_after);

+                    strftime(retry_after_string, 26, "%Y-%m-%d %H:%M:%S", tm_info);

                     logg("^FreshClam received error code 429 from the ClamAV Content Delivery Network (CDN).\n");

                     logg("This means that you have been rate limited by the CDN.\n");

                     logg(" 1. Run FreshClam no more than once an hour to check for updates.\n");

@@ -677,6 +692,7 @@ fc_error_t fc_update_database(

                     logg("    CDN and your own network.\n");

                     logg(" 3. Please do not open a ticket asking for an exemption from the rate limit,\n");

                     logg("    it will not be granted.\n");

+                    logg("^You are on cool-down until after: %s\n", retry_after_string);

                     goto success;

                     break;

                 }

@@ -726,6 +742,33 @@ fc_error_t fc_update_databases(

     *nUpdated = 0;

+    if (g_mirrorsDat->retry_after > 0) {

+        if (g_mirrorsDat->retry_after > time(NULL)) {

+            /* We're on cool-down, try again later. */

+            char retry_after_string[26];

+            struct tm *tm_info;

+            tm_info = localtime(&g_mirrorsDat->retry_after);

+            strftime(retry_after_string, 26, "%Y-%m-%d %H:%M:%S", tm_info);

+            logg("^FreshClam previously received error code 429 from the ClamAV Content Delivery Network (CDN).\n");

+            logg("This means that you have been rate limited by the CDN.\n");

+            logg(" 1. Run FreshClam no more than once an hour to check for updates.\n");

+            logg("    Freshclam should check DNS first to see if an update is needed.\n");

+            logg(" 2. If you have more than 10 hosts on your network attempting to download,\n");

+            logg("    it is recommended that you set up a private mirror on your network using\n");

+            logg("    cvdupdate (https://pypi.org/project/cvdupdate/) to save bandwidth on the\n");

+            logg("    CDN and your own network.\n");

+            logg(" 3. Please do not open a ticket asking for an exemption from the rate limit,\n");

+            logg("    it will not be granted.\n");

+            logg("^You are still on cool-down until after: %s\n", retry_after_string);

+            status = FC_SUCCESS;

+            goto done;

+        } else {

+            g_mirrorsDat->retry_after = 0;

+            logg("^Cool-down expired, ok to try again.\n");

+            save_mirrors_dat();

+        }

+    }

+

     for (i = 0; i < nDatabases; i++) {

         if (FC_SUCCESS != (ret = fc_update_database(

                                databaseList[i],

@@ -814,6 +857,41 @@ fc_error_t fc_download_url_database(

                 }

                 break;

             }

+            case FC_EFORBIDDEN: {

+                logg("^FreshClam received error code 403 from the ClamAV Content Delivery Network (CDN).\n");

+                logg("This could mean several things:\n");

+                logg(" 1. You are running an out of date version of ClamAV / FreshClam.\n");

+                logg("    Ensure you are the most updated version by visiting https://www.clamav.net/downloads\n");

+                logg(" 2. Your network is explicitly denied by the FreshClam CDN.\n");

+                logg("    In order to rectify this please check that you are:\n");

+                logg("   a. Running an up to date version of FreshClam\n");

+                logg("   b. Running FreshClam no more than once an hour\n");

+                logg("   c. If you have checked (a) and (b), please open a ticket at\n");

+                logg("      https://bugzilla.clamav.net under the “Mirrors” component\n");

+                logg("      and we will investigate why your network is blocked.\n");

+                status = ret;

+                goto done;

+                break;

+            }

+            case FC_ERETRYLATER: {

+                char retry_after_string[26];

+                struct tm *tm_info;

+                tm_info = localtime(&g_mirrorsDat->retry_after);

+                strftime(retry_after_string, 26, "%Y-%m-%d %H:%M:%S", tm_info);

+                logg("^FreshClam received error code 429 from the ClamAV Content Delivery Network (CDN).\n");

+                logg("This means that you have been rate limited by the CDN.\n");

+                logg(" 1. Run FreshClam no more than once an hour to check for updates.\n");

+                logg("    Freshclam should check DNS first to see if an update is needed.\n");

+                logg(" 2. If you have more than 10 hosts on your network attempting to download,\n");

+                logg("    it is recommended that you set up a private mirror on your network using\n");

+                logg("    cvdupdate (https://pypi.org/project/cvdupdate/) to save bandwidth on the\n");

+                logg("    CDN and your own network.\n");

+                logg(" 3. Please do not open a ticket asking for an exemption from the rate limit,\n");

+                logg("    it will not be granted.\n");

+                logg("^You are on cool-down until after: %s\n", retry_after_string);

+                goto success;

+                break;

+            }

             default: {

                 logg("Unexpected error when attempting to update from custom database URL: %s\n", urlDatabase);

                 status = ret;

@@ -66,6 +66,7 @@

 #include <math.h>

 #include <curl/curl.h>

+#include <openssl/rand.h>

 #include "target.h"

@@ -115,6 +116,242 @@ uint32_t g_requestTimeout = 0;

 uint32_t g_bCompressLocalDatabase = 0;

+mirrors_dat_v1_t *g_mirrorsDat = NULL;

+

+/** @brief Generate a Version 4 UUID according to RFC-4122

+ *

+ * Uses the openssl RAND_bytes function to generate a Version 4 UUID.

+ *

+ * Copyright 2021 Karthik Velakur

+ * License: MIT

+ * From: https://gist.github.com/kvelakur/9069c9896577c3040030

+ *

+ * @param buffer A buffer that is SIZEOF_UUID_V4

+ * @retval 1 on success, 0 otherwise.

+ */

+static int uuid_v4_gen(char *buffer)

+{

+    union {

+        struct

+        {

+            uint32_t time_low;

+            uint16_t time_mid;

+            uint16_t time_hi_and_version;

+            uint8_t clk_seq_hi_res;

+            uint8_t clk_seq_low;

+            uint8_t node[6];

+        };

+        uint8_t __rnd[16];

+    } uuid;

+

+    int rc = RAND_bytes(uuid.__rnd, sizeof(uuid));

+

+    // Refer Section 4.2 of RFC-4122

+    // https://tools.ietf.org/html/rfc4122#section-4.2

+    uuid.clk_seq_hi_res      = (uint8_t)((uuid.clk_seq_hi_res & 0x3F) | 0x80);

+    uuid.time_hi_and_version = (uint16_t)((uuid.time_hi_and_version & 0x0FFF) | 0x4000);

+

+    snprintf(buffer, SIZEOF_UUID_V4, "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x",

+             uuid.time_low, uuid.time_mid, uuid.time_hi_and_version,

+             uuid.clk_seq_hi_res, uuid.clk_seq_low,

+             uuid.node[0], uuid.node[1], uuid.node[2],

+             uuid.node[3], uuid.node[4], uuid.node[5]);

+    buffer[SIZEOF_UUID_V4 - 1] = 0;

+

+    return rc;

+}

+

+fc_error_t load_mirrors_dat(void)

+{

+    fc_error_t status      = FC_EINIT;

+    int handle             = -1;

+    ssize_t bread          = 0;

+    mirrors_dat_v1_t *mdat = NULL;

+    uint32_t version       = 0;

+    char magic[13]         = {0};

+

+    /* Change directory to database directory */

+    if (chdir(g_databaseDirectory)) {

+        logg("!Can't change dir to %s\n", g_databaseDirectory);

+        status = FC_EDIRECTORY;

+        goto done;

+    }

+    logg("*Current working dir is %s\n", g_databaseDirectory);

+

+    if (-1 == (handle = safe_open("mirrors.dat", O_RDONLY | O_BINARY))) {

+        char currdir[PATH_MAX];

+

+        if (getcwd(currdir, sizeof(currdir)))

+            logg("*Can't open mirrors.dat in %s\n", currdir);

+        else

+            logg("*Can't open mirrors.dat in the current directory\n");

+

+        logg("*It probably doesn't exist yet. That's ok.\n");

+        status = FC_EFILE;

+        goto done;

+    }

+

+    if (strlen(MIRRORS_DAT_MAGIC) != (bread = read(handle, &magic, strlen(MIRRORS_DAT_MAGIC)))) {

+        char error_message[260];

+        cli_strerror(errno, error_message, 260);

+        logg("!Can't read version from mirrors.dat. Bytes read: %zi, error: %s\n", bread, error_message);

+        goto done;

+    }

+    if (0 != strncmp(magic, MIRRORS_DAT_MAGIC, strlen(MIRRORS_DAT_MAGIC))) {

+        logg("*Magic bytes for mirrors.dat did not match expectations.\n");

+        goto done;

+    }

+

+    if (sizeof(uint32_t) != (bread = read(handle, &version, sizeof(uint32_t)))) {

+        char error_message[260];

+        cli_strerror(errno, error_message, 260);

+        logg("!Can't read version from mirrors.dat. Bytes read: %zi, error: %s\n", bread, error_message);

+        goto done;

+    }

+

+    switch (version) {

+        case 1: {

+            /* Verify that file size is as expected. */

+            off_t file_size = lseek(handle, 0L, SEEK_END);

+

+            if (strlen(MIRRORS_DAT_MAGIC) + sizeof(mirrors_dat_v1_t) != (size_t)file_size) {

+                logg("*mirrors.dat is bigger than expected: %zu != %ld\n", sizeof(mirrors_dat_v1_t), file_size);

+                goto done;

+            }

+

+            /* Rewind to just after the magic bytes and read data struct */

+            lseek(handle, strlen(MIRRORS_DAT_MAGIC), SEEK_SET);

+

+            mdat = malloc(sizeof(mirrors_dat_v1_t));

+            if (NULL == mdat) {

+                logg("!Failed to allocate memory for mirrors.dat\n");

+                status = FC_EMEM;

+                goto done;

+            }

+

+            if (sizeof(mirrors_dat_v1_t) != (bread = read(handle, mdat, sizeof(mirrors_dat_v1_t)))) {

+                char error_message[260];

+                cli_strerror(errno, error_message, 260);

+                logg("!Can't read from mirrors.dat. Bytes read: %zi, error: %s\n", bread, error_message);

+                goto done;

+            }

+

+            /* Got it.*/

+            close(handle);

+

+            /* This is the latest version.

+               If we change the format in the future, we may wish to create a new

+               mirrors dat struct, import the relevant bits to the new format,

+               and then save (overwrite) mirrors.dat with the new data. */

+            g_mirrorsDat = mdat;

+            break;

+        }

+        default: {

+            logg("*mirrors.dat version is different than expected: %u != %u\n", 1, g_mirrorsDat->version);

+            goto done;

+        }

+    }

+

+    logg("*Loaded mirrors.dat:\n");

+    logg("*  version:    %d\n", g_mirrorsDat->version);

+    logg("*  uuid:       %s\n", g_mirrorsDat->uuid);

+    if (g_mirrorsDat->retry_after > 0) {

+        char retry_after_string[26];

+        struct tm *tm_info = localtime(&g_mirrorsDat->retry_after);

+        strftime(retry_after_string, 26, "%Y-%m-%d %H:%M:%S", tm_info);

+        logg("*  retry-after: %s\n", retry_after_string);

+    }

+

+    status = FC_SUCCESS;

+

+done:

+    if (-1 != handle) {

+        close(handle);

+    }

+    if (FC_SUCCESS != status) {

+        free(mdat);

+    }

+

+    return status;

+}

+

+fc_error_t save_mirrors_dat(void)

+{

+    fc_error_t status = FC_EINIT;

+    int handle        = -1;

+

+    if (-1 == (handle = safe_open("mirrors.dat", O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644))) {

+        char currdir[PATH_MAX];

+

+        if (getcwd(currdir, sizeof(currdir)))

+            logg("!Can't create mirrors.dat in %s\n", currdir);

+        else

+            logg("!Can't create mirrors.dat in the current directory\n");

+

+        logg("Hint: The database directory must be writable for UID %d or GID %d\n", getuid(), getgid());

+        status = FC_EDBDIRACCESS;

+        goto done;

+    }

+    if (-1 == write(handle, MIRRORS_DAT_MAGIC, strlen(MIRRORS_DAT_MAGIC))) {

+        logg("!Can't write to mirrors.dat\n");

+    }

+    if (-1 == write(handle, g_mirrorsDat, sizeof(mirrors_dat_v1_t))) {

+        logg("!Can't write to mirrors.dat\n");

+    }

+

+    logg("*Saved mirrors.dat\n");

+

+    status = FC_SUCCESS;

+done:

+    if (-1 != handle) {

+        close(handle);

+    }

+

+    return status;

+}

+

+fc_error_t new_mirrors_dat(void)

+{

+    fc_error_t status = FC_EINIT;

+

+    mirrors_dat_v1_t *mdat = malloc(sizeof(mirrors_dat_v1_t));

+    if (NULL == mdat) {

+        logg("!Failed to allocate memory for mirrors.dat\n");

+        status = FC_EMEM;

+        goto done;

+    }

+

+    mdat->version     = 1;

+    mdat->retry_after = 0;

+    if (0 == uuid_v4_gen(mdat->uuid)) {

+        /* Failed to create UUID */

+        status = FC_EINIT;

+        logg("!Failed to create random UUID!\n");

+        goto done;

+    }

+

+    g_mirrorsDat = mdat;

+

+    logg("*Creating new mirrors.dat\n");

+

+    if (FC_SUCCESS != save_mirrors_dat()) {

+        logg("!Failed to save mirrors.dat!\n");

+        status = FC_EFILE;

+        goto done;

+    }

+

+    status = FC_SUCCESS;

+

+done:

+    if (FC_SUCCESS != status) {

+        if (NULL != mdat) {

+            free(mdat);

+        }

+        g_mirrorsDat = NULL;

+    }

+    return status;

+}

+

 /**

  * @brief Get DNS text record field # for official databases.

  *

@@ -328,12 +565,19 @@ static fc_error_t create_curl_handle(

         goto done;

     }

-    if (g_userAgent)

+    if (g_userAgent) {

         strncpy(userAgent, g_userAgent, sizeof(userAgent));

-    else

+    } else {

+        /*

+         * Use a randomly generated UUID in the User-Agent

+         * We'll try to load it from a file in the database directory.

+         * If none exists, we'll create a new one and save it to said file.

+         */

         snprintf(userAgent, sizeof(userAgent),

-                 PACKAGE "/%s (OS: " TARGET_OS_TYPE ", ARCH: " TARGET_ARCH_TYPE ", CPU: " TARGET_CPU_TYPE ")",

-                 get_version());

+                 PACKAGE "/%s (OS: " TARGET_OS_TYPE ", ARCH: " TARGET_ARCH_TYPE ", CPU: " TARGET_CPU_TYPE ", UUID: %s)",

+                 get_version(),

+                 g_mirrorsDat->uuid);

+    }

     userAgent[sizeof(userAgent) - 1] = 0;

     if (mprintf_verbose) {

@@ -730,6 +974,27 @@ static fc_error_t remote_cvdhead(

             status = FC_UPTODATE;

             goto done;

         }

+        case 403: {

+            status = FC_EFORBIDDEN;

+            break;

+        }

+        case 429: {

+            status = FC_ERETRYLATER;

+            curl_off_t retry_after;

+

+            /* Find out how long we should wait before allowing a retry. */

+            curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);

+            if (retry_after > 0) {

+                /* The response gave us a Retry-After date. Use that. */

+                g_mirrorsDat->retry_after = time(NULL) + (time_t)retry_after;

+            } else {

+                /* Try again in no less than 4 hours if the response didn't specify. */

+                g_mirrorsDat->retry_after = time(NULL) + 60 * 60 * 4;

+            }

+            (void)save_mirrors_dat();

+

+            break;

+        }

         case 404: {

             if (g_proxyServer)

                 logg("^remote_cvdhead: file not found: %s (Proxy: %s:%u)\n", url, g_proxyServer, g_proxyPort);

@@ -1004,6 +1269,19 @@ static fc_error_t downloadFile(

         }

         case 429: {

             status = FC_ERETRYLATER;

+            curl_off_t retry_after;

+

+            /* Find out how long we should wait before allowing a retry. */

+            curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);

+            if (retry_after > 0) {

+                /* The response gave us a Retry-After date. Use that. */

+                g_mirrorsDat->retry_after = time(NULL) + (time_t)retry_after;

+            } else {

+                /* Try again in no less than 4 hours if the response didn't specify. */

+                g_mirrorsDat->retry_after = time(NULL) + 60 * 60 * 4;

+            }

+            (void)save_mirrors_dat();

+

             break;

         }

         case 404: {

@@ -1792,7 +2070,7 @@ static fc_error_t check_for_new_database_version(

                      database, localver, remotever);

                 break;

             }

-            // else: Fall-through to Up-to-date case.

+            /* fall-through */

         }

         case FC_UPTODATE: {

             if (NULL == local_database) {

@@ -1812,6 +2090,12 @@ static fc_error_t check_for_new_database_version(

             remotever = localver;

             break;

         }

+        case FC_EFORBIDDEN: {

+            /* We tried to look up the version using HTTP and were actively blocked. */

+            logg("!check_for_new_database_version: Blocked from using server %s.\n", server);

+            status = FC_EFORBIDDEN;

+            goto done;

+        }

         default: {

             logg("!check_for_new_database_version: Failed to find %s database using server %s.\n", database, server);

             status = FC_EFAILEDGET;

@@ -32,6 +32,14 @@

 #define DNS_EXTRADBINFO_RECORDTIME      1

 // clang-format on

+#define SIZEOF_UUID_V4 37                 /** For uuid_v4_gen(), includes NULL byte */

+#define MIRRORS_DAT_MAGIC "FreshClamData" /** Magic bytes for mirrors.dat found before mirrors_dat_v1_t */

+typedef struct _mirrors_dat_v1 {

+    uint32_t version;          /** version of this dat format */

+    char uuid[SIZEOF_UUID_V4]; /** uuid to be used in user-agent */

+    time_t retry_after;        /** retry date. If > 0, don't update until after this date */

+} mirrors_dat_v1_t;

+

 /* ----------------------------------------------------------------------------

  * Internal libfreshclam globals

  */

@@ -55,6 +63,12 @@ extern uint32_t g_requestTimeout;

 extern uint32_t g_bCompressLocalDatabase;

+extern mirrors_dat_v1_t *g_mirrorsDat;

+

+fc_error_t load_mirrors_dat(void);

+fc_error_t save_mirrors_dat(void);

+fc_error_t new_mirrors_dat(void);

+

 fc_error_t updatedb(

     const char *database,

     const char *dnsUpdateInfo,