Merge branch 'fw-addr-lists' into next

2 files changed, 65 insertions, 37 deletions

diff --git a/doc/fw-addr-lists.md b/doc/fw-addr-lists.md
index 70ca6e9..ac34c88 100644
--- a/doc/fw-addr-lists.md
+++ b/doc/fw-addr-lists.md
@@ -4,7 +4,7 @@ Download, import and update firewall address-lists
 [![GitHub stars](https://img.shields.io/github/stars/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=red)](https://github.com/eworm-de/routeros-scripts/stargazers)
 [![GitHub forks](https://img.shields.io/github/forks/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=green)](https://github.com/eworm-de/routeros-scripts/network)
 [![GitHub watchers](https://img.shields.io/github/watchers/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=blue)](https://github.com/eworm-de/routeros-scripts/watchers)
-[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.12-yellow?style=flat)](https://mikrotik.com/download/changelogs/)
+[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.13-yellow?style=flat)](https://mikrotik.com/download/changelogs/)
 [![Telegram group @routeros_scripts](https://img.shields.io/badge/Telegram-%40routeros__scripts-%2326A5E4?logo=telegram&style=flat)](https://t.me/routeros_scripts)
 [![donate with PayPal](https://img.shields.io/badge/Like_it%3F-Donate!-orange?logo=githubsponsors&logoColor=orange&style=flat)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A4ZXBD6YS2W8J)
 
@@ -29,6 +29,10 @@ see situation when the lists are not populated.
 To mitigate man-in-the-middle attacks with altered lists the server's
 certificate is checked.
 
+> ⚠️ **Warning**: The script does not limit the size of a list, but keep in
+> mind that huge lists can exhaust your device's resources (RAM and CPU),
+> and may take a long time to process.
+
 Requirements and installation
 -----------------------------
 
diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index 68775b4..007282c 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -3,7 +3,7 @@
 # Copyright (c) 2023-2024 Christian Hesse <mail@eworm.de>
 # https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md
 #
-# requires RouterOS, version=7.12
+# requires RouterOS, version=7.13
 #
 # download, import and update firewall address-lists
 # https://git.eworm.de/cgit/routeros-scripts/about/doc/fw-addr-lists.md
@@ -19,7 +19,8 @@
 
   :global CertificateAvailable;
   :global EitherOr;
-  :global FetchUserAgentStr;
+  :global FetchHuge;
+  :global HumanReadableNum;
   :global LogPrint;
   :global LogPrintOnce;
   :global ScriptLock;
@@ -50,25 +51,25 @@
     :local Failure false;
 
     :foreach List in=$FwList do={
-      :local CheckCertificate "no";
+      :local CheckCertificate false;
       :local Data false;
       :local TimeOut [ $EitherOr [ :totime ($List->"timeout") ] $FwAddrListTimeOut ];
 
       :if ([ :len ($List->"cert") ] > 0) do={
-        :set CheckCertificate "yes-without-crl";
+        :set CheckCertificate true;
         :if ([ $CertificateAvailable ($List->"cert") ] = false) do={
-          $LogPrint warning $ScriptName ("Downloading required certificate failed, trying anyway.");
+          $LogPrint warning $ScriptName ("Downloading required certificate (" . $FwListName . \
+              " / " . $List->"url" . ") failed, trying anyway.");
         }
       }
 
       :for I from=1 to=5 do={
         :if ($Data = false) do={
-          :do {
-            :set Data ([ /tool/fetch check-certificate=$CheckCertificate output=user \
-              http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) ($List->"url") as-value ]->"data");
-          } on-error={
+          :set Data [ $FetchHuge $ScriptName ($List->"url") $CheckCertificate ];
+          :if ($Data = false) do={
             :if ($I < 5) do={
-              $LogPrint debug $ScriptName ("Failed downloading, " . $I . ". try: " . $List->"url");
+              $LogPrint debug $ScriptName ("Failed downloading for list '" . $FwListName . \
+                  "', " . $I . ". try from: " . $List->"url");
               :delay (($I * $I) . "s");
             }
           }
@@ -78,54 +79,67 @@
       :if ($Data = false) do={
         :set Data "";
         :set Failure true;
-        $LogPrint warning $ScriptName ("Failed downloading list from: " . $List->"url");
-      }
-
-      :if ([ :len $Data ] > 63000) do={
-        $LogPrintOnce warning $ScriptName ("The list is huge and may be truncated: " . $List->"url");
+        $LogPrint warning $ScriptName ("Failed downloading for list '" . $FwListName . \
+            "' from: " . $List->"url");
+      } else={
+        $LogPrint debug $ScriptName ("Downloaded " . [ $HumanReadableNum [ :len $Data ] 1024 ] . \
+            "B for list '" . $FwListName . "' from: " . $List->"url");
       }
 
       :while ([ :len $Data ] != 0) do={
         :local Line [ :pick $Data 0 [ :find $Data "\n" ] ];
         :local Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr"));
-        :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$" || \
-             $Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={
-          :set ($IPv4Addresses->$Address) $TimeOut;
-        }
-        :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$" || \
-             $Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={
-          :set ($IPv6Addresses->$Address) $TimeOut;
-        }
+        :do {
+          :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={
+            :set ($IPv4Addresses->$Address) $TimeOut;
+            :error true;
+          }
+          :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$") do={
+            :set ($IPv6Addresses->$Address) $TimeOut;
+            :error true;
+          }
+          :if ($Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={
+            :set ($IPv4Addresses->$Address) $TimeOut;
+            :set ($IPv6Addresses->$Address) $TimeOut;
+            :error true;
+          }
+        } on-error={ }
         :set Data [ :pick $Data ([ :len $Line ] + 1) [ :len $Data ] ];
       }
     }
 
-    :foreach Entry in=[ /ip/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={
+    :foreach Entry in=[ /ip/firewall/address-list/find where \
+        list=$FwListName comment=$ListComment ] do={
       :local Address [ /ip/firewall/address-list/get $Entry address ];
       :if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={
-        $LogPrint debug $ScriptName ("Renewing IPv4 address for " . ($IPv4Addresses->$Address) . ": " . $Address);
+        $LogPrint debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
+            "' with " . ($IPv4Addresses->$Address) . ": " . $Address);
         /ip/firewall/address-list/set $Entry timeout=($IPv4Addresses->$Address);
         :set ($IPv4Addresses->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
         :if ($Failure = false) do={
-          $LogPrint debug $ScriptName ("Removing IPv4 address: " . $Address);
+          $LogPrint debug $ScriptName ("Removing IPv4 address from list '" . $FwListName . \
+              "': " . $Address);
           /ip/firewall/address-list/remove $Entry;
           :set CntRemove ($CntRemove + 1);
         }
       }
     }
 
-    :foreach Entry in=[ /ipv6/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={
+    :foreach Entry in=[ /ipv6/firewall/address-list/find where \
+        list=$FwListName comment=$ListComment ] do={
       :local Address [ /ipv6/firewall/address-list/get $Entry address ];
       :if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={
-        $LogPrint debug $ScriptName ("Renewing IPv6 address for " . ($IPv6Addresses->$Address) . ": " . $Address);
+        $LogPrint debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \
+            "' with " . ($IPv6Addresses->$Address) . ": " . $Address);
         /ipv6/firewall/address-list/set $Entry timeout=($IPv6Addresses->$Address);
         :set ($IPv6Addresses->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
         :if ($Failure = false) do={
-          $LogPrint debug $ScriptName ("Removing: " . $Address);
+          $LogPrint debug $ScriptName ("Removing IPv6 address from list '" . $FwListName . \
+              "': " . $Address);
           /ipv6/firewall/address-list/remove $Entry;
           :set CntRemove ($CntRemove + 1);
         }
@@ -133,27 +147,37 @@
     }
 
     :foreach Address,Timeout in=$IPv4Addresses do={
-      $LogPrint debug $ScriptName ("Adding IPv4 address for " . $Timeout . ": " . $Address);
+      $LogPrint debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \
+          "' with " . $Timeout . ": " . $Address);
       :do {
-        /ip/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout;
+        /ip/firewall/address-list/add list=$FwListName comment=$ListComment \
+            address=$Address timeout=$Timeout;
         :set ($IPv4Addresses->$Address);
         :set CntAdd ($CntAdd + 1);
       } on-error={
-        $LogPrint warning $ScriptName ("Failed to add IPv4 address " . $Address . " to list '" . $FwListName . "'.");
+        $LogPrint warning $ScriptName ("Failed to add IPv4 address to list '" . $FwListName . \
+            "': " . $Address);
       }
     }
 
     :foreach Address,Timeout in=$IPv6Addresses do={
-      $LogPrint debug $ScriptName ("Adding IPv6 address for " . $Timeout . ": " . $Address);
+      $LogPrint debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \
+          "' with " . $Timeout . ": " . $Address);
       :do {
-        /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout;
+        /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \
+            address=$Address timeout=$Timeout;
         :set ($IPv6Addresses->$Address);
         :set CntAdd ($CntAdd + 1);
       } on-error={
-        $LogPrint warning $ScriptName ("Failed to add IPv6 address " . $Address . " to list '" . $FwListName . "'.");
+        $LogPrint warning $ScriptName ("Failed to add IPv6 address to list '" . $FwListName . \
+            "': " . $Address);
       }
     }
 
-    $LogPrint info $ScriptName ("list: " . $FwListName . " -- added: " . $CntAdd . " - renewed: " . $CntRenew . " - removed: " . $CntRemove);
+    $LogPrint info $ScriptName ("list: " . $FwListName . \
+        " (" . [ $HumanReadableNum ($CntAdd + $CntRenew) 1000 ] . ")" . \
+        " -- added: " . [ $HumanReadableNum $CntAdd 1000 ] . \
+        " - renewed: " . [ $HumanReadableNum $CntRenew 1000 ] . \
+        " - removed: " . [ $HumanReadableNum $CntRemove 1000 ]);
   }
 } on-error={ }