From c3809f240d292e80c936e104e8aa9414b3dd3367 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Wed, 27 Mar 2024 23:39:55 +0100 Subject: fw-addr-lists: use $FetchHuge --- doc/fw-addr-lists.md | 6 +++++- fw-addr-lists.rsc | 18 ++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/doc/fw-addr-lists.md b/doc/fw-addr-lists.md index 70ca6e9..ac34c88 100644 --- a/doc/fw-addr-lists.md +++ b/doc/fw-addr-lists.md @@ -4,7 +4,7 @@ Download, import and update firewall address-lists [![GitHub stars](https://img.shields.io/github/stars/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=red)](https://github.com/eworm-de/routeros-scripts/stargazers) [![GitHub forks](https://img.shields.io/github/forks/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=green)](https://github.com/eworm-de/routeros-scripts/network) [![GitHub watchers](https://img.shields.io/github/watchers/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=blue)](https://github.com/eworm-de/routeros-scripts/watchers) -[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.12-yellow?style=flat)](https://mikrotik.com/download/changelogs/) +[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.13-yellow?style=flat)](https://mikrotik.com/download/changelogs/) [![Telegram group @routeros_scripts](https://img.shields.io/badge/Telegram-%40routeros__scripts-%2326A5E4?logo=telegram&style=flat)](https://t.me/routeros_scripts) [![donate with PayPal](https://img.shields.io/badge/Like_it%3F-Donate!-orange?logo=githubsponsors&logoColor=orange&style=flat)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A4ZXBD6YS2W8J) @@ -29,6 +29,10 @@ see situation when the lists are not populated. To mitigate man-in-the-middle attacks with altered lists the server's certificate is checked. +> ⚠️ **Warning**: The script does not limit the size of a list, but keep in +> mind that huge lists can exhaust your device's resources (RAM and CPU), +> and may take a long time to process. + Requirements and installation ----------------------------- diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index 68775b4..66f8581 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -3,7 +3,7 @@ # Copyright (c) 2023-2024 Christian Hesse # https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md # -# requires RouterOS, version=7.12 +# requires RouterOS, version=7.13 # # download, import and update firewall address-lists # https://git.eworm.de/cgit/routeros-scripts/about/doc/fw-addr-lists.md @@ -19,7 +19,7 @@ :global CertificateAvailable; :global EitherOr; - :global FetchUserAgentStr; + :global FetchHuge; :global LogPrint; :global LogPrintOnce; :global ScriptLock; @@ -50,12 +50,12 @@ :local Failure false; :foreach List in=$FwList do={ - :local CheckCertificate "no"; + :local CheckCertificate false; :local Data false; :local TimeOut [ $EitherOr [ :totime ($List->"timeout") ] $FwAddrListTimeOut ]; :if ([ :len ($List->"cert") ] > 0) do={ - :set CheckCertificate "yes-without-crl"; + :set CheckCertificate true; :if ([ $CertificateAvailable ($List->"cert") ] = false) do={ $LogPrint warning $ScriptName ("Downloading required certificate failed, trying anyway."); } @@ -63,10 +63,8 @@ :for I from=1 to=5 do={ :if ($Data = false) do={ - :do { - :set Data ([ /tool/fetch check-certificate=$CheckCertificate output=user \ - http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) ($List->"url") as-value ]->"data"); - } on-error={ + :set Data [ $FetchHuge $ScriptName ($List->"url") $CheckCertificate ]; + :if ($Data = false) do={ :if ($I < 5) do={ $LogPrint debug $ScriptName ("Failed downloading, " . $I . ". try: " . $List->"url"); :delay (($I * $I) . "s"); @@ -81,10 +79,6 @@ $LogPrint warning $ScriptName ("Failed downloading list from: " . $List->"url"); } - :if ([ :len $Data ] > 63000) do={ - $LogPrintOnce warning $ScriptName ("The list is huge and may be truncated: " . $List->"url"); - } - :while ([ :len $Data ] != 0) do={ :local Line [ :pick $Data 0 [ :find $Data "\n" ] ]; :local Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr")); -- cgit v1.2.3-54-g00ecf From 4b6d0c02f134f4c6535d4df44016dc9ae2ae5db2 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Fri, 5 Apr 2024 00:20:33 +0200 Subject: fw-addr-lists: try with less regexp matches --- fw-addr-lists.rsc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index 66f8581..7e7ee38 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -82,14 +82,21 @@ :while ([ :len $Data ] != 0) do={ :local Line [ :pick $Data 0 [ :find $Data "\n" ] ]; :local Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr")); - :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$" || \ - $Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={ - :set ($IPv4Addresses->$Address) $TimeOut; - } - :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$" || \ - $Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={ - :set ($IPv6Addresses->$Address) $TimeOut; - } + :do { + :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={ + :set ($IPv4Addresses->$Address) $TimeOut; + :error true; + } + :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$") do={ + :set ($IPv6Addresses->$Address) $TimeOut; + :error true; + } + :if ($Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={ + :set ($IPv4Addresses->$Address) $TimeOut; + :set ($IPv6Addresses->$Address) $TimeOut; + :error true; + } + } on-error={ } :set Data [ :pick $Data ([ :len $Line ] + 1) [ :len $Data ] ]; } } -- cgit v1.2.3-54-g00ecf From af6556bdba9d68174b45561215ea3be2e098e349 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Thu, 4 Apr 2024 19:37:38 +0200 Subject: fw-addr-lists: add debug message on successful download --- fw-addr-lists.rsc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index 7e7ee38..5cdd379 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -20,6 +20,7 @@ :global CertificateAvailable; :global EitherOr; :global FetchHuge; + :global HumanReadableNum; :global LogPrint; :global LogPrintOnce; :global ScriptLock; @@ -77,6 +78,8 @@ :set Data ""; :set Failure true; $LogPrint warning $ScriptName ("Failed downloading list from: " . $List->"url"); + } else={ + $LogPrint debug $ScriptName ("Downloaded " . [ $HumanReadableNum [ :len $Data ] 1024 ] . "B from: " . $List->"url"); } :while ([ :len $Data ] != 0) do={ -- cgit v1.2.3-54-g00ecf From 70cc1ff53b7d113f8aff24034e5f5a25efb8b261 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Fri, 5 Apr 2024 23:41:17 +0200 Subject: fw-addr-lists: rework log messages, always include list name --- fw-addr-lists.rsc | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index 5cdd379..c9f894b 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -58,7 +58,8 @@ :if ([ :len ($List->"cert") ] > 0) do={ :set CheckCertificate true; :if ([ $CertificateAvailable ($List->"cert") ] = false) do={ - $LogPrint warning $ScriptName ("Downloading required certificate failed, trying anyway."); + $LogPrint warning $ScriptName ("Downloading required certificate (" . $FwListName . \ + " / " . $List->"url" . ") failed, trying anyway."); } } @@ -67,7 +68,8 @@ :set Data [ $FetchHuge $ScriptName ($List->"url") $CheckCertificate ]; :if ($Data = false) do={ :if ($I < 5) do={ - $LogPrint debug $ScriptName ("Failed downloading, " . $I . ". try: " . $List->"url"); + $LogPrint debug $ScriptName ("Failed downloading for list '" . $FwListName . \ + "', " . $I . ". try from: " . $List->"url"); :delay (($I * $I) . "s"); } } @@ -77,9 +79,11 @@ :if ($Data = false) do={ :set Data ""; :set Failure true; - $LogPrint warning $ScriptName ("Failed downloading list from: " . $List->"url"); + $LogPrint warning $ScriptName ("Failed downloading for list '" . $FwListName . \ + "' from: " . $List->"url"); } else={ - $LogPrint debug $ScriptName ("Downloaded " . [ $HumanReadableNum [ :len $Data ] 1024 ] . "B from: " . $List->"url"); + $LogPrint debug $ScriptName ("Downloaded " . [ $HumanReadableNum [ :len $Data ] 1024 ] . \ + "B for list '" . $FwListName . "' from: " . $List->"url"); } :while ([ :len $Data ] != 0) do={ @@ -107,13 +111,15 @@ :foreach Entry in=[ /ip/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={ :local Address [ /ip/firewall/address-list/get $Entry address ]; :if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={ - $LogPrint debug $ScriptName ("Renewing IPv4 address for " . ($IPv4Addresses->$Address) . ": " . $Address); + $LogPrint debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \ + "' with " . ($IPv4Addresses->$Address) . ": " . $Address); /ip/firewall/address-list/set $Entry timeout=($IPv4Addresses->$Address); :set ($IPv4Addresses->$Address); :set CntRenew ($CntRenew + 1); } else={ :if ($Failure = false) do={ - $LogPrint debug $ScriptName ("Removing IPv4 address: " . $Address); + $LogPrint debug $ScriptName ("Removing IPv4 address from list '" . $FwListName . \ + "': " . $Address); /ip/firewall/address-list/remove $Entry; :set CntRemove ($CntRemove + 1); } @@ -123,13 +129,15 @@ :foreach Entry in=[ /ipv6/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={ :local Address [ /ipv6/firewall/address-list/get $Entry address ]; :if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={ - $LogPrint debug $ScriptName ("Renewing IPv6 address for " . ($IPv6Addresses->$Address) . ": " . $Address); + $LogPrint debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \ + "' with " . ($IPv6Addresses->$Address) . ": " . $Address); /ipv6/firewall/address-list/set $Entry timeout=($IPv6Addresses->$Address); :set ($IPv6Addresses->$Address); :set CntRenew ($CntRenew + 1); } else={ :if ($Failure = false) do={ - $LogPrint debug $ScriptName ("Removing: " . $Address); + $LogPrint debug $ScriptName ("Removing IPv6 address from list '" . $FwListName . \ + "': " . $Address); /ipv6/firewall/address-list/remove $Entry; :set CntRemove ($CntRemove + 1); } @@ -137,27 +145,32 @@ } :foreach Address,Timeout in=$IPv4Addresses do={ - $LogPrint debug $ScriptName ("Adding IPv4 address for " . $Timeout . ": " . $Address); + $LogPrint debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \ + "' with " . $Timeout . ": " . $Address); :do { /ip/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout; :set ($IPv4Addresses->$Address); :set CntAdd ($CntAdd + 1); } on-error={ - $LogPrint warning $ScriptName ("Failed to add IPv4 address " . $Address . " to list '" . $FwListName . "'."); + $LogPrint warning $ScriptName ("Failed to add IPv4 address to list '" . $FwListName . \ + "': " . $Address); } } :foreach Address,Timeout in=$IPv6Addresses do={ - $LogPrint debug $ScriptName ("Adding IPv6 address for " . $Timeout . ": " . $Address); + $LogPrint debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \ + "' with " . $Timeout . ": " . $Address); :do { /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout; :set ($IPv6Addresses->$Address); :set CntAdd ($CntAdd + 1); } on-error={ - $LogPrint warning $ScriptName ("Failed to add IPv6 address " . $Address . " to list '" . $FwListName . "'."); + $LogPrint warning $ScriptName ("Failed to add IPv6 address to list '" . $FwListName . \ + "': " . $Address); } } - $LogPrint info $ScriptName ("list: " . $FwListName . " -- added: " . $CntAdd . " - renewed: " . $CntRenew . " - removed: " . $CntRemove); + $LogPrint info $ScriptName ("list: " . $FwListName . " -- added: " . $CntAdd . \ + " - renewed: " . $CntRenew . " - removed: " . $CntRemove); } } on-error={ } -- cgit v1.2.3-54-g00ecf From 491d85000d607db09b7abd67d6002c9ca1a870f2 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Sun, 7 Apr 2024 20:54:41 +0200 Subject: fw-addr-lists: human readable numbers for counts --- fw-addr-lists.rsc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index c9f894b..15d292e 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -170,7 +170,9 @@ } } - $LogPrint info $ScriptName ("list: " . $FwListName . " -- added: " . $CntAdd . \ - " - renewed: " . $CntRenew . " - removed: " . $CntRemove); + $LogPrint info $ScriptName ("list: " . $FwListName . \ + " -- added: " . [ $HumanReadableNum $CntAdd 1000 ] . \ + " - renewed: " . [ $HumanReadableNum $CntRenew 1000 ] . \ + " - removed: " . [ $HumanReadableNum $CntRemove 1000 ]); } } on-error={ } -- cgit v1.2.3-54-g00ecf From 19c52ed2a7a59c7ecfb472a9f0bbf4843fc5d713 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Sun, 7 Apr 2024 22:50:04 +0200 Subject: fw-addr-lists: show count of active addresses --- fw-addr-lists.rsc | 1 + 1 file changed, 1 insertion(+) diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index 15d292e..ab417cd 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -171,6 +171,7 @@ } $LogPrint info $ScriptName ("list: " . $FwListName . \ + " (" . [ $HumanReadableNum ($CntAdd + $CntRenew) 1000 ] . ")" . \ " -- added: " . [ $HumanReadableNum $CntAdd 1000 ] . \ " - renewed: " . [ $HumanReadableNum $CntRenew 1000 ] . \ " - removed: " . [ $HumanReadableNum $CntRemove 1000 ]); -- cgit v1.2.3-54-g00ecf From c4d2ea19dda3434c92f604605c1d3f06ec8148c2 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Fri, 5 Apr 2024 23:53:05 +0200 Subject: fw-addr-lists: break long lines --- fw-addr-lists.rsc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc index ab417cd..007282c 100644 --- a/fw-addr-lists.rsc +++ b/fw-addr-lists.rsc @@ -108,7 +108,8 @@ } } - :foreach Entry in=[ /ip/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={ + :foreach Entry in=[ /ip/firewall/address-list/find where \ + list=$FwListName comment=$ListComment ] do={ :local Address [ /ip/firewall/address-list/get $Entry address ]; :if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={ $LogPrint debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \ @@ -126,7 +127,8 @@ } } - :foreach Entry in=[ /ipv6/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={ + :foreach Entry in=[ /ipv6/firewall/address-list/find where \ + list=$FwListName comment=$ListComment ] do={ :local Address [ /ipv6/firewall/address-list/get $Entry address ]; :if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={ $LogPrint debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \ @@ -148,7 +150,8 @@ $LogPrint debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \ "' with " . $Timeout . ": " . $Address); :do { - /ip/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout; + /ip/firewall/address-list/add list=$FwListName comment=$ListComment \ + address=$Address timeout=$Timeout; :set ($IPv4Addresses->$Address); :set CntAdd ($CntAdd + 1); } on-error={ @@ -161,7 +164,8 @@ $LogPrint debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \ "' with " . $Timeout . ": " . $Address); :do { - /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout; + /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \ + address=$Address timeout=$Timeout; :set ($IPv6Addresses->$Address); :set CntAdd ($CntAdd + 1); } on-error={ -- cgit v1.2.3-54-g00ecf