From 5323052290177ff3cfbae5f80a846f5657ae9afd Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Wed, 27 Mar 2024 23:01:38 +0100 Subject: global-functions: introduce $FetchHuge --- README.md | 2 +- global-functions.rsc | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5109173..4516736 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ RouterOS Scripts [![GitHub stars](https://img.shields.io/github/stars/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=red)](https://github.com/eworm-de/routeros-scripts/stargazers) [![GitHub forks](https://img.shields.io/github/forks/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=green)](https://github.com/eworm-de/routeros-scripts/network) [![GitHub watchers](https://img.shields.io/github/watchers/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=blue)](https://github.com/eworm-de/routeros-scripts/watchers) -[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.12-yellow?style=flat)](https://mikrotik.com/download/changelogs/) +[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.13-yellow?style=flat)](https://mikrotik.com/download/changelogs/) [![Telegram group @routeros_scripts](https://img.shields.io/badge/Telegram-%40routeros__scripts-%2326A5E4?logo=telegram&style=flat)](https://t.me/routeros_scripts) [![donate with PayPal](https://img.shields.io/badge/Like_it%3F-Donate!-orange?logo=githubsponsors&logoColor=orange&style=flat)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A4ZXBD6YS2W8J) diff --git a/global-functions.rsc b/global-functions.rsc index e19ed65..bd35cdf 100644 --- a/global-functions.rsc +++ b/global-functions.rsc @@ -4,7 +4,7 @@ # Michael Gisbers # https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md # -# requires RouterOS, version=7.12 +# requires RouterOS, version=7.13 # # global functions # https://git.eworm.de/cgit/routeros-scripts/about/ @@ -32,6 +32,7 @@ :global DownloadPackage; :global EitherOr; :global EscapeForRegEx; +:global FetchHuge; :global FetchUserAgentStr; :global FormatLine; :global FormatMultiLines; @@ -389,6 +390,42 @@ :return $Return; } +# fetch huge data to file, read in chunks +:set FetchHuge do={ + :local ScriptName [ :tostr $1 ]; + :local Url [ :tostr $2 ]; + + :global GetRandom20CharAlNum; + :global LogPrint; + :global MkDir; + :global WaitForFile; + + :if ([ $MkDir "tmpfs/" . $ScriptName ] = false) do={ + $LogPrint error $0 ("Failed creating directory!"); + :return false; + } + + :local FileName ("tmpfs/" . $ScriptName . "/" . $0 . "-" . [ $GetRandom20CharAlNum ]); + + :do { + /tool/fetch check-certificate=yes-without-crl $Url dst-path=$FileName as-value; + } on-error={ + $LogPrint debug $0 ("Failed downloading from: " . $Url); + :return false; + } + $WaitForFile $FileName; + + :local FileSize [ /file/get $FileName size ]; + :local Return ""; + :local VarSize 0; + :while ($VarSize < $FileSize) do={ + :set Return ($Return . ([ /file/read offset=$VarSize chunk-size=32768 file=$FileName as-value ]->"data")); + :set VarSize [ :len $Return ]; + } + /file/remove $FileName; + :return $Return; +} + # generate user agent string for fetch :set FetchUserAgentStr do={ :local Caller [ :tostr $1 ]; -- cgit v1.2.3-54-g00ecf From ab6fd8855881681d7c69af3e5ce18b3a6e2e69fb Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Wed, 27 Mar 2024 23:37:31 +0100 Subject: global-functions: $FetchHuge: use custom user agent string --- global-functions.rsc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/global-functions.rsc b/global-functions.rsc index bd35cdf..0ea7120 100644 --- a/global-functions.rsc +++ b/global-functions.rsc @@ -395,6 +395,7 @@ :local ScriptName [ :tostr $1 ]; :local Url [ :tostr $2 ]; + :global FetchUserAgentStr; :global GetRandom20CharAlNum; :global LogPrint; :global MkDir; @@ -408,7 +409,8 @@ :local FileName ("tmpfs/" . $ScriptName . "/" . $0 . "-" . [ $GetRandom20CharAlNum ]); :do { - /tool/fetch check-certificate=yes-without-crl $Url dst-path=$FileName as-value; + /tool/fetch check-certificate=yes-without-crl $Url dst-path=$FileName \ + http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) as-value; } on-error={ $LogPrint debug $0 ("Failed downloading from: " . $Url); :return false; -- cgit v1.2.3-54-g00ecf From 2edf983698a18d45de1c9e7823dad0959bff2504 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Wed, 27 Mar 2024 23:35:01 +0100 Subject: global-functions: $FetchHuge: control check-certificate with parameter --- global-functions.rsc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/global-functions.rsc b/global-functions.rsc index 0ea7120..65dd06e 100644 --- a/global-functions.rsc +++ b/global-functions.rsc @@ -392,15 +392,19 @@ # fetch huge data to file, read in chunks :set FetchHuge do={ - :local ScriptName [ :tostr $1 ]; - :local Url [ :tostr $2 ]; + :local ScriptName [ :tostr $1 ]; + :local Url [ :tostr $2 ]; + :local CheckCert [ :tobool $3 ]; :global FetchUserAgentStr; :global GetRandom20CharAlNum; + :global IfThenElse; :global LogPrint; :global MkDir; :global WaitForFile; + :set CheckCert [ $IfThenElse ($CheckCert = false) "no" "yes-without-crl" ]; + :if ([ $MkDir "tmpfs/" . $ScriptName ] = false) do={ $LogPrint error $0 ("Failed creating directory!"); :return false; @@ -409,7 +413,7 @@ :local FileName ("tmpfs/" . $ScriptName . "/" . $0 . "-" . [ $GetRandom20CharAlNum ]); :do { - /tool/fetch check-certificate=yes-without-crl $Url dst-path=$FileName \ + /tool/fetch check-certificate=$CheckCert $Url dst-path=$FileName \ http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) as-value; } on-error={ $LogPrint debug $0 ("Failed downloading from: " . $Url); -- cgit v1.2.3-54-g00ecf From 61ee5cbd6cbce28385857f9cca184387234fe326 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Wed, 3 Apr 2024 17:03:27 +0200 Subject: global-functions: $FetchHuge: make sure to work with clean file names --- global-functions.rsc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/global-functions.rsc b/global-functions.rsc index 65dd06e..3c14f22 100644 --- a/global-functions.rsc +++ b/global-functions.rsc @@ -396,6 +396,7 @@ :local Url [ :tostr $2 ]; :local CheckCert [ :tobool $3 ]; + :global CleanName; :global FetchUserAgentStr; :global GetRandom20CharAlNum; :global IfThenElse; @@ -405,13 +406,13 @@ :set CheckCert [ $IfThenElse ($CheckCert = false) "no" "yes-without-crl" ]; - :if ([ $MkDir "tmpfs/" . $ScriptName ] = false) do={ + :local FileName ("tmpfs/" . [ $CleanName $ScriptName ]); + :if ([ $MkDir $FileName ] = false) do={ $LogPrint error $0 ("Failed creating directory!"); :return false; } - :local FileName ("tmpfs/" . $ScriptName . "/" . $0 . "-" . [ $GetRandom20CharAlNum ]); - + :set FileName ($FileName . "/" . [ $CleanName $0 ] . "-" . [ $GetRandom20CharAlNum ]); :do { /tool/fetch check-certificate=$CheckCert $Url dst-path=$FileName \ http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) as-value; -- cgit v1.2.3-54-g00ecf From 402f847db2b7446a67634d3720d824a7e6b87e71 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Wed, 3 Apr 2024 17:36:47 +0200 Subject: global-functions: $FetchHuge: remove file on failed download --- global-functions.rsc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/global-functions.rsc b/global-functions.rsc index 3c14f22..ab8860f 100644 --- a/global-functions.rsc +++ b/global-functions.rsc @@ -417,6 +417,9 @@ /tool/fetch check-certificate=$CheckCert $Url dst-path=$FileName \ http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) as-value; } on-error={ + :if ([ $WaitForFile $FileName 500ms ] = true) do={ + /file/remove $FileName; + } $LogPrint debug $0 ("Failed downloading from: " . $Url); :return false; } -- cgit v1.2.3-54-g00ecf