From 15ca5bfca464e16212435df159e4b60bf97e2627 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Wed, 30 Apr 2025 16:57:00 +0000 Subject: [PATCH 01/54] test --- x | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 x diff --git a/x b/x new file mode 100644 index 00000000..e69de29b From 41b6a93c4c802b0ff6fadc450636aedffccfbc90 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Wed, 30 Apr 2025 16:57:36 +0000 Subject: [PATCH 02/54] test --- x | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 x diff --git a/x b/x deleted file mode 100644 index e69de29b..00000000 From 968d65ddf3c88858749437353fc9bbb1e650ae66 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 5 May 2025 15:49:32 -0700 Subject: [PATCH 03/54] fixing regex raw strings --- srv/dshield/DShield.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/srv/dshield/DShield.py b/srv/dshield/DShield.py index 8c1c7011..a7bfa6e2 100644 --- a/srv/dshield/DShield.py +++ b/srv/dshield/DShield.py @@ -33,10 +33,10 @@ class DshieldSubmit: url = 'https://www.dshield.org/submitapi/' types = ['email', 'firewall', 'sshlogin', 'telnetlogin', '404report', 'httprequest', 'webhoneypot'] - logtypesregex={'generic': '^([A-Z][a-z]{2})\s+([0-9]+)\s([0-9:]+).*(IN=.*)', - 'pi': '(^\d+) \S+ kernel:\[[0-9\. ]+\]\s+DSHIELDINPUT IN=\S+ .* (SRC=.*)', - 'iptables': '(^\d+) \S+ kernel:\s+DSHIELDINPUT IN=\S+ .* (SRC=.*)', - 'aws': '(^\d+) \S+ kernel: DSHIELDINPUT IN=\S+ .* (SRC=.*)'} + logtypesregex={'generic': r'^([A-Z][a-z]{2})\s+([0-9]+)\s([0-9:]+).*(IN=.*)', + 'pi': r'(^\d+) \S+ kernel:\[[0-9\. ]+\]\s+DSHIELDINPUT IN=\S+ .* (SRC=.*)', + 'iptables': r'(^\d+) \S+ kernel:\s+DSHIELDINPUT IN=\S+ .* (SRC=.*)', + 'aws': r'(^\d+) \S+ kernel: DSHIELDINPUT IN=\S+ .* (SRC=.*)'} authheader = '' def __init__(self, filename): @@ -51,10 +51,10 @@ def __init__(self, filename): @staticmethod def testurl(string): - urlre = re.compile('(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:[a-z]{2,13})/)' - '(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+' - '(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'".,<>?])' - '|(?:(?{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+' + r'(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'".,<>?])' + r'|(?:(? Date: Tue, 5 Aug 2025 10:59:54 -0400 Subject: [PATCH 04/54] adding better ini handling and log permissions --- bin/install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index f061e21c..99bfe3f5 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -243,6 +243,7 @@ if [ ! -f "${DSHIELDINI}" ]; then sudo mkdir -p "${ETCDIR}" sudo mv /etc/dshield.ini "${DSHIELDINI}" sudo chown -R "${SYSUSERID}":"${GROUPID}" "${ETCDIR}" + sudo ln -s "{$ETCDIR}/dshield.ini" "/etc/dshield.ini" fi fi @@ -295,10 +296,11 @@ HONEYPORTS="${SSHHONEYPORT} ${TELNETHONEYPORT} ${WEBHONEYPORT}" # create and setup log directory if [ ! -d ${LOGDIR} ]; then - sudo mkdir -m 0770 -p ${LOGDIR} + sudo mkdir -m 1777 -p ${LOGDIR} fi # for legacy systems that used to run as root sudo chown -R "${SYSUSERID}":"${GROUPID}" "${LOGDIR}" +sudo chmod 1777 "${LOGDIR}" # and the local etc dir may need cleaning up from legacy files as root sudo chown -R "${SYSUSERID}":"${GROUPID}" "$progdir" # which port the real sshd should listen to @@ -2008,6 +2010,11 @@ echo "${offset1} ${offset2} * * * root /sbin/reboot" >>"${TMPDIR}"/cron.dshield offset1=$(shuf -i0-59 -n1) offset2=$(shuf -i0-23 -n1) echo "${offset1} ${offset2} * * * root ${DSHIELDDIR}/updatehoneypotip.sh" >>"${TMPDIR}"/cron.dshield +offset1=$(shuf -i0-59 -n1) +offset2=$(shuf -i0-23 -n1) +echo "${offset1} ${offset2} * * * root /usr/bin/journalctl --vacuum-time=7d" >>"${TMPDIR}"/cron.dshield + + # run status check 5 minutes before reboot if [ "$offset1" -gt 5 ]; then offset1=$((offset1 - 5)) From ecb281b3fa5eaa1de6c05e6e372b3b1322846d14 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 11:00:17 -0400 Subject: [PATCH 05/54] fixes to cleanup script --- .gitignore | 3 ++- bin/cleanup.sh | 55 +++++++++++++++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index d9f1eb88..64639f58 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ etc/CA/index.txt.old etc/openssl.cnf.bak etc/dshield.sslca etc/CA/keys/ -etc/openssl.cnf \ No newline at end of file +etc/openssl.cnf +srv/web/isc_agent.pyz \ No newline at end of file diff --git a/bin/cleanup.sh b/bin/cleanup.sh index 2bfbe0e2..52c4c828 100755 --- a/bin/cleanup.sh +++ b/bin/cleanup.sh @@ -5,7 +5,12 @@ # Cleanup script to safe disk space # # the script will ask for confirmation, unless you -# add the optional 'cron' commandline parameter +# add the optional 'cron' commandline parameter. +# If run with 'cron', only files older than 14 days are +# deleted. +# +# in addition, logs from journald older than 7 days are +# deleted. # #### @@ -17,28 +22,36 @@ if [ ! "$uid" = "0" ]; then fi if [[ $1 != cron ]]; then -echo -echo " This script will delete backups and log files." -echo " Do you want to proceed? (Y/N)" -echo -while : ; do - read -s -p "Press Y/N key: " -n 1 k <&1 - k=${k^} - echo $k - if [[ $k = N ]]; then - echo - echo "abort" + echo + echo " This script will delete backups and log files." + echo " Do you want to proceed? (Y/N)" + echo + while : ; do + read -s -p "Press Y/N key: " -n 1 k <&1 + k=${k^} + echo $k + if [[ $k = N ]]; then + echo + echo "abort" exit - fi - if [[ $k = Y ]]; then + fi + if [[ $k = Y ]]; then echo echo "removing logs and backups" break - fi -done + fi + done + rm -rf /srv/www.2* + rm -rf /srv/dshield.2* + rm -rf /srv/cowrie.2* + rm -rf /srv/log/* + rm -rf /srv/cowrie/var/log/cowrie/* + /usr/bin/journalctl --vacuum-time=7d +else + find /srv -name 'www.2*' -ctime +14 -delete + find /srv -name 'dshield.2*' -ctime +14 -delete + find /srv -name 'cowrie.2*' -ctime +14 -delete + find /srv/log -ctime +14 -delete + find /srv/cowrie/var/log/cowrie/* -ctime +14 -delete + /usr/bin/journalctl --vacuum-time=7d fi -rm -rf /srv/www.2* -rm -rf /srv/dshield.2* -rm -rf /srv/cowrie.2* -rm -rf /srv/log/* -rm -rf /srv/cowrie/var/log/cowrie/* From b01fc8c0c39a241f8846410f014d5215572424f8 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 12:13:54 -0400 Subject: [PATCH 06/54] allowing for date in json log --- bin/install.sh | 2 ++ srv/web/isc_agent/__main__.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/bin/install.sh b/bin/install.sh index 19df4f5d..f0744ca7 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2085,6 +2085,8 @@ run "echo 'queue_trigger=100 ; logs are submitted if this many entries are in th run "echo 'web_log_limit=1000 ; how many logs in the queue are considered excessive and trigger skipping' >> ${DSHIELDINI}" run "echo 'web_log_purgefactor=2 ; if excessive logs are received, each 2nd log entry is dropped' >> ${DSHIELDINI}" run "echo 'debug=false ; true enabled additional logging' >> ${DSHIELDINI}" +run "echo 'local_response_path=/var/log/dshield/local_response_{date}.json ; where to store local responses' >> ${DSHIELDINI}" +run "echo 'record_local_responses=false ; true to record local responses' >> ${DSHIELDINI}" run "echo '# this section may disappear in future versions' >> ${DSHIELDINI}" run "echo '[iscagent]' >> ${DSHIELDINI}" database=$(quotespace "$database") diff --git a/srv/web/isc_agent/__main__.py b/srv/web/isc_agent/__main__.py index 841ea397..d9e78397 100644 --- a/srv/web/isc_agent/__main__.py +++ b/srv/web/isc_agent/__main__.py @@ -194,6 +194,9 @@ def __str__(self): if record_local_responses: try: + # if the file name "local_response_path" includes the string {date} then replace it with the current date + if "{date}" in local_response_path: + local_response_path = local_response_path.replace("{date}", datetime.datetime.now().strftime("%Y-%m-%d")) fh = open(local_response_path, "a") fh.write(f"{json.dumps(log_data)}\n") fh.close() From 71493c8d8b151a8f38d0dc622a0a6c403c3a40d1 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 12:48:05 -0400 Subject: [PATCH 07/54] fixing prefil of dshield.ini --- bin/install.sh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index f0744ca7..02cb3b9c 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1076,6 +1076,29 @@ else dlog "old piid ${piid}" fi +if [ "$record_local_repsonses" == "" ]; then + record_local_repsonses=false +fi +if [ "$local_response_path" == "" ]; then + local_response_path='/srv/log/dshield/local_response_{date}.json' +fi + +if [ "$submit_log_rate" == "" ]; then + submit_log_rate=900 +fi + +if [ "$queue_trigger" == "" ]; then + queue_trigger=100 +fi + +if [ "$web_log_limit" == "" ]; then + web_log_limit=1000 +fi + +if [ "$web_log_purge_factor" == "" ]; then + web_log_purge_factor=2 +} + ########################################################### ## DShield Account ########################################################### @@ -2080,13 +2103,13 @@ run "echo 'telnet=$telnet ; enable telnet' >> ${DSHIELDINI} " run "echo '[plugin:tcp:http]' >> ${DSHIELDINI}" run "echo 'http_ports=[8000] ; what http port to listen on' >> ${DSHIELDINI}" run "echo 'https_ports=[8443] ; what https port to listen on with stunnel' >> ${DSHIELDINI}" -run "echo 'submit_logs_rate=300 ; how often to submit logs in seconds' >> ${DSHIELDINI}" -run "echo 'queue_trigger=100 ; logs are submitted if this many entries are in the queue even' >> ${DSHIELDINI}" -run "echo 'web_log_limit=1000 ; how many logs in the queue are considered excessive and trigger skipping' >> ${DSHIELDINI}" -run "echo 'web_log_purgefactor=2 ; if excessive logs are received, each 2nd log entry is dropped' >> ${DSHIELDINI}" +run "echo 'submit_log_rate=$submit_log_rate ; how often to submit logs in seconds' >> ${DSHIELDINI}" +run "echo 'queue_trigger=$queue_trigger ; logs are submitted if this many entries are in the queue even' >> ${DSHIELDINI}" +run "echo 'web_log_limit=$web_log_limit ; how many logs in the queue are considered excessive and trigger skipping' >> ${DSHIELDINI}" +run "echo 'web_log_purgefactor=$web_log_purgefactor ; if excessive logs are received, each 2nd log entry is dropped' >> ${DSHIELDINI}" run "echo 'debug=false ; true enabled additional logging' >> ${DSHIELDINI}" -run "echo 'local_response_path=/var/log/dshield/local_response_{date}.json ; where to store local responses' >> ${DSHIELDINI}" -run "echo 'record_local_responses=false ; true to record local responses' >> ${DSHIELDINI}" +run "echo 'local_response_path=$local_response_path ; where to store local responses' >> ${DSHIELDINI}" +run "echo 'record_local_responses=$record_local_repsonses ; true to record local responses' >> ${DSHIELDINI}" run "echo '# this section may disappear in future versions' >> ${DSHIELDINI}" run "echo '[iscagent]' >> ${DSHIELDINI}" database=$(quotespace "$database") From fe5bc9e8f8f96ee97a2dcb6696b1ce12ef0cb7ff Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 13:49:37 -0400 Subject: [PATCH 08/54] removing semicolon comments --- bin/install.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 02cb3b9c..db6eb4cf 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1080,7 +1080,7 @@ if [ "$record_local_repsonses" == "" ]; then record_local_repsonses=false fi if [ "$local_response_path" == "" ]; then - local_response_path='/srv/log/dshield/local_response_{date}.json' + local_response_path='/srv/log/local_response_{date}.json' fi if [ "$submit_log_rate" == "" ]; then @@ -2095,21 +2095,21 @@ run "echo 'localips=$CONIPS' >> ${DSHIELDINI}" ADMINPORTS=$(quotespace "$ADMINPORTS") run "echo 'adminports=$ADMINPORTS ' >> ${DSHIELDINI}" nohoneyips=$(quotespace "$nohoneyips") -run "echo 'nohoneyips=$nohoneyips ; ignore connections from these IPs and do not log' >> ${DSHIELDINI}" +run "echo 'nohoneyips=$nohoneyips' >> ${DSHIELDINI}" nohoneyports=$(quotespace "$nohoneyports") -run "echo 'nohoneyports=$nohoneyports ; which ports not to log' >> ${DSHIELDINI}" -run "echo 'manualupdates=$MANUPDATES ; false to enable automatic updates' >> ${DSHIELDINI}" -run "echo 'telnet=$telnet ; enable telnet' >> ${DSHIELDINI} " +run "echo 'nohoneyports=$nohoneyports' >> ${DSHIELDINI}" +run "echo 'manualupdates=$MANUPDATES' >> ${DSHIELDINI}" +run "echo 'telnet=$telnet' >> ${DSHIELDINI} " run "echo '[plugin:tcp:http]' >> ${DSHIELDINI}" -run "echo 'http_ports=[8000] ; what http port to listen on' >> ${DSHIELDINI}" -run "echo 'https_ports=[8443] ; what https port to listen on with stunnel' >> ${DSHIELDINI}" -run "echo 'submit_log_rate=$submit_log_rate ; how often to submit logs in seconds' >> ${DSHIELDINI}" -run "echo 'queue_trigger=$queue_trigger ; logs are submitted if this many entries are in the queue even' >> ${DSHIELDINI}" -run "echo 'web_log_limit=$web_log_limit ; how many logs in the queue are considered excessive and trigger skipping' >> ${DSHIELDINI}" -run "echo 'web_log_purgefactor=$web_log_purgefactor ; if excessive logs are received, each 2nd log entry is dropped' >> ${DSHIELDINI}" -run "echo 'debug=false ; true enabled additional logging' >> ${DSHIELDINI}" -run "echo 'local_response_path=$local_response_path ; where to store local responses' >> ${DSHIELDINI}" -run "echo 'record_local_responses=$record_local_repsonses ; true to record local responses' >> ${DSHIELDINI}" +run "echo 'http_ports=[8000]' >> ${DSHIELDINI}" +run "echo 'https_ports=[8443]' >> ${DSHIELDINI}" +run "echo 'submit_log_rate=$submit_log_rate' >> ${DSHIELDINI}" +run "echo 'queue_trigger=$queue_trigger' >> ${DSHIELDINI}" +run "echo 'web_log_limit=$web_log_limit' >> ${DSHIELDINI}" +run "echo 'web_log_purgefactor=$web_log_purgefactor' >> ${DSHIELDINI}" +run "echo 'debug=false' >> ${DSHIELDINI}" +run "echo 'local_response_path=$local_response_path' >> ${DSHIELDINI}" +run "echo 'record_local_responses=$record_local_repsonses' >> ${DSHIELDINI}" run "echo '# this section may disappear in future versions' >> ${DSHIELDINI}" run "echo '[iscagent]' >> ${DSHIELDINI}" database=$(quotespace "$database") From ec599de0a03324f0cd1b5db4ac461324b5a1ad5e Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 13:57:00 -0400 Subject: [PATCH 09/54] fix type --- bin/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index db6eb4cf..0b350b15 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1076,7 +1076,7 @@ else dlog "old piid ${piid}" fi -if [ "$record_local_repsonses" == "" ]; then +if [ "$record_local_responses" == "" ]; then record_local_repsonses=false fi if [ "$local_response_path" == "" ]; then @@ -1097,7 +1097,7 @@ fi if [ "$web_log_purge_factor" == "" ]; then web_log_purge_factor=2 -} +fi ########################################################### ## DShield Account From 179235ef109ee72db341d0fc76d32e084daf8e9b Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 14:02:18 -0400 Subject: [PATCH 10/54] fix quotes --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 0b350b15..f099ffd8 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1077,7 +1077,7 @@ else fi if [ "$record_local_responses" == "" ]; then - record_local_repsonses=false + record_local_repsonses='false' fi if [ "$local_response_path" == "" ]; then local_response_path='/srv/log/local_response_{date}.json' From 619bec6f827de499c71e48299b1ad6aff8fca748 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 14:05:12 -0400 Subject: [PATCH 11/54] fix quotes --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index f099ffd8..4f1515dd 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1077,7 +1077,7 @@ else fi if [ "$record_local_responses" == "" ]; then - record_local_repsonses='false' + record_local_responses='false' fi if [ "$local_response_path" == "" ]; then local_response_path='/srv/log/local_response_{date}.json' From 361363be351572b6e350bb215d90838d0c22dfaf Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 14:11:58 -0400 Subject: [PATCH 12/54] fix type --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 4f1515dd..6462c6e1 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2109,7 +2109,7 @@ run "echo 'web_log_limit=$web_log_limit' >> ${DSHIELDINI}" run "echo 'web_log_purgefactor=$web_log_purgefactor' >> ${DSHIELDINI}" run "echo 'debug=false' >> ${DSHIELDINI}" run "echo 'local_response_path=$local_response_path' >> ${DSHIELDINI}" -run "echo 'record_local_responses=$record_local_repsonses' >> ${DSHIELDINI}" +run "echo 'record_local_responses=$record_local_responses' >> ${DSHIELDINI}" run "echo '# this section may disappear in future versions' >> ${DSHIELDINI}" run "echo '[iscagent]' >> ${DSHIELDINI}" database=$(quotespace "$database") From 8b4ac9f3066eab8540c4db77ba668ceabe765ac2 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 18:12:12 +0000 Subject: [PATCH 13/54] x --- bin/install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 6462c6e1..9a270a87 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1075,7 +1075,6 @@ if [ "$piid" == "" ]; then else dlog "old piid ${piid}" fi - if [ "$record_local_responses" == "" ]; then record_local_responses='false' fi From f2f228176323ae04e32da5f8032381f0674c9455 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 15:06:14 -0400 Subject: [PATCH 14/54] typo --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 6462c6e1..2cb9696c 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -243,7 +243,7 @@ if [ ! -f "${DSHIELDINI}" ]; then sudo mkdir -p "${ETCDIR}" sudo mv /etc/dshield.ini "${DSHIELDINI}" sudo chown -R "${SYSUSERID}":"${GROUPID}" "${ETCDIR}" - sudo ln -s "{$ETCDIR}/dshield.ini" "/etc/dshield.ini" + sudo ln -s "${ETCDIR}/dshield.ini" "/etc/dshield.ini" fi fi From 8f40f11130fa18c275700b569a084a0c4bb27cda Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 15:18:17 -0400 Subject: [PATCH 15/54] fixing CA permissions --- bin/install.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/install.sh b/bin/install.sh index 61c2f594..3135ca6d 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2481,6 +2481,13 @@ else fi +### +# fixing legacy permissions +### + +sudo find CA -uid 0 -exec chown ${SYSUSERID} {} \; + + if [ ${GENCERT} -eq 1 ]; then dlog "generating new CERTs using ./makecert.sh" From 8c99737bb01f1e0a1f2b980ac54a79523ef42785 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 15:20:38 -0400 Subject: [PATCH 16/54] fixing permis --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 3135ca6d..b4ac5209 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2485,7 +2485,7 @@ fi # fixing legacy permissions ### -sudo find CA -uid 0 -exec chown ${SYSUSERID} {} \; +sudo find ${SCRIPTDIR}/../etc/CA -uid 0 -exec chown ${SYSUSERID} {} \; From cb6842451216c9b83e571582f986a2c5049876c8 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 5 Aug 2025 15:34:34 -0400 Subject: [PATCH 17/54] updating version --- bin/install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index b4ac5209..c4e91253 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -13,13 +13,17 @@ ## CONFIG SECTION ########################################################### -# version 2025/04/23 01 +# version 2025/08/05 -readonly myversion=97 +readonly myversion=98 # Major Changes (for details, see Github): # +# - V98 (Johannes) +# - new web hpot (Mark Baggett) +# - installer no longer requires root / better priv separation +# # - V97 (Johannes) # - swap in Mark's web honeypot to replace isc-agent # - remove dependency to run as root From c91b9721ce0797638aa42d9a20186acf0a3dbee2 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Wed, 6 Aug 2025 08:37:05 -0400 Subject: [PATCH 18/54] fixing typo in status script --- bin/status.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/status.sh b/bin/status.sh index 4ef29157..2563d33b 100755 --- a/bin/status.sh +++ b/bin/status.sh @@ -309,7 +309,7 @@ for key in "${!TESTS[@]}"; do data="$data, { '${key}': '${TESTS[$key]}' }" done data="$data ]" -curl -s https://isc.sans.edu/api/hpstatusreport/$user/$nonce/$hash/$verison/$piid -d "$data" > /dev/null +curl -s https://isc.sans.edu/api/hpstatusreport/$user/$nonce/$hash/$version/$piid -d "$data" > /dev/null echo echo "also check https://isc.sans.edu/myreports.html (after logging in)" echo "to see that your reports arrive." From 64957eb50a26fb7398a972f8ccdd3158f0f29284 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 11 Aug 2025 14:13:10 -0400 Subject: [PATCH 19/54] fixing sg vs sudo for cowrie --- bin/install.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index c4e91253..5bbcddb8 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2125,6 +2125,9 @@ drun "cat ${DSHIELDINI}" # making sure permissions are right for the ini file sudorun "chown -R ${SYSUSERID}:webhpot ${DSHIELDDIR}" sudorun "chmod 0640 ${DSHIELDINI}" +if [ -f /etc/dshield.ini ]; then + sudorun "rm /etc/dshield.ini" +fi sudorun "ln -s ${DSHIELDINI} /etc/dshield.ini" @@ -2229,10 +2232,10 @@ dlog "activating virtual environment" run 'source cowrie-env/bin/activate' if [ "$FAST" == "0" ]; then dlog "installing cowrie dependencies: requirements.txt" - run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade pip"' - run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade bcrypt"' - run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade requests"' - run 'sg cowrie -c "pip3 install --require-virtualenv -r requirements.txt"' + run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade pip"' + run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade bcrypt"' + run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade requests"' + run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv -r requirements.txt"' # shellcheck disable=SC2181 if [ ${?} -ne 0 ]; then outlog "Error installing dependencies from requirements.txt. See ${LOGFILE} for details." @@ -2248,7 +2251,7 @@ fi # dlog "installing dependencies requirements-output.txt" # run 'pip3 install --upgrade -r requirements-output.txt' if [ "$ID" != "opensuse" ] ; then - run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade requests"' + run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade requests"' # shellcheck disable=SC2181 if [ ${?} -ne 0 ]; then outlog "Error installing dependencies from requirements-output.txt. See ${LOGFILE} for details." From b7bb16c479fdbd6fc2f7a4fc984de32c88a95e2b Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 11 Aug 2025 14:18:00 -0400 Subject: [PATCH 20/54] fixing -c --- bin/install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 5bbcddb8..4f172b91 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2232,10 +2232,10 @@ dlog "activating virtual environment" run 'source cowrie-env/bin/activate' if [ "$FAST" == "0" ]; then dlog "installing cowrie dependencies: requirements.txt" - run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade pip"' - run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade bcrypt"' - run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade requests"' - run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv -r requirements.txt"' + run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade pip"' + run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade bcrypt"' + run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade requests"' + run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv -r requirements.txt"' # shellcheck disable=SC2181 if [ ${?} -ne 0 ]; then outlog "Error installing dependencies from requirements.txt. See ${LOGFILE} for details." @@ -2251,7 +2251,7 @@ fi # dlog "installing dependencies requirements-output.txt" # run 'pip3 install --upgrade -r requirements-output.txt' if [ "$ID" != "opensuse" ] ; then - run 'sudo -u cowrie -g cowrie -c "pip3 install --require-virtualenv --upgrade requests"' + run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade requests"' # shellcheck disable=SC2181 if [ ${?} -ne 0 ]; then outlog "Error installing dependencies from requirements-output.txt. See ${LOGFILE} for details." From 356866f63f28f4f46cbcb93cf6bca1881c61e8bc Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 11 Aug 2025 14:26:11 -0400 Subject: [PATCH 21/54] switching back go sg --- bin/install.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 4f172b91..d236d86a 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2227,15 +2227,17 @@ sudorun "chown ${SYSUSERID}:cowrie ${COWRIEDIR}" sudorun "chmod 0770 $COWRIEDIR" cd ${COWRIEDIR} || exit dlog "setting up virtual environment" +run 'sudo chgrp -R cowrie /srv/cowrie' +run 'sudo chmod g+w cowrie-env/lib' run 'sudo -u cowrie virtualenv --python=python3 cowrie-env' dlog "activating virtual environment" run 'source cowrie-env/bin/activate' if [ "$FAST" == "0" ]; then dlog "installing cowrie dependencies: requirements.txt" - run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade pip"' - run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade bcrypt"' - run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade requests"' - run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv -r requirements.txt"' + run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade pip"' + run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade bcrypt"' + run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade requests"' + run 'sg cowrie -c "pip3 install --require-virtualenv -r requirements.txt"' # shellcheck disable=SC2181 if [ ${?} -ne 0 ]; then outlog "Error installing dependencies from requirements.txt. See ${LOGFILE} for details." @@ -2251,7 +2253,7 @@ fi # dlog "installing dependencies requirements-output.txt" # run 'pip3 install --upgrade -r requirements-output.txt' if [ "$ID" != "opensuse" ] ; then - run 'sudo -u cowrie -g cowrie "pip3 install --require-virtualenv --upgrade requests"' + run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade requests"' # shellcheck disable=SC2181 if [ ${?} -ne 0 ]; then outlog "Error installing dependencies from requirements-output.txt. See ${LOGFILE} for details." From 4e92a564f2312ebe92582b2450acc81fdf30e7b3 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 11 Aug 2025 23:05:01 -0400 Subject: [PATCH 22/54] fixing path for chmod --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index d236d86a..e4926d61 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2228,7 +2228,7 @@ sudorun "chmod 0770 $COWRIEDIR" cd ${COWRIEDIR} || exit dlog "setting up virtual environment" run 'sudo chgrp -R cowrie /srv/cowrie' -run 'sudo chmod g+w cowrie-env/lib' +run 'sudo chmod g+w /srv/cowrie/cowrie-env/lib' run 'sudo -u cowrie virtualenv --python=python3 cowrie-env' dlog "activating virtual environment" run 'source cowrie-env/bin/activate' From e91fa7e7ea2b3af6ac5a0a19289715bbb7a44140 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 12 Aug 2025 13:28:17 -0400 Subject: [PATCH 23/54] fixing permissions --- bin/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/install.sh b/bin/install.sh index e4926d61..2cfc5de9 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2056,6 +2056,7 @@ fi echo "${offset1} ${offset2} * * * root cd ${DSHIELDDIR}; ./status.sh >/dev/null " >> "${TMPDIR}"/cron.dshield echo "0 10 * * * root find /srv/cowrie/var/log/cowrie -name 'cowrie.*' -ctime +7 -delete" >> "${TMPDIR}"/cron.dshield sudorun "cp ${TMPDIR}/cron.dshield /etc/cron.d/dshield" +sudorun "chmod 775 /etc/cron.d/dshield" dsudorun 'cat /etc/cron.d/dshield' # From 11e095cc1ef6156a4779b4aec28997b89b8af2da Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Wed, 13 Aug 2025 11:50:49 -0400 Subject: [PATCH 24/54] switch order of chgrp and venv --- bin/install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 2cfc5de9..44f43669 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2228,11 +2228,12 @@ sudorun "chown ${SYSUSERID}:cowrie ${COWRIEDIR}" sudorun "chmod 0770 $COWRIEDIR" cd ${COWRIEDIR} || exit dlog "setting up virtual environment" +run 'sudo -u cowrie virtualenv --python=python3 cowrie-env' run 'sudo chgrp -R cowrie /srv/cowrie' run 'sudo chmod g+w /srv/cowrie/cowrie-env/lib' -run 'sudo -u cowrie virtualenv --python=python3 cowrie-env' dlog "activating virtual environment" run 'source cowrie-env/bin/activate' + if [ "$FAST" == "0" ]; then dlog "installing cowrie dependencies: requirements.txt" run 'sg cowrie -c "pip3 install --require-virtualenv --upgrade pip"' From 7183fa245ecd2b5b1299b33886de74573579cae2 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Wed, 13 Aug 2025 12:27:39 -0400 Subject: [PATCH 25/54] added -R --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 44f43669..618e0e4e 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2230,7 +2230,7 @@ cd ${COWRIEDIR} || exit dlog "setting up virtual environment" run 'sudo -u cowrie virtualenv --python=python3 cowrie-env' run 'sudo chgrp -R cowrie /srv/cowrie' -run 'sudo chmod g+w /srv/cowrie/cowrie-env/lib' +run 'sudo chmod -R g+w /srv/cowrie/cowrie-env/' dlog "activating virtual environment" run 'source cowrie-env/bin/activate' From cb6111cb94758987dd00bc2483d308cfec7b0e25 Mon Sep 17 00:00:00 2001 From: sec573 <573@573.local> Date: Wed, 13 Aug 2025 14:06:47 -0400 Subject: [PATCH 26/54] Fix {date} processing in local log file name --- srv/web/isc_agent/__main__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/srv/web/isc_agent/__main__.py b/srv/web/isc_agent/__main__.py index d9e78397..e8a83750 100644 --- a/srv/web/isc_agent/__main__.py +++ b/srv/web/isc_agent/__main__.py @@ -195,9 +195,11 @@ def __str__(self): if record_local_responses: try: # if the file name "local_response_path" includes the string {date} then replace it with the current date - if "{date}" in local_response_path: - local_response_path = local_response_path.replace("{date}", datetime.datetime.now().strftime("%Y-%m-%d")) - fh = open(local_response_path, "a") + #"local_response_path" is a global variable and can only be read not assigned so a local variable log_path is used + log_path = local_response_path + if "{date}" in log_path: + log_path = log_path.replace("{date}", datetime.datetime.now().strftime("%Y-%m-%d")) + fh = open(log_path, "a") fh.write(f"{json.dumps(log_data)}\n") fh.close() except Exception as e: From 25150024ea051b4cad29cdc50359fec245408efb Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 15 Aug 2025 15:22:25 -0400 Subject: [PATCH 27/54] Update STATUSERRORS.md fix /etc/cowrie to /srv/cowrie --- STATUSERRORS.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/STATUSERRORS.md b/STATUSERRORS.md index e47c9be8..ef19cafa 100644 --- a/STATUSERRORS.md +++ b/STATUSERRORS.md @@ -44,19 +44,15 @@ Please ask for additional help if your system uses nft) ## /etc/cron.d/dshield -missing this file usally indicates a failed install. Reinstall the honeypot. +Missing this file usually indicates a failed install. Reinstall the honeypot. ## /etc/dshield.ini -missing this file usally indicates a failed install. Reinstall the honeypot. +Missing this file usually indicates a failed install. Reinstall the honeypot. -## /etc/cowrie/cowrie.cfg +## /srv/cowrie/cowrie.cfg -missing this file usally indicates a failed install. Reinstall the honeypot. - -## /etc/cowrie/cowrie.cfg - -missing this file usally indicates a failed install. Reinstall the honeypot. +Missing this file usually indicates a failed install. Reinstall the honeypot. ## ip-firewall rules @@ -91,7 +87,7 @@ If that doesn't work, see below or send the content of /srv/log/isc-agent.err to ### Details ### -Check the file ```/srv/log/isc-agent.err```. It should display any startup errors. Often the issue is caused by a missing python module. For example: +Check the file ```/srv/log/isc-agent.err```. It should display any startup errors. Often, the issue is caused by a missing Python module. For example: ``` Traceback (most recent call last): From 0c871137958bfa5355c889c020d1fb0ffb651fed Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 15 Aug 2025 15:36:38 -0400 Subject: [PATCH 28/54] Update README.md --- README.md | 85 +++++++++++++------------------------------------------ 1 file changed, 19 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 1ce3503c..e14b1480 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,20 @@ ## DShield Raspberry Pi Sensor -This is a set of scripts to setup a Raspberry Pi as a DShield Sensor. +This is a set of scripts to set up a Raspberry Pi as a DShield Sensor. Current design goals and prerequisites for using the automated installation procedure: - use of a __dedicated__ device (Raspberry Pi 3 or later, n100 mini PC or a virtual machine work fine) - minimum of 1GB of RAM and 16GB of Disk (SD Card for Raspberry Pis). 4GB of RAM works better. Larger SD Cards (e.g. 64 GB) are recommended for longevity and to prevent logs from filling up the disk. - current Raspberry Pi OS ("Lite" version will suffice) or current version of Ubuntu Linux -- easy installation / configuration (and therefore not that much configurable) -- disposable (when something breaks (e.g. during upgrade): re-install from scratch) -- minimize complexity and overhead (e.g. no virtualization like docker) +- easy installation/configuration (and therefore not that configurable) +- disposable (when something breaks (e.g., during upgrade): re-install from scratch) +- minimize complexity and overhead (e.g., no virtualization like Docker) - support for IPv4 only (for the internal net) -- one interface only (e.g. eth0) +- one interface only (e.g,. eth0) The current version is only tested on Raspberry Pi OS and Ubuntu 22.04 LTS Server, not on other distros, sorry. -If there is the need for other distros, "someone" has to check and maintain the installation script. +If there is a need for other distros, "someone" has to check and maintain the installation script. ## Installation @@ -29,17 +29,18 @@ Reference the following files for OS-specific installation instructions: This script will: -- disable IPv6 on the Pi -- enable firewall logging and submitting of logs to DShield -- change your ssh server to listen on port 12222 -- install the ssh honeypot cowrie (for ssh) -- install needed environment (e.g. MySQL server, Python packages, ...) +- Disable IPv6 on the Pi +- enable firewall logging and submission of logs to DShield +- Change your SSH server to listen on port 12222 +- install the SSH honeypot Cowrie (for SSH) +- install the HTTP honeypot ("isc_agent") +- install needed environment (Python packages, stunnel for https, ...) ## Troubleshooting -- logs are sent twice an hour to the [dshield portal](https://www.dshield.org) by the cron job `/etc/cron.d/dshield`, this can be verified by ['My Account' -> 'My Reports'](https://www.dshield.org/myreports.html) -- have a look at the output from the status script: `/home/pi/install/dshield/bin/status.sh` -- if you get strange python / pip errors during installation / updates you may try the following commands as root: +- Logs are sent twice an hour to the [dshield portal](https://www.dshield.org) by the cron job `/etc/cron.d/dshield`, this can be verified by ['My Account' -> 'My Reports'](https://www.dshield.org/myreports.html) +- Have a look at the output from the status script: `/srv/dshield/status.sh` +- If you get strange Python or pip-related errors during installation or during updates, try the following commands as root: `pip freeze --local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip install -U` - Check our [Trouble Shooting Guide](https://github.com/DShield-ISC/dshield/blob/main/docs/general-guides/Troubleshooting.md) for help identifying issues. @@ -60,56 +61,8 @@ Configuration parameters like your API Key will be retained. To edit the configu A new feature has been introduced, especially for automatic updates. At the end of the installation the install.sh script will search for the file /root/bin/postinstall.sh and execute its content, if it exists. If you need some extra changes in the newly installed files, this is the location to put them. This file NEEDS execute rights. -Please make sure to keep special port and network configuration up to date (e.g. manually configure recently added telnet / web ports in firewall config), e.g. no-log config, no-honey config, ... unfortunately this can't be done automagically as of now. If unsure delete respective lines in `/etc/dshield.ini` and re-run the installation script. +Please make sure to keep special port and network configuration up to date (e.g., manually configure recently added telnet / web ports in firewall config), e.g., no-log config, no-honey config, ... unfortunately, this can't be done automagically as of now. If unsure, delete respective lines in `/etc/dshield.ini` and re-run the installation script. -Testing of update procedure is normally done (between two releases) as follows: -- update on Pi 3 from the last version to current -- install on a current clean image of raspbian lite on a Pi 3 - -### Special Update Note: Versions < 0.4 to >= 0.4 - -The handling of Python packages had to be changed from distro package manager to pip. This means the update is pain. Sorry for that. - -You have three alternatives: - -#### Easy - -The easiest, preferred and warmly recommended way: backup old installation (if you can't stand a complete loss), reinstall from scratch using current Raspbian image. - -#### Manual - -The manual procedure: uninstall all below mentioned packages and then autoremove and cross fingers: -``` -sudo su - -/etc/init.d/cowrie stop -dpkg --remove python-crypto -dpkg --remove python-gmpy -dpkg --remove python-gmpy2 -dpkg --remove python-mysqldb -dpkg --remove python-pip -dpkg --remove python-pyasn1 -dpkg --remove python-twisted -dpkg --remove python-virtualenv -dpkg --remove python-zope.interface -apt-get autoremove -apt-get update -apt-get dist-upgrade -``` - -#### Automatic - -The "automatic" **brutal** procedure (chances to break your system are **VERY** high, but hey, it's a disposable honeypot anyway ...): backup (if needed), uninstall all Python distro packages (and hope that's it): -``` -sudo su - -/etc/init.d/cowrie stop -for PKG in `dpkg --list | grep python- | cut -d " " -f 3 | grep "^python"` ; do echo "uninstalling ${PKG}"; dpkg --force-depends --purge ${PKG}; done -apt-get update -apt-get -f install -apt-get dist-upgrade -apt-get autoremove -apt-get update -apt-get dist-upgrade -``` ## Hints @@ -133,14 +86,14 @@ To test your set up you may use a public port scanner and point it to the router - ESC: exit the form (Cancel) - cursor up / down: navigate through form / between input fields - cursor left / right: navigate within an input field -- TAB: switch between input field and "buttons" +- TAB: switch between the input fields and "buttons" - don't use Pos 1 / End -## Todos +## TODOs - see comments in `install.sh` - provide a script to update all Python packages to most recent version using pip -- configure a default web server and submit logs to DShield +- Configure a default web server and submit logs to DShield - enable other honeypot ports than ssh - do all the user input stuff at the beginning of the script so it will run the long lasting stuff afterwards - create update script From b218c896b428e57de1ca0ba69015ded357d221ce Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 15 Aug 2025 15:37:45 -0400 Subject: [PATCH 29/54] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e14b1480..048ff869 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # DShield -## DShield Raspberry Pi Sensor +## DShield Sensor -This is a set of scripts to set up a Raspberry Pi as a DShield Sensor. +This is a set of scripts to set up a DShield Sensor. Current design goals and prerequisites for using the automated installation procedure: - use of a __dedicated__ device (Raspberry Pi 3 or later, n100 mini PC or a virtual machine work fine) From 4c6f84e0f8c477d568e3449e2b9bfd0d2ada4434 Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 15 Aug 2025 15:39:17 -0400 Subject: [PATCH 30/54] Update Raspbian.md --- docs/install-instructions/Raspbian.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/install-instructions/Raspbian.md b/docs/install-instructions/Raspbian.md index fb5329d8..9931563c 100644 --- a/docs/install-instructions/Raspbian.md +++ b/docs/install-instructions/Raspbian.md @@ -48,7 +48,8 @@ git clone https://github.com/DShield-ISC/dshield.git - run the installation script, in case you do have an earlier system, copy the files `/etc/dshield.ini` and `/etc/dshield.sslca` from that system to `/etc`; you will be able to reuse the data entered for that system. ``` cd dshield/bin -sudo ./install.sh +./install.sh +(older versions required sudo) ``` - if curious watch the debug log file in parallel to the installation: connect with an additional ssh session to the system and run (name of the log file will be printed out by the installation script): ``` From 6b8aabc222351ba68a98422965881e3764629db6 Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 15 Aug 2025 15:40:33 -0400 Subject: [PATCH 31/54] Update Ubuntu.md --- docs/install-instructions/Ubuntu.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index b513c6a2..5846d97e 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -23,8 +23,10 @@ Finally, clone the following Git repository and run the install script. Make sur ``` git clone https://github.com/DShield-ISC/dshield.git -sudo dshield/bin/install.sh +dshield/bin/install.sh sudo reboot ``` For additional details, see the global README.md file. + +Older versions required the "install.sh" script to run as "sudo". As of August 2025, only specific commands inside install.sh are run as sudo. From 8c523b3386f672c603bd0918d59626fc4f203f58 Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 15 Aug 2025 15:41:52 -0400 Subject: [PATCH 32/54] Update AWS.md --- docs/install-instructions/AWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/install-instructions/AWS.md b/docs/install-instructions/AWS.md index 31c2703c..33e63daf 100644 --- a/docs/install-instructions/AWS.md +++ b/docs/install-instructions/AWS.md @@ -80,8 +80,9 @@ mkdir install cd install git clone https://github.com/DShield-ISC/dshield.git cd dshield/bin -sudo ./install.sh +./install.sh ``` +(as of August 2025, install.sh should no longer be executed as root via sudo) Post-Install Notes ------------------- From dbfed421e3e1875414557aa59e53ab4e51fbf592 Mon Sep 17 00:00:00 2001 From: jslagrew <82918323+jslagrew@users.noreply.github.com> Date: Sat, 16 Aug 2025 06:57:14 -0500 Subject: [PATCH 33/54] Correct web honeypot local logging variables Also updated web honeypot log file to start with "webhoneypot" to assist with DShield-SIEM ingestion settings. --- bin/install.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 618e0e4e..63224c06 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -1079,11 +1079,11 @@ if [ "$piid" == "" ]; then else dlog "old piid ${piid}" fi -if [ "$record_local_responses" == "" ]; then - record_local_responses='false' +if [ "$enable_local_logs" == "" ]; then + enable_local_logs='false' fi -if [ "$local_response_path" == "" ]; then - local_response_path='/srv/log/local_response_{date}.json' +if [ "$local_logs_file" == "" ]; then + local_logs_file='/srv/log/webhoneypot_{date}.json' fi if [ "$submit_log_rate" == "" ]; then @@ -2112,8 +2112,8 @@ run "echo 'queue_trigger=$queue_trigger' >> ${DSHIELDINI}" run "echo 'web_log_limit=$web_log_limit' >> ${DSHIELDINI}" run "echo 'web_log_purgefactor=$web_log_purgefactor' >> ${DSHIELDINI}" run "echo 'debug=false' >> ${DSHIELDINI}" -run "echo 'local_response_path=$local_response_path' >> ${DSHIELDINI}" -run "echo 'record_local_responses=$record_local_responses' >> ${DSHIELDINI}" +run "echo 'local_logs_file=$local_logs_file' >> ${DSHIELDINI}" +run "echo 'enable_local_logs=$enable_local_logs' >> ${DSHIELDINI}" run "echo '# this section may disappear in future versions' >> ${DSHIELDINI}" run "echo '[iscagent]' >> ${DSHIELDINI}" database=$(quotespace "$database") From d97dc30d9b9c57fd85bdab6e56a765597caf56dc Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Sun, 17 Aug 2025 21:47:01 -0400 Subject: [PATCH 34/54] Update Ubuntu.md --- docs/install-instructions/Ubuntu.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index 5846d97e..102bb92d 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -1,8 +1,8 @@ -Installing on Ubuntu Server 22.04 LTS +Installing on Ubuntu Server 24.04 LTS ===================================== **Via ISO Image** -Install the default version Ubuntu Server 22.04 LTS. Don't select any additional packages when prompted. During installation, create a user called "dshield." +Install the default version Ubuntu Server 24.04 LTS. Don't select any additional packages when prompted. During installation, create a user called "dshield." **Deploying As VPS** From f550d1e07f298c68e672cb29c11dfb7668ab90fa Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 18 Aug 2025 08:16:37 -0400 Subject: [PATCH 35/54] fixing cron --- bin/install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index 618e0e4e..193c6195 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -733,7 +733,7 @@ if [ "$FAST" == "0" ]; then sudorun 'apt -y -q install python3-defusedxml' sudorun 'apt -y -q install python-babel-localedata python3-babel python3-markupsafe python3-tz' - for b in python3 python3-pip python3-requests python3-appdirs python3-attr python3-certifi authbind build-essential curl dialog gcc git jq libffi-dev libmariadb-dev-compat libmpc-dev libmpfr-dev libpython3-dev libssl-dev libswitch-perl libwww-perl net-tools python3-dev python3-minimal python3-requests python3-urllib3 python3-virtualenv rng-tools sqlite3 unzip wamerican zip libsnappy-dev virtualenv lsof iptables rsyslog stunnel python3-openssl python3-hamcrest python3-priority; do + for b in cron python3 python3-pip python3-requests python3-appdirs python3-attr python3-certifi authbind build-essential curl dialog gcc git jq libffi-dev libmariadb-dev-compat libmpc-dev libmpfr-dev libpython3-dev libssl-dev libswitch-perl libwww-perl net-tools python3-dev python3-minimal python3-requests python3-urllib3 python3-virtualenv rng-tools sqlite3 unzip wamerican zip libsnappy-dev virtualenv lsof iptables rsyslog stunnel python3-openssl python3-hamcrest python3-priority; do run "sudo apt -y -q install $b" if ! sudo dpkg -l $b >/dev/null 2>/dev/null; then outlog "ERROR I was unable to install the $b package via apt" @@ -746,6 +746,9 @@ if [ "$FAST" == "0" ]; then done fi + sudorun "systemctl start cron" + sudorun "systemctl enable cron" + if [ "$ID" == "amzn" ]; then outlog "Updating your Operating System" sudorun 'yum -q update -y' From 0fb48fbff24413965812e535397d53d3d4d702d4 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 18 Aug 2025 08:59:57 -0400 Subject: [PATCH 36/54] fixing cron file permissions --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index ffbabb6c..a4695d08 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2059,7 +2059,7 @@ fi echo "${offset1} ${offset2} * * * root cd ${DSHIELDDIR}; ./status.sh >/dev/null " >> "${TMPDIR}"/cron.dshield echo "0 10 * * * root find /srv/cowrie/var/log/cowrie -name 'cowrie.*' -ctime +7 -delete" >> "${TMPDIR}"/cron.dshield sudorun "cp ${TMPDIR}/cron.dshield /etc/cron.d/dshield" -sudorun "chmod 775 /etc/cron.d/dshield" +sudorun "chmod 755 /etc/cron.d/dshield" dsudorun 'cat /etc/cron.d/dshield' # From 5fa7c14757c4997aedd33b93a53a6efbfddb9d9f Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 13 Oct 2025 13:44:06 -0400 Subject: [PATCH 37/54] added latest debian trixie --- bin/install.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/install.sh b/bin/install.sh index a4695d08..33fe382a 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -592,6 +592,11 @@ if [ "$ID" == "debian" ] && [ "$VERSION_ID" == "12" ]; then distversion=r12 fi +if [ "$ID" == "debian" ] && [ "$VERSION_ID" == "13" ]; then + dist='apt' + distversion=r13 +fi + if [ "$ID" == "raspbian" ] && [ "$VERSION_ID" == "8" ]; then dist='apt' distversion=r8 @@ -2360,6 +2365,9 @@ sudo -u webhpot python3 -m zipapp ./isc_agent sudo -u webhpot find ./isc_agent -mindepth 1 -type d -exec rm -rf {} + sudo_copy "${progdir}"/../srv/web/web-honeypot.service /etc/systemd/system/web-honeypot.service 644 cd "${WEBHPOTDIR}" || exit +# disable old service in case it is still enabled +sudorun "systemctl disable isc-agent.service" +# enable new service sudorun "systemctl daemon-reload" sudorun "systemctl enable web-honeypot.service" From 700980b8aa846ba59a9d01d0432ff8aa7ace2401 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Sun, 26 Oct 2025 15:36:52 -0400 Subject: [PATCH 38/54] fixing skipvalue and removing python3-appdirs and adding quiet to cron --- bin/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 33fe382a..94e43533 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -738,7 +738,7 @@ if [ "$FAST" == "0" ]; then sudorun 'apt -y -q install python3-defusedxml' sudorun 'apt -y -q install python-babel-localedata python3-babel python3-markupsafe python3-tz' - for b in cron python3 python3-pip python3-requests python3-appdirs python3-attr python3-certifi authbind build-essential curl dialog gcc git jq libffi-dev libmariadb-dev-compat libmpc-dev libmpfr-dev libpython3-dev libssl-dev libswitch-perl libwww-perl net-tools python3-dev python3-minimal python3-requests python3-urllib3 python3-virtualenv rng-tools sqlite3 unzip wamerican zip libsnappy-dev virtualenv lsof iptables rsyslog stunnel python3-openssl python3-hamcrest python3-priority; do + for b in cron python3 python3-pip python3-requests python3-attr python3-certifi authbind build-essential curl dialog gcc git jq libffi-dev libmariadb-dev-compat libmpc-dev libmpfr-dev libpython3-dev libssl-dev libswitch-perl libwww-perl net-tools python3-dev python3-minimal python3-requests python3-urllib3 python3-virtualenv rng-tools sqlite3 unzip wamerican zip libsnappy-dev virtualenv lsof iptables rsyslog stunnel python3-openssl python3-hamcrest python3-priority; do run "sudo apt -y -q install $b" if ! sudo dpkg -l $b >/dev/null 2>/dev/null; then outlog "ERROR I was unable to install the $b package via apt" @@ -2046,7 +2046,7 @@ offset2=$(shuf -i0-23 -n1) echo "${offset1} ${offset2} * * * root ${DSHIELDDIR}/updatehoneypotip.sh" >>"${TMPDIR}"/cron.dshield offset1=$(shuf -i0-59 -n1) offset2=$(shuf -i0-23 -n1) -echo "${offset1} ${offset2} * * * root /usr/bin/journalctl --vacuum-time=7d" >>"${TMPDIR}"/cron.dshield +echo "${offset1} ${offset2} * * * root /usr/bin/journalctl --vacuum-time=7d --quiet" >>"${TMPDIR}"/cron.dshield # run status check 5 minutes before reboot From ddb089ae015360d2150b8a7ca35e6e22ff90f0b7 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 3 Nov 2025 14:52:55 -0500 Subject: [PATCH 39/54] adjusting for latest raspberry pi os --- bin/install.sh | 10 ++++++++-- bin/status.sh | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 94e43533..c6c4fb02 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -622,6 +622,12 @@ if [ "$ID" == "raspbian" ] && [ "$VERSION_ID" == "12" ]; then distversion=r12 fi +if [ "$ID" == "debian" ] && [ "$VERSION_ID" == "13" ]; then + dist='apt' + distversion=r13 +fi + + if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "20.04" ]; then dist='apt' distversion='u20' @@ -657,9 +663,9 @@ if [ "$dist" == "invalid" ]; then exit 9 fi -if [ "$ID" != "raspbian" ] && [ "$ID" != "opensuse" ] && [ "$ID" != "raspbian" ] && [ "$VERSION_ID" != "24.04" ] && [ "$VERSION_ID" != "22.04" ] && [ "$VERSION_ID" != "20.04" ]; then +if [ "$ID" != "raspbian" ] && [ "$ID" != "opensuse" ] && [ "$ID" != "raspbian" ] && [ "$VERSION_ID" != "24.04" ] && [ "$VERSION_ID" != "22.04" ] && [ "$VERSION_ID" != "20.04" ] && [ "$VERSION_ID" != "13" ] ; then outlog "ATTENTION: the latest versions of this script have been tested on:" - outlog " - Raspbian OS" + outlog " - Raspbian OS (up to trixie, release October 1st 2025)" outlog " - Ubuntu 20.04" outlog " - Ubuntu 22.04" outlog " - Ubuntu 24.04" diff --git a/bin/status.sh b/bin/status.sh index 2563d33b..bdaef07e 100755 --- a/bin/status.sh +++ b/bin/status.sh @@ -186,7 +186,7 @@ else fi if [ -f /var/tmp/dshield/skipvalue ]; then - skip=$(cat /var/tmp/dshield/skipvalue) + skip=$(cat /var/tmp/dshield/skipvalue | cut -f1 -d'.') if [ "$skip" == "" ]; then skip=1 rm /var/tmp/dshield/skipvalue From e12a904d845de14cc132b5637d82cc7cdaeac585 Mon Sep 17 00:00:00 2001 From: Jim Clausing Date: Mon, 3 Nov 2025 16:27:49 -0500 Subject: [PATCH 40/54] [20251103-000] fix 3.12 deprecation warning for datetime.utcnow() --- srv/dshield/fwlogparser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/srv/dshield/fwlogparser.py b/srv/dshield/fwlogparser.py index bc372b0d..2b03896d 100755 --- a/srv/dshield/fwlogparser.py +++ b/srv/dshield/fwlogparser.py @@ -11,7 +11,7 @@ from time import strptime from time import mktime from time import time -from datetime import datetime +from datetime import datetime, timezone from DShield import DshieldSubmit maxlines=100000; @@ -105,7 +105,10 @@ def checklock(lockfile): lastdir="/var/tmp/dshield/" config = "/etc/dshield.ini" startdate = 0 -now = datetime.utcnow() +if sys.version_info < (3, 12): + now = datetime.utcnow() +else: + now = datetime.now(timezone.utc) fieldmap = {'SRC': 'sip', 'DST': 'dip', 'PROTO': 'proto', 'TYPE': 'sport', 'CODE': 'dport', 'SPT': 'sport', 'DPT': 'dport'} protomap = {'ICMP': 1, 'IGMP': 2, 'TCP': 6, 'UDP': 17, 'ESP': 50, 'AH': 51, 'ICMPv6': 58} From eca4aaf3046f2e463af7d5e6da89f33387f086d9 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 4 Nov 2025 09:59:05 -0500 Subject: [PATCH 41/54] better exception handling for post data issue #318 --- srv/web/isc_agent/__main__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/srv/web/isc_agent/__main__.py b/srv/web/isc_agent/__main__.py index e8a83750..3e74d18f 100644 --- a/srv/web/isc_agent/__main__.py +++ b/srv/web/isc_agent/__main__.py @@ -80,7 +80,7 @@ def responder(self, response_id): resp = dict(isc_agent.responses.get(response_id)) if not resp: - logger.error(f"responder({response_id}) called and that response_id does not exist.") + logger.error("responder(%s) called and that response_id does not exist.", (response_id,)) #Before we call send_response we must set the new server string. headers = resp.get("headers", {}) @@ -145,7 +145,11 @@ def __str__(self): request = RequestShim(path, method, remote_addr, headers) content_length = int(request.headers.get('Content-Length',0)) - post_data = self.rfile.read(content_length).decode() + try: + post_data = self.rfile.read(content_length).decode() + except UnicodeDecodeError as e: + self.logger.exception("Post data unicode decode error for request from {remote_addr} Exception {e}") + post_data = '' best_score = -1 best_signature = None @@ -370,11 +374,11 @@ def handle_error(self, request, client_address): exc_type, exc_value, _ = sys.exc_info() # Expected errors: log as INFO to track without clutter if exc_type == ConnectionResetError: - logger.info(f"Connection reset by {client_address[0]}:{client_address[1]}") + logger.info("Connection reset by %s:%s", (client_address[0], client_address[1])) elif exc_type == BrokenPipeError: - logger.info(f"Broken pipe to {client_address[0]}:{client_address[1]}") + logger.info(f"Broken pipe to %s:%s", (client_address[0], client_address[1])) elif exc_type in (TimeoutError, socket.timeout): - logger.info(f"Timeout from {client_address[0]}:{client_address[1]}") + logger.info(f"Timeout from %s:%s", (client_address[0], client_address[1])) elif exc_type == OSError and exc_value.errno in (113, 111): logger.info(f"Network error (errno {exc_value.errno}) from {client_address[0]}:{client_address[1]}") elif exc_type == IncompleteRead: From cec8e49ee8ae79403537c1536cd1ead0b2deda3f Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Tue, 4 Nov 2025 10:47:44 -0500 Subject: [PATCH 42/54] fixing path of ini file and adding piid in case it is missing --- bin/updatehoneypotip.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/updatehoneypotip.sh b/bin/updatehoneypotip.sh index 31dcce07..1d458941 100755 --- a/bin/updatehoneypotip.sh +++ b/bin/updatehoneypotip.sh @@ -8,13 +8,17 @@ if [ ! "$userid" = "0" ]; then exit 9 fi -if [ ! -f /etc/dshield.ini ]; then - echo "missing /etc/dshield.ini file" +if [ ! -f /srv/dshield/etc/dshield.ini ]; then + echo "missing /srv/dshield/etc/dshield.ini file" exit 9 fi honeypotip=$(curl -s https://www4.dshield.org/api/myip?json | jq .ip | tr -d '"') if echo -n $honeypotip | egrep -q '^[0-9\.]+$'; then - sed -i "s/^honeypotip=.*/honeypotip=$honeypotip/" /etc/dshield.ini + sed -i "s/^honeypotip=.*/honeypotip=$honeypotip/" /srv/dshield/etc/dshield.ini + if ! grep -q '^piid=' dshield.ini; then + piid=$(openssl rand -hex 10) + sed -i "^apikey/a piid=$piid" /srv/dshield/etc/dshield.ini + fi else echo "Bad IP address" exit 9 From 8751d48d5f66f98344f87d7e4f0abfd7f31bdb53 Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Mon, 12 Jan 2026 11:12:08 -0500 Subject: [PATCH 43/54] Update Ubuntu version in installation instructions --- docs/install-instructions/Ubuntu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index 102bb92d..b4097563 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -6,7 +6,7 @@ Install the default version Ubuntu Server 24.04 LTS. Don't select any additional **Deploying As VPS** -Select the appropriate Ubuntu Server 22.04 package from your provider. You won't be prompted to install additional packages or add users when using this method. You'll need to manually add a user named "dshield" here as well: +Select the appropriate Ubuntu Server 24.04 package from your provider. You won't be prompted to install additional packages or add users when using this method. You'll need to manually add a user named "dshield" here as well: ```adduser --disabled-password --gecos "DShield Honeypot" dshield``` From fcde8aa00a6da48c9e7d6b051a6cbe562c73c9ab Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Mon, 12 Jan 2026 11:14:13 -0500 Subject: [PATCH 44/54] Update Ubuntu installation instructions Clarified user creation instructions and updated sudo usage details. --- docs/install-instructions/Ubuntu.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index b4097563..751315d3 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -6,7 +6,7 @@ Install the default version Ubuntu Server 24.04 LTS. Don't select any additional **Deploying As VPS** -Select the appropriate Ubuntu Server 24.04 package from your provider. You won't be prompted to install additional packages or add users when using this method. You'll need to manually add a user named "dshield" here as well: +Select the appropriate Ubuntu Server 24.04 package from your provider. You won't be prompted to install additional packages or add users when using this method. You'll need to manually add a user named "dshield" here as well (optionally): ```adduser --disabled-password --gecos "DShield Honeypot" dshield``` @@ -29,4 +29,4 @@ sudo reboot For additional details, see the global README.md file. -Older versions required the "install.sh" script to run as "sudo". As of August 2025, only specific commands inside install.sh are run as sudo. +Older versions required the "install.sh" script to run as "sudo". As of August 2025, only specific commands inside install.sh are run as root as needed. The script will prompt you for your password to use sudo as needed. From 1af2e9f0588ac9f0cbe24e2fe6d29bebcb2da9a8 Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Tue, 27 Jan 2026 16:01:22 -0500 Subject: [PATCH 45/54] Update README with OS version and clarity improvements Updated README to reflect changes in supported OS versions and improved clarity in several sections. --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 048ff869..51c01535 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a set of scripts to set up a DShield Sensor. Current design goals and prerequisites for using the automated installation procedure: - use of a __dedicated__ device (Raspberry Pi 3 or later, n100 mini PC or a virtual machine work fine) -- minimum of 1GB of RAM and 16GB of Disk (SD Card for Raspberry Pis). 4GB of RAM works better. Larger SD Cards (e.g. 64 GB) are recommended for longevity and to prevent logs from filling up the disk. +- minimum of 1GB of RAM and 16GB of Disk (SD Card for Raspberry Pis). 4GB of RAM works better. Larger SD Cards (e.g., 64 GB) are recommended for longevity and to prevent logs from filling up the disk. - current Raspberry Pi OS ("Lite" version will suffice) or current version of Ubuntu Linux - easy installation/configuration (and therefore not that configurable) - disposable (when something breaks (e.g., during upgrade): re-install from scratch) @@ -14,13 +14,13 @@ Current design goals and prerequisites for using the automated installation proc - support for IPv4 only (for the internal net) - one interface only (e.g,. eth0) -The current version is only tested on Raspberry Pi OS and Ubuntu 22.04 LTS Server, not on other distros, sorry. +The current version is only tested on Raspberry Pi OS (64-bit) and Ubuntu 24.04 LTS Server, not on other distros, sorry. If there is a need for other distros, "someone" has to check and maintain the installation script. ## Installation Reference the following files for OS-specific installation instructions: -[Raspbian](docs/install-instructions/Raspbian.md) (Recommended), +[Raspberry Pi OS](docs/install-instructions/Raspbian.md) (Recommended), [Ubuntu](docs/install-instructions/Ubuntu.md), [openSUSE](docs/install-instructions/openSUSE.md) and [AWS](docs/install-instructions/AWS.md) @@ -57,27 +57,27 @@ sudo bin/install.sh --update ``` The "--update" parameter will automatically use the existing configuration and not prompt the user for any configuration options. -Configuration parameters like your API Key will be retained. To edit the configuration, edit `/etc/dshield.ini`, to configure the firewall edit `/etc/network/iptables` (note: nat table is also used). +Configuration parameters like your API Key will be retained. To edit the configuration, edit `/etc/dshield.ini`, to configure the firewall, edit `/etc/network/iptables` (note: nat table is also used). -A new feature has been introduced, especially for automatic updates. At the end of the installation the install.sh script will search for the file /root/bin/postinstall.sh and execute its content, if it exists. If you need some extra changes in the newly installed files, this is the location to put them. This file NEEDS execute rights. +A new feature has been introduced, especially for automatic updates. At the end of the installation, the install.sh script will search for the file /root/bin/postinstall.sh and execute its content, if it exists. If you need some extra changes in the newly installed files, this is the location to put them. This file NEEDS execute rights. -Please make sure to keep special port and network configuration up to date (e.g., manually configure recently added telnet / web ports in firewall config), e.g., no-log config, no-honey config, ... unfortunately, this can't be done automagically as of now. If unsure, delete respective lines in `/etc/dshield.ini` and re-run the installation script. +Please make sure to keep special port and network configuration up to date (e.g., manually configure recently added telnet/web ports in firewall config), e.g., no-log config, no-honey config, ... unfortunately, this can't be done automagically as of now. If unsure, delete respective lines in `/etc/dshield.ini` and re-run the installation script. ## Hints -### How to place the DShield sensor / honeypot +### How to place the DShield sensor/honeypot -This DShield sensor and honeypot is meant to only analyze Internet related traffic, i.e. traffic which is issued from public IP addresses: -- this is due to how the DShield project works (collection of information about the current state of the Internet) +This DShield sensor and honeypot is meant to only analyze Internet-related traffic, i.e. traffic which is issued from public IP addresses: +- This is due to how the DShield project works (collection of information about the current state of the Internet) - only in this way information which is interesting for the Internet security community can be gathered - only in this way it can be ensured that no internal, non-public information is leaked from your Pi to DShield So you must place the Pi on a network where it can be exposed to the Internet (and won't be connected to from the inner networks, except for administrative tasks). For a maximum sensor benefit it is desirable that the Pi is exposed to the whole traffic the Internet routes to a public IP (and not only selected ports). -For SoHo users there is normally an option in the DSL or cable router to direct all traffic from the public IP the router is using (i.e. has been assigned by the ISP) to an internal IP. This has to be the Pi. This feature is named e.g. "exposed host", "DMZ" (here you may have to enable further configuration to ensure ___all___ traffic is being routed to the Pi's internal IP address and not only e.g. port 80). +For SoHo users, there is normally an option in the DSL or cable router to direct all traffic from the public IP the router is using (i.e. has been assigned by the ISP) to an internal IP. This has to be the Pi. This feature is named e.g. "exposed host", "DMZ" (here you may have to enable further configuration to ensure ___all___ traffic is being routed to the Pi's internal IP address and not only e.g. port 80). -For enterprises a protected DMZ would be a suitable place (protected: if the sensor / honeypot is hacked this incident is contained and doesn't affect other hosts in the DMZ). Please be aware that - if using static IPs - you're exposing attacks / scans to your IP to the DShield project and the community which can be tracked via whois to your company. +For enterprises, a protected DMZ would be a suitable place (protected: if the sensor/honeypot is hacked, this incident is contained and doesn't affect other hosts in the DMZ). Please be aware that - if using static IPs - you're exposing attacks / scans to your IP to the DShield project and the community which can be tracked via whois to your company. To test your set up you may use a public port scanner and point it to the router's public IP (which is then internally forwarded to the Pi). This port scan should be directly visible in `/var/log/dshield.log` and later in your online report accessible via your DShield account. Use only for quick and limited testing purposes, please, so that DShield data isn't falsified. @@ -86,7 +86,7 @@ To test your set up you may use a public port scanner and point it to the router - ESC: exit the form (Cancel) - cursor up / down: navigate through form / between input fields - cursor left / right: navigate within an input field -- TAB: switch between the input fields and "buttons" +- TAB: switch between the input fields and "buttons." - don't use Pos 1 / End ## TODOs @@ -120,7 +120,7 @@ python sitecopy.py http://www.yoursite.com web.py - do not need to run sitecopy however it will serve up a very basic page that can accept input and files. Todo: - Need to figure out how to serve up vulnerable pages - probably from the path -- SQL Injection - will likely use separate dorked database +- SQL Injection - will likely use a separate dorked database - Would like to integrate with cowrie for shell attacks - (BHAG) Any input appreciated - Please file a bug report / issue via github - thanks! From 152222b0e8ade5ec6b4953125a3a99d96f845c74 Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Wed, 8 Apr 2026 14:17:13 -0400 Subject: [PATCH 46/54] Update Ubuntu installation instructions Added a note about the end of support for Ubuntu Server 22.04 LTS and updated the upgrade command to include the '-y' flag. --- docs/install-instructions/Ubuntu.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index 751315d3..6c11fd06 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -1,6 +1,8 @@ Installing on Ubuntu Server 24.04 LTS ===================================== +Note: Ubuntu Server 22.04 LTS is no longer supported. + **Via ISO Image** Install the default version Ubuntu Server 24.04 LTS. Don't select any additional packages when prompted. During installation, create a user called "dshield." @@ -14,7 +16,7 @@ Select the appropriate Ubuntu Server 24.04 package from your provider. You won't Upgrade the base system and ensure git and openssh-server are already installed: ``` -sudo apt update && sudo apt upgrade +sudo apt update && sudo apt upgrade -y sudo apt install -y git openssh-server sudo reboot ``` From 5a49d8e9fe14b07ba2e0065f312ad94c4b2c32df Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Wed, 8 Apr 2026 15:23:23 -0400 Subject: [PATCH 47/54] Update Ubuntu.md --- docs/install-instructions/Ubuntu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index 6c11fd06..e982bab7 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -31,4 +31,4 @@ sudo reboot For additional details, see the global README.md file. -Older versions required the "install.sh" script to run as "sudo". As of August 2025, only specific commands inside install.sh are run as root as needed. The script will prompt you for your password to use sudo as needed. +Older versions required the "install.sh" script to run as "root". As of August 2025, only specific commands inside install.sh are run as root as needed. The script will prompt you for your password to use sudo as needed. From 3753a4d1d6166e8b0cca43048ab5cda6e47fb1d0 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Fri, 1 May 2026 09:29:42 -0400 Subject: [PATCH 48/54] adding ubuntu 26.04 support --- bin/install.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index c6c4fb02..c1ebb0af 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -15,11 +15,14 @@ # version 2025/08/05 -readonly myversion=98 +readonly myversion=99 # Major Changes (for details, see Github): # +# - V99 (Johannes) +# - removing support for Ubuntu 20/22, adding support for 26.04 +# # - V98 (Johannes) # - new web hpot (Mark Baggett) # - installer no longer requires root / better priv separation @@ -627,20 +630,14 @@ if [ "$ID" == "debian" ] && [ "$VERSION_ID" == "13" ]; then distversion=r13 fi - -if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "20.04" ]; then - dist='apt' - distversion='u20' -fi - -if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "22.04" ]; then +if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "24.04" ]; then dist='apt' - distversion='u22' + distversion='u24' fi -if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "24.04" ]; then +if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "26.04" ]; then dist='apt' - distversion='u24' + distversion='u26' fi if [ "$ID" == "amzn" ] && [ "$VERSION_ID" == "2" ]; then From 0f98b0e5f6dd962c75642f289852ba7bedca75f9 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Fri, 1 May 2026 09:30:27 -0400 Subject: [PATCH 49/54] fixing link for dshield.ini --- bin/updatehoneypotip.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/bin/updatehoneypotip.sh b/bin/updatehoneypotip.sh index 1d458941..2b03871b 100755 --- a/bin/updatehoneypotip.sh +++ b/bin/updatehoneypotip.sh @@ -1,4 +1,11 @@ #!/bin/bash + +# this script will update the honeypot's public IPv4 IP address +# it will also do some cleanup: +# - check if /etc/dshield.ini is a symlink +# - add a unique "Pi ID" to the dshield.ini file if it is not already present +# - fix permissions for dshield.ini + userid=$(id -u) if [ ! "$userid" = "0" ]; then echo "You have to run this script as root. eg." @@ -12,6 +19,13 @@ if [ ! -f /srv/dshield/etc/dshield.ini ]; then echo "missing /srv/dshield/etc/dshield.ini file" exit 9 fi + +if [ ! -L /etc/dshield.ini ]; then + ln -s /srv/dshield/etc/dshield.ini /etc/dshield.ini +fi + + + honeypotip=$(curl -s https://www4.dshield.org/api/myip?json | jq .ip | tr -d '"') if echo -n $honeypotip | egrep -q '^[0-9\.]+$'; then sed -i "s/^honeypotip=.*/honeypotip=$honeypotip/" /srv/dshield/etc/dshield.ini @@ -23,3 +37,8 @@ else echo "Bad IP address" exit 9 fi + +# TODO eventually the permissions should be 600, but to support legacy installs, +# leaving them at 644 for now +chown webhpot:webhpot /srv/dshield/etc/dshield.ini +chmod 644 /srv/dshield/etc/dshield.ini From 7b526dddf5bd5bdbb70b00bd72c1d9829663b06f Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 1 May 2026 09:47:06 -0400 Subject: [PATCH 50/54] Enhance Ubuntu installation instructions Updated installation instructions for Ubuntu to include adding the 'dshield' user to the sudo group and installing a text editor. --- docs/install-instructions/Ubuntu.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index e982bab7..130af580 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -10,7 +10,12 @@ Install the default version Ubuntu Server 24.04 LTS. Don't select any additional **Deploying As VPS** Select the appropriate Ubuntu Server 24.04 package from your provider. You won't be prompted to install additional packages or add users when using this method. You'll need to manually add a user named "dshield" here as well (optionally): -```adduser --disabled-password --gecos "DShield Honeypot" dshield``` +``` +sudo adduser --disabled-password --gecos "DShield Honeypot" dshield +sudo adduser dshield sudo +``` + +If you installed the "minimum server": Make sure to install the editor of your choice. (for example, "sudo apt install emacs-nox") **After Completing Installation** Upgrade the base system and ensure git and openssh-server are already installed: @@ -24,6 +29,7 @@ sudo reboot Finally, clone the following Git repository and run the install script. Make sure to retrieve your API key from either dshield.org or isc.sans.edu. ``` +sudo su - dshield git clone https://github.com/DShield-ISC/dshield.git dshield/bin/install.sh sudo reboot From 8cc97298916526578bff4517bee7a70ba5723ba8 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Fri, 1 May 2026 10:05:42 -0400 Subject: [PATCH 51/54] adding extra util install --- bin/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install.sh b/bin/install.sh index c1ebb0af..0768e840 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -741,7 +741,7 @@ if [ "$FAST" == "0" ]; then sudorun 'apt -y -q install python3-defusedxml' sudorun 'apt -y -q install python-babel-localedata python3-babel python3-markupsafe python3-tz' - for b in cron python3 python3-pip python3-requests python3-attr python3-certifi authbind build-essential curl dialog gcc git jq libffi-dev libmariadb-dev-compat libmpc-dev libmpfr-dev libpython3-dev libssl-dev libswitch-perl libwww-perl net-tools python3-dev python3-minimal python3-requests python3-urllib3 python3-virtualenv rng-tools sqlite3 unzip wamerican zip libsnappy-dev virtualenv lsof iptables rsyslog stunnel python3-openssl python3-hamcrest python3-priority; do + for b in cron python3 python3-pip python3-requests python3-attr python3-certifi authbind build-essential curl dialog gcc git jq libffi-dev libmariadb-dev-compat libmpc-dev libmpfr-dev libpython3-dev libssl-dev libswitch-perl libwww-perl net-tools python3-dev python3-minimal python3-requests python3-urllib3 python3-virtualenv rng-tools sqlite3 unzip wamerican zip libsnappy-dev virtualenv lsof iptables util-linux-extra rsyslog stunnel python3-openssl python3-hamcrest python3-priority; do run "sudo apt -y -q install $b" if ! sudo dpkg -l $b >/dev/null 2>/dev/null; then outlog "ERROR I was unable to install the $b package via apt" From 8fadd650fa4adcebcc0776813b9b775bf0c875db Mon Sep 17 00:00:00 2001 From: Johannes Ullrich Date: Fri, 1 May 2026 11:25:58 -0400 Subject: [PATCH 52/54] Revise Ubuntu installation instructions for LTS versions Updated installation instructions for Ubuntu to include support for version 26.04 LTS and clarified support status for older versions. --- docs/install-instructions/Ubuntu.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/install-instructions/Ubuntu.md b/docs/install-instructions/Ubuntu.md index 130af580..2f9a3f42 100644 --- a/docs/install-instructions/Ubuntu.md +++ b/docs/install-instructions/Ubuntu.md @@ -1,7 +1,7 @@ -Installing on Ubuntu Server 24.04 LTS -===================================== +Installing on Ubuntu Server 24.04 LTS and 26.04 LTS +=================================================== -Note: Ubuntu Server 22.04 LTS is no longer supported. +Note: Ubuntu server versions 22.04 LTS and older are no longer supported. **Via ISO Image** Install the default version Ubuntu Server 24.04 LTS. Don't select any additional packages when prompted. During installation, create a user called "dshield." @@ -15,7 +15,14 @@ sudo adduser --disabled-password --gecos "DShield Honeypot" dshield sudo adduser dshield sudo ``` -If you installed the "minimum server": Make sure to install the editor of your choice. (for example, "sudo apt install emacs-nox") +If you installed the "minimum server": Make sure to install the editor of your choice. (for example, "sudo apt install emacs-nox"). + +**Optional: sudo without passwords +The "adduser" command does not configure a password for the dshield user. However, the user must have the abilitiy to use sudo. Either add a password for the "dshield" account, or modify the /etc/sudoers file: + +``` + +``` **After Completing Installation** Upgrade the base system and ensure git and openssh-server are already installed: From 91c89120288227af0fa2f63cc43f573c5e03bef1 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 4 May 2026 10:56:27 -0400 Subject: [PATCH 53/54] updated ubuntu versions --- bin/install.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 0768e840..529e55bb 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -654,19 +654,17 @@ fi dlog "dist: ${dist}, distversion: ${distversion}" if [ "$dist" == "invalid" ]; then - outlog "You are not running a supported operating system. Right now, this script only works for Raspbian, Ubuntu 20.04/22.04/24.04, " + outlog "You are not running a supported operating system. Right now, this script only works for Raspbian, Ubuntu 24.04,26.04," outlog "openSUSE Tumbleweed and Amazon Linux AMI." outlog "Please ask info@dshield.org for help to add support for your OS. Include the /etc/os-release file." exit 9 fi -if [ "$ID" != "raspbian" ] && [ "$ID" != "opensuse" ] && [ "$ID" != "raspbian" ] && [ "$VERSION_ID" != "24.04" ] && [ "$VERSION_ID" != "22.04" ] && [ "$VERSION_ID" != "20.04" ] && [ "$VERSION_ID" != "13" ] ; then +if [ "$ID" != "raspbian" ] && [ "$ID" != "opensuse" ] && [ "$ID" != "raspbian" ] && [ "$VERSION_ID" != "24.04" ] && [ "$VERSION_ID" != "26.04" ] && [ "$VERSION_ID" != "13" ] ; then outlog "ATTENTION: the latest versions of this script have been tested on:" outlog " - Raspbian OS (up to trixie, release October 1st 2025)" - outlog " - Ubuntu 20.04" - outlog " - Ubuntu 22.04" - outlog " - Ubuntu 24.04" - outlog " - openSUSE Tumbleweed." + outlog " - Ubuntu 24.04" + outlog " - Ubuntu 26.04" outlog "It may or may not work with your distro. Feel free to test and contribute." outlog "Press ENTER to continue, CTRL+C to abort." read From 2f9e96dd9dfcbe040dc275f4bc978083bb3c37e1 Mon Sep 17 00:00:00 2001 From: "Johannes B. Ullrich" Date: Mon, 4 May 2026 10:58:19 -0400 Subject: [PATCH 54/54] fixing stale agents --- .github/workflows/isc-agent-action.yml | 15 ------------- .github/workflows/poetry-to-legacy.yml | 29 -------------------------- .github/workflows/snyk.yml | 23 -------------------- 3 files changed, 67 deletions(-) delete mode 100644 .github/workflows/isc-agent-action.yml delete mode 100644 .github/workflows/poetry-to-legacy.yml delete mode 100644 .github/workflows/snyk.yml diff --git a/.github/workflows/isc-agent-action.yml b/.github/workflows/isc-agent-action.yml deleted file mode 100644 index b898d854..00000000 --- a/.github/workflows/isc-agent-action.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: ISC Agent Code Check Action - -on: - pull_request: - branches: [ISC-1-isc-agent-beta] - -jobs: - code-check: - name: Code Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Code scan - run: make ops-scan - working-directory: ./srv/isc-agent diff --git a/.github/workflows/poetry-to-legacy.yml b/.github/workflows/poetry-to-legacy.yml deleted file mode 100644 index 0daad1c2..00000000 --- a/.github/workflows/poetry-to-legacy.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Convert isc-agent's Poetry file to requirements.txt - -on: - push: - branches: - - main - -jobs: - convert_poetry_to_requirements: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install python3-pip -y - pip install poetry - - name: Convert poetry file to requirements.txt - run: | - cd srv/isc-agent - poetry export -f requirements.txt -o requirements.txt --without-hashes - grep -v 'Windows' requirements.txt | cut -f1 -d '=' | sed 's/all-non-platform/all_non_platform/' > ../../bin/requirements.txt - - name: Commit changes - uses: EndBug/add-and-commit@v9 - with: - author_name: "GitHub Action" - author_email: "action@github.com" - message: "Converted pyproject.toml to requirements.txt" diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml deleted file mode 100644 index 7f8f8ec3..00000000 --- a/.github/workflows/snyk.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Snyk Static Application Security Testing - -on: push -jobs: - security: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - uses: snyk/actions/setup@master - - uses: actions/setup-go@v1 - with: - go-version: "1.13" - - run: - pip install -r bin/requirements.txt - - name: Snyk test - run: snyk test --all-projects --detection-depth=10 --skip-unresolved --sarif-file-output=snyk.sarif - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - - name: Upload result to GitHub Code Scanning - if: always() - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: snyk.sarif \ No newline at end of file