Bug#509292: rsyslog: random crashes with remote logging

External Since: Dec 20, 2008 Posts: 1

Package: rsyslog
Version: 3.18.5-1
Severity: critical
Justification: causes serious data loss

Hello,

I have noticed that rsyslog randomly crashes with remote logging
(client) enabled. This happens almost once a day and all log messages
are lost until rsyslog is started again. I don't know if this is related
to remote logging but all my other installations with no remote logging
work fine.

Normally there is nothing special in the log files when rsyslog crashes
and I have not been able to find out the cause. Today I noticed the
following:

*** syslog
*** glibc detected *** /usr/sbin/rsyslogd: malloc(): memory corruption (fast): 0x00007f633001d33f ***
======= Backtrace: =========
/lib/libc.so.6[0x7f63378c3948]
/lib/libc.so.6[0x7f63378c667f]
/lib/libc.so.6(__libc_malloc+0x98)[0x7f63378c7a78]
/usr/sbin/rsyslogd(rsCStrGetSzStr+0x5d)[0x414b1d]
/usr/sbin/rsyslogd(rsCStrConvSzStrAndDestruct+0x24)[0x414cc4]
/usr/sbin/rsyslogd(logmsg+0x50c)[0x40ba4c]
/usr/sbin/rsyslogd(printline+0xed)[0x40be8d]
/usr/sbin/rsyslogd(parseAndSubmitMessage+0x126)[0x40c046]
/usr/lib/rsyslog/imuxsock.so[0x7f633703b2d5]
/usr/lib/rsyslog/imuxsock.so[0x7f633703b4a7]
/usr/sbin/rsyslogd[0x416470]
/lib/libpthread.so.0[0x7f6337fb6fc7]
/lib/libc.so.6(clone+0x6d)[0x7f633791f5ad]
======= Memory map: ========
00400000-0043b000 r-xp 00000000 fd:02 49160 /usr/sbin/rsyslogd
0063a000-0063e000 rw-p 0003a000 fd:02 49160 /usr/sbin/rsyslogd
0063e000-0063f000 rw-p 0063e000 00:00 0
02058000-0211f000 rw-p 02058000 00:00 0 [heap]
408ed000-408ee000 ---p 408ed000 00:00 0
408ee000-410ee000 rw-p 408ee000 00:00 0
410ee000-410ef000 ---p 410ee000 00:00 0
410ef000-418ef000 rw-p 410ef000 00:00 0
41b14000-41b15000 ---p 41b14000 00:00 0
41b15000-42315000 rw-p 41b15000 00:00 0
42315000-42316000 ---p 42315000 00:00 0
42316000-42b16000 rw-p 42316000 00:00 0
7f6328000000-7f6328021000 rw-p 7f6328000000 00:00 0
7f6328021000-7f632c000000 ---p 7f6328021000 00:00 0
7f6330000000-7f6330021000 rw-p 7f6330000000 00:00 0
7f6330021000-7f6334000000 ---p 7f6330021000 00:00 0
7f6336c1c000-7f6336c32000 r-xp 00000000 fd:00 8226 /lib/libgcc_s.so.1
7f6336c32000-7f6336e32000 ---p 00016000 fd:00 8226 /lib/libgcc_s.so.1
7f6336e32000-7f6336e33000 rw-p 00016000 fd:00 8226 /lib/libgcc_s.so.1
7f6336e33000-7f6336e38000 r-xp 00000000 fd:02 73841 /usr/lib/rsyslog/imklog.so
7f6336e38000-7f6337038000 ---p 00005000 fd:02 73841 /usr/lib/rsyslog/imklog.so
7f6337038000-7f6337039000 rw-p 00005000 fd:02 73841 /usr/lib/rsyslog/imklog.so
7f6337039000-7f633703a000 rw-p 7f6337039000 00:00 0
7f633703a000-7f633703c000 r-xp 00000000 fd:02 73834 /usr/lib/rsyslog/imuxsock.so
7f633703c000-7f633723c000 ---p 00002000 fd:02 73834 /usr/lib/rsyslog/imuxsock.so
7f633723c000-7f633723d000 rw-p 00002000 fd:02 73834 /usr/lib/rsyslog/imuxsock.so
7f633723d000-7f633723f000 r-xp 00000000 fd:02 73833 /usr/lib/rsyslog/lmtcpclt.so
7f633723f000-7f633743e000 ---p 00002000 fd:02 73833 /usr/lib/rsyslog/lmtcpclt.so
7f633743e000-7f633743f000 rw-p 00001000 fd:02 73833 /usr/lib/rsyslog/lmtcpclt.so
7f633743f000-7f6337449000 r-xp 00000000 fd:00 8447 /lib/libnss_files-2.7.so
7f6337449000-7f6337649000 ---p 0000a000 fd:00 8447 /lib/libnss_files-2.7.so
7f6337649000-7f633764b000 rw-p 0000a000 fd:00 8447 /lib/libnss_files-2.7.so
7f633764b000-7f633764f000 r-xp 00000000 fd:02 73832 /usr/lib/rsyslog/lmnet.so
7f633764f000-7f633784f000 ---p 00004000 fd:02 73832 /usr/lib/rsyslog/lmnet.so
7f633784f000-7f6337850000 rw-p 00004000 fd:02 73832 /usr/lib/rsyslog/lmnet.so
7f6337850000-7f633799a000 r-xp 00000000 fd:00 8459 /lib/libc-2.7.so
7f633799a000-7f6337b99000 ---p 0014a000 fd:00 8459 /lib/libc-2.7.so
7f6337b99000-7f6337b9c000 r--p 00149000 fd:00 8459 /lib/libc-2.7.so
7f6337b9c000-7f6337b9e000 rw-p 0014c000 fd:00 8459 /lib/libc-2.7.so
7f6337b9e000-7f6337ba3000 rw-p 7f6337b9e000 00:00 0
7f6337ba3000-7f6337bab000 r-xp 00000000 fd:00 8498 /lib/librt-2.7.so
7f6337bab000-7f6337daa000 ---p 00008000 fd:00 8498 /lib/librt-2.7.so
7f6337daa000-7f6337dac000 rw-p 00007000 fd:00 8498 /lib/librt-2.7.so
7f6337dac000-7f6337dae000 r-xp 00000000 fd:00 8496 /lib/libdl-2.7.so
7f6337dae000-7f6337fae000 ---p 00002000 fd:00 8496 /lib/libdl-2.7.so
7f6337fae000-7f6337fb0000 rw-p 00002000 fd:00 8496 /lib/libdl-2.7.so
7f6337fb0000-7f6337fc6000 r-xp 00000000 fd:00 8442 /lib/libpthread-2.7.so
7f6337fc6000-7f63381c6000 ---p 00016000 fd:00 8442 /lib/libpthread-2.7.so
7f63381c6000-7f63381c8000 rw-p 00016000 fd:00 8442 /lib/libpthread-2.7.so
7f63381c8000-7f63381cc000 rw-p 7f63381c8000 00:00 0
7f63381cc000-7f63381e2000 r-xp 00000000 fd:02 58672 /usr/lib/libz.so.1.2.3.3
7f63381e2000-7f63383e2000 ---p 00016000 fd:02 58672 /usr/lib/libz.so.1.2.3.3
7f63383e2000-7f63383e3000 rw-p 00016000 fd:02 58672 /usr/lib/libz.so.1.2.3.3
7f63383e3000-7f63383ff000 r-xp 00000000 fd:00 8495 /lib/ld-2.7.so
7f63385f3000-7f63385f6000 rw-p 7f63385f3000 00:00 0
7f63385fa000-7f63385fe000 rw-p 7f63385fa000 00:00 0
7f63385fe000-7f6338600000 rw-p 0001b000 fd:00 8495 /lib/ld-2.7.so
7fff405ea000-7fff405ff000 rw-p 7ffffffea000 00:00 0 [stack]
7fff405ff000-7fff40600000 r-xp 7fff405ff000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]

Here's my rsyslog.conf and remote logging configuration in
/etc/rsyslog.d/remote-logging.conf:

*** rsyslog.conf
# /etc/rsyslog.conf Configuration file for rsyslog v3.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html

#################
#### MODULES ####
#################

$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support (previously done by rklogd)
#$ModLoad immark # provides --MARK-- message capability

# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514

###########################
#### GLOBAL DIRECTIVES ####
###########################

#
# Use default timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

###############
#### RULES ####
###############

#
# First some standard log files. Log by facility.
#
auth,authpriv.* /var/log/auth.log
*.*;auth,authpriv.none -/var/log/syslog
#cron.* /var/log/cron.log
daemon.* -/var/log/daemon.log
kern.* -/var/log/kern.log
lpr.* -/var/log/lpr.log
mail.* -/var/log/mail.log
user.* -/var/log/user.log

#
# Logging for the mail system. Split it up so that
# it is easy to write scripts to parse these files.
#
mail.info -/var/log/mail.info
mail.warn -/var/log/mail.warn
mail.err /var/log/mail.err

#
# Logging for INN news system.
#
news.crit /var/log/news/news.crit
news.err /var/log/news/news.err
news.notice -/var/log/news/news.notice

#
# Some "catch-all" log files.
#
*.=debug;\
auth,authpriv.none;\
news.none;mail.none -/var/log/debug
*.=info;*.=notice;*.=warn;\
auth,authpriv.none;\
cron,daemon.none;\
mail,news.none -/var/log/messages

#
# Emergencies are sent to everybody logged in.
#
*.emerg *

#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
# news.=crit;news.=err;news.=notice;\
# *.=debug;*.=info;\
# *.=notice;*.=warn /dev/tty8

# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
# you must invoke `xconsole' with the `-file' option:
#
# $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
# busy site..
#
daemon.*;mail.*;\
news.err;\
*.=debug;*.=info;\
*.=notice;*.=warn |/dev/xconsole

*** remote-logging.conf
$WorkDirectory /var/log/work # default location for work (spool) files

$ActionQueueType LinkedList # use asynchronous processing
$ActionQueueFileName srvrfwd # set file name, also enables disk mode
$ActionResumeRetryCount -1 # infinite retries on insert failure
$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
*.* @@hostname.removed:514

-- System Information:
Debian Release: 5.0
APT prefers testing
APT policy: (500, 'testing')
Architecture: amd64 (x86_64)

Kernel: Linux 2.6.26-1-amd64 (SMP w/4 CPU cores)
Locale: LANG=en_US.UTF-8, LC_CTYPE=fi_FI.UTF-8 (charmap=UTF- Cool

Shell: /bin/sh linked to /bin/bash

Versions of packages rsyslog depends on:
ii libc6 2.7-16 GNU C Library: Shared libraries
ii lsb-base 3.2-20 Linux Standard Base 3.2 init scrip
ii zlib1g 1:1.2.3.3.dfsg-12 compression library - runtime

Versions of packages rsyslog recommends:
ii logrotate 3.7.1-5 Log rotation utility

Versions of packages rsyslog suggests:
pn rsyslog-doc <none> (no description available)
pn rsyslog-mysql | rsyslog-pgsql <none> (no description available)

-- no debconf information

--
To UNSUBSCRIBE, email to debian-bugs-rc-REQUEST DeleteThis @lists.debian.org
with a subject of "unsubscribe". Trouble? Contact listmaster DeleteThis @lists.debian.org

External Since: Dec 23, 2008 Posts: 3

Michael Biebl <biebl.DeleteThis@debian.org> wrote:
> Could you try to run rsyslog in debug mode:
> rsyslogd -c 3 -d

I have been running rsyslog in debug mode for a couple of days now
without problems. But... this morning my remote logging server had
crashed which has never happened before. Server has the latest version
(3.18.6-2) of rsyslog. Again there was nothing in log files. I will
try to run the server in debug mode also.

> Do you still get the crashes, if you comment out the $Action* and $WorkDirectory
> directives?

I have not yet tested this. I will try to get some debugging
information in the debug mode first.

Regards,
Juha

--
To UNSUBSCRIBE, email to debian-bugs-rc-REQUEST.DeleteThis@lists.debian.org
with a subject of "unsubscribe". Trouble? Contact listmaster.DeleteThis@lists.debian.org