Date: 20140207
Auth: Jin设置一个LOG收集系统
1. 收集原生（不解析，不压缩）的业务日志和WEB日志（NGINX,PHP）2. 提供给开发，测试直接阅读和下载需求分析
原生日志，所以不需要其他程序介入，需要收集和阅读下载，具体分析开发测试人员拿到日子自行处理1、收集可以通过FTP采集2、阅读和下载可以通过WEB形式3、收集周期可以通过crontab控制脚本实现, 初步设置为每小时4、核心是收集日子的脚本，考虑到扩展和重复使用，我使用python编写d步骤
一、搭建FTP服务器1.install
yum install pure-ftpd
2.config
mkdir /data/ftproot/logs
chown webroot.webroot /data/ftproot/logs
vim /etc/pure-ftpd/pure-ftpd.conf
注意流量参数
Bind 10.0.0.221,21ChrootEveryone yesBrokenClientsCompatibility yesDaemonize yesMaxClientsPerIP 20VerboseLog yesDisplayDotFiles noAnonymousOnly noNoAnonymous yesSyslogFacility noneDontResolve yesMaxIdleTime 15LimitRecursion 10000 8AnonymousCanCreateDirs noMaxLoad 4PassivePortRange 45000 50000#AnonymousRatio 1 10#UserRatio 1 10AntiWarez yes#AnonymousBandwidth 200UserBandwidth 8 Umask 133:02MinUID 100 AllowUserFXP no AllowAnonymousFXP no ProhibitDotFilesWrite no ProhibitDotFilesRead no AutoRename yesAnonymousCantUpload yes AltLog clf:/var/log/pureftpd.logPureDB /etc/pure-ftpd/pureftpd.pdbMaxDiskUsage 99CreateHomeDir noCustomerProof yes
3.start ftp seriver
/etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]Starting pure-ftpd: [ OK ]chkconfig pure-ftpd on
4.add account
# pure-pw useradd logsftp -u 9999 -g 9999 -d /data/ftproot/logs
Password: Enter it again:# pure-pw mkdb
# /etc/init.d/pure-ftpd restartStopping pure-ftpd: [ OK ]Starting pure-ftpd: [ OK ]5.test
on opensuse desktop test
# zypper install lftp
lftp 10.0.0.221
lftp 10.0.0.221:~> user logsftpPassword: lftp logsftp@10.0.0.221:~> ls ls: Login failed: 530 ��֤ʧ�ܣ���Ǹ lftp logsftp@10.0.0.221:~>二、搭建WEB服务器
DOCMENTROOT为server { listen 80; server_name logs.test.com; root /data/ftproot/logs/; location / { index index.htm index.html; autoindex on; autoindex_localtime on; auth_basic "logs access"; auth_basic_user_file log-auth.conf; error_log off; access_log /var/log/nginx/log.test.com-access.log; #access_log off; }} 注意点：
autoindex_localtime on; 参考：http://blog.chinaunix.net/uid-26719405-id-3508444.html autoindex_localtime on; 默认为off，显示的文件时间为GMT时间。 改为on后，显示的文件时间为文件的服务器时间
三、代码编写
无LOG版本
加入计划任务
5 * * * * /home/gbin/logcollect.py >> /home/gbin/logcollect.log 2>&1
计划任务报错 登录用户问题
删除userName = os.getlogin()
- 配置文件logcollect.ini
[global]ip = 127.0.0.1username = logsftppassword = pwdmode = 0[applogs]dms-log = /home/dms/logs/service.logstore-logdir = /home/store/logs/stock-logdir = /home/stock/logs/[weblogs]nginxlog = /var/log/nginx.logphplog = /var/log/php-fpm/php-fpm.log
- 函数和对象：gbopt.py
#!/usr/bin/env python # coding=utf8 ''' Created on 2014-03-05 @author: Jin ''' __version__ = "0.0.1#date:2014-03-05" __all__ = ['readConfig', 'ftpPutGb', 'OptFile' ] import os import sys import ConfigParserimport ftplibimport timeimport socketfrom ftplib import FTPfrom time import strftime as printtimefrom time import sleep as waitprogName = 'logcollect'userName = os.getlogin()hostName = socket.gethostname()pwdDir = os.getcwd()workDir = os.path.dirname(sys.argv[0])configFile = progName+'.ini'todayDate = time.strftime("%Y%m%d", time.localtime())if workDir == '' or workDir == '.': progDir = pwdDirelif workDir.startswith('/'): progDir = workDirelif workDir.startswith('./'): progDir = pwdDir+workDir.lstrip('.')else: progDir = pwdDir+'/'+workDirconfPath = progDir+'/'+configFilelogPath = progDir+'/'+progName+'.log'def readConfig(filename='',section=''): '''read ini config return dict''' Call = 'Call '+sys._getframe().f_code.co_name if not os.path.isfile(filename): print "%s: Error, File %s is not exists,Please check it!" % (Call,filename) sys.exit(10) else: try: Config = ConfigParser.ConfigParser() Config.read(filename) sections = Config.sections() configDict = {} for conf in Config.items(section): ckey = conf[0] cvalue = conf[1] configDict[ckey] = cvalue except ConfigParser.ParsingError,e: print "%s: Read<%s> Section<%s> Parsing Error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n'))) sys.exit(11) except Exception, e: print "%s: Read<%s> Section<%s> Exception error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n'))) sys.exit(12) else: return configDictdef ftpPutGb(filename,inittime,server=None,username=None,password=None,localdir=None,remotedir='testdir',debuglevel=0,model=1,retrytime=10,timeout=120): """Put file to FTP server,use local path exitcode introduce: timeout | 99 connect failed | 21 user or passwd error | 22 create dir fails | 23 change dir fails | 24 read file fails | 25 transfer file fails | 26 any ftp transfer fails | 27 server or username or password not exist | 28 call ftpPutGb('/home/jin/code/python/tt.ini','127.0.0.1','logsftp','passwd',remotedir='testdir1/testdir2/testdir3',model=0)""" Call = 'Call '+sys._getframe().f_code.co_name if not os.path.isfile(filename): print "Notice: %s is not exists!" % filename return False if not localdir: localdir=os.path.dirname(filename) filename=os.path.basename(filename) uploadfile='STOR '+filename ftp=FTP() bufsize=1024 os.chdir(localdir) ftp.set_debuglevel(debuglevel) remotedirList=remotedir.split('/') appName=remotedirList[0] machineName=remotedirList[1] dayName=remotedirList[2] if server and username and password: print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) Begin." % (username,filename,server,remotedir) try: ftp.connect(server) except socket.error: waittime=inittime wait(retrytime) waittime+=retrytime if waittime==timeout: print "%s: Put %s to FTP(%s) timeout(%s),exit!" % (Call,filename,server,timeout) sys.exit(20) print "Notice: Retry Put %s to FTP(%s) again after %s s" % (filename,server,retrytime) ftpPutGb(filename,waittime,server,username,password,localdir,remotedir,debuglevel,model,retrytime,timeout) except ftplib.error_perm,e: print "Error: Connect to FTP(%s) failed with<%s>" % (server,e) sys.exit(21) else: try: ftp.login(username,password) ftp.set_pasv(model) except ftplib.error_perm,e: print "Error: User(%s) Login FTP(%s) failed with<%s>" % (username,server,e) sys.exit(22) else: try: try: ftp.mkd(appName) except ftplib.error_perm,e: pass finally: ftp.cwd(appName) try: ftp.mkd(machineName) except ftplib.error_perm,e: pass finally: ftp.cwd(machineName) try: ftp.mkd(dayName) except ftplib.error_perm,e: pass finally: ftp.cwd(dayName) try: file_handler = open(filename,'rb') except ftplib.error_perm,e: print "Error: User(%s) on FTP(%s) Open file(%s) failed with %s!" % (username,server,filename,e) sys.exit(25) else: try: ftp.storbinary(uploadfile,file_handler,bufsize) except ftplib.error_perm,e: print "Error: User(%s) on FTP(%s) storbinary file(%s) failed with %s!" % (username,server,filename,e) sys.exit(26) except Exception, e: print "Error: User(%s) Put file(%s) to FTP(%s:%s) failed exit with %s!" % (username,filename,server,remotedir,e) sys.exit(27) else: print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) successful." % (username,filename,server,remotedir) return True finally: ftp.set_debuglevel(0) ftp.close() else: print "Error: server(%s) or username(%s) or password(%s) is not exist!" % (server,username,password) sys.exit(28)class OptFile(file): '''log file object''' def __init__(self,path,remotedir): self.filePath = path self.remotedir = remotedir globalDict = readConfig(confPath,'global') self.ftpIp = globalDict['ip'] self.ftpUser = globalDict['username'] self.ftpPass = globalDict['password'] self.ftpMode = globalDict['mode'] file.__init__ def put(self): ftpPutGb(self.filePath,0,self.ftpIp,self.ftpUser,self.ftpPass,remotedir=self.remotedir,model=self.ftpMode)def main(): passif __name__ == '__main__': main()
- 调用脚本：logcollect.py
#!/usr/bin/env python# coding=utf8'''Created on 2014-03-05@author: Jin'''import osimport sysimport gboptdef collectLogs(logtype): applogsDict=gbopt.readConfig(gbopt.confPath,logtype) for l in applogsDict: appName=l.split('-')[0] hostName=gbopt.hostName dayName=gbopt.todayDate remoteDir=appName+'/'+hostName+'/'+dayName if l.endswith('dir'): locallogPath=applogsDict[l]+gbopt.todayDate+'.log' else: locallogPath=applogsDict[l] logfile=gbopt.OptFile(locallogPath,remoteDir) logfile.put()def main(): for i in ['applogs','weblogs']: collectLogs(i)if __name__ == '__main__': main()