我的日誌分析之道 | 簡單的Web日誌分析腳本

dt = datetime.now()
date = str(dt.date())
loglist = [] #
iplist = [] # ip統計
urllist = [] # url統計列表
needlist = [] # 需要統計的
errorlist = [] # 格式錯誤的列表
ipdict,urldict = {},{}
rizhi = str(input("請輸入要分析的日誌文件名"))
def find_log():
print(">>>>>>>開始解析日誌")
with open(rizhi,"r",encoding="UTF-8",errors="ignore") as f:
#loglist = f.readlines()
for i in f.readlines(): #
if i[0] != "#":
b = re.split(" ",i)
iplist.append(b[10])
urllist.append(b[6])
try:
needlist.append([b[10],b[1],b[5],b[6],b[15]])
except:
errorlist.append(i)
print(">>>>>>>日誌解析完畢")

def count(iplist,urllist): #統計ip url訪問量函數
print(">>>>>>>開始分析url與ip訪問量")
global ipdict,urldict
for i in set(iplist):
ipdict[i] = iplist.count(i)
for i in set(urllist):
urldict[i] = urllist.count(i)
ipdict = sorted(ipdict.items(),key=lambda d: d[1], reverse=True)
urldict = sorted(urldict.items(),key=lambda d: d[1], reverse=True)
print(type(urldict))
iplist = list(ipdict)
urllist = list(urldict)
ipdict,urldict = {},{}
print(">>>>>url與ip分析完畢.......")
return [iplist,urllist]

def save_count():
print(">>>>>>>正在保存分析結果")
ipname = "ip-"+date+".txt"
urlname = "url-"+date+".txt"
with open(ipname,"w") as f:
for i in iplist:
f.write(str(list(i))+"
")
with open(urlname,"w") as f:
for i in urllist:
f.write(str(list(i))+"
")
print(">>>>>>>分析結果保存完畢")
find_log()
[iplist,urllist] = count(iplist,urllist)
save_count()

sqllist,xsslist,senlist = [],[],[]
otherurl,xssip,sqlip,senip = [],[],[],[]
feifa = []
def find_attack(needlist):
print(">>>>>>>開始檢測攻擊")
sql = r"product.php|preg_w+|execute|echo|print|print_r|var_dump|(fp)open|^eval$|file_get_contents|include|require|require_once|shell_exec|phpinfo|system|passthru|(?:define|base64_decode(|groups+by.+(|%20or%20|%20and%20|sleep|delay|nvarchar|exec|union|^select$|version|insert|information_schema|chr(|concat|%bf|sleep((s*)(d*)(s*))|current|having|database"
xss = r"alert|^script$|<|>|%3E|%3c|>|u003c|u003e|&#x"
sen = r".{2,}|%2e{2,}|%252e{2,}|%uff0e{2,}0x2e{2,}|./|{FILE}|%00+|json|.shtml|.pl|.sh|.do|.action|zabbix|phpinfo|/var/|/opt/|/local/|/etc|/apache/|.log|invest|.xml|apple-touch-icon-152x152|.zip|.rar|.asp|.php|.bak|.tar.gz|phpmyadmin|admin|.exe|.7z|.zip|attachments|upimg|uploadfiles|templets|template|data|forumdata|includes|cache|jmxinvokerservlet|vhost|bbs|host|wwwroot|site|root|hytop|flashfxp|bak|old|mdb|sql|backup|^java$|class"

for i in needlist:
if i[2] == "POST" or i[2] == "HEAD" or i[2] == "GET":
response = re.findall(sql,i[3],re.I)
if response == []:
responsexss = re.findall(xss,i[3],re.I)
if responsexss == []:
responsesen = re.findall(sen,i[3],re.I)
if responsesen == []:
otherurl.append(i)
else:
senlist.append(i)
senip.append(i[0])
print(responsesen)
print("檢測出敏感目錄掃描")
print(i)
else:
xsslist.append(i)
xssip.append(i[0])
print(responsexss)
print("檢測出xss攻擊")
print(i)
else:
sqllist.append(i)
sqlip.append(i[0])
print(responsexss)
print("檢測出sql攻擊")
print(i)
else:
feifa.append(i[0])
print("非法請求:"+str(len(feifa))+"次"+str(len(list(set(feifa))))+"個ip")
print(">>>>>>>攻擊檢測完畢")
return [xssip,sqlip,senip,sqllist,xsslist,senlist,otherurl]

def url_open(ip):
url = "http://www.ip138.com/ips138.asp?ip="+ip
response = urllib.request.urlopen(url)
html = response.read().decode("gb2312")
return html
def find_ip(html):
a = r"本站數據.{20,}

"
p = re.compile(a,re.I)
response = re.findall(p,html)
for i in response:
b = i
response = re.split(r"