Tools for parsing nginx/apache access logs for malicious traffic

Question

What are some tools for monitoring our access logs for malicious traffic? Paid or open source, it doesn't really matter.

We're hosting a number of sites on both apache and nginx. Some of these sites got infected with malware and we have a hard time finding out the initial successful vector of attack.

I can find some tools that can act on malicious requests (fail2ban for example) but I have a hard time identifying tools which enable us to manually review traffic and identify potential malicious requests.

Answer 1

this isn't a tool exactly, but below is a small python script that I run in Jupyter for same purpose. It uses the lars library for parsing nginx logs. You can use it to browse/query your logs and identify what you would consider malicious/suspicious. Then see where it's coming from in case you wish to ban. The output looks like

import httpx
import gzip
import lars.dns
import pandas as pd
import glob
import os

from pathlib import Path
from lars import csv
from lars.apache import ApacheSource, COMBINED


localdir = 'path to your nginx logs'
localpath = Path(localdir)

def decompress(infile, tofile):
    with open(infile, 'rb') as inf, open(tofile, 'w', encoding='utf8') as tof:
        decom_str = gzip.decompress(inf.read()).decode('utf-8')
        tof.write(decom_str)

# Change this if your access logs are named differently
all_access_gz = list(localpath.glob('access.log.*.gz'))
all_access_gz = [x.as_posix() for x in all_access_gz]

# Unzip them all
for f in all_access_gz:
    decompress(f, f.removesuffix('.gz'))

# Delete the .gz files
for f in all_access_gz:
    os.remove(f)


access_logs = list(localpath.glob('*access.*'))
access_logs = [x.as_posix() for x in access_logs]

access_csvfile = localdir + '/all_accss.csv'
with open(access_csvfile, 'wb') as outfile:
    for f in access_logs:
        with open(f) as infile:
            with ApacheSource(infile, log_format=COMBINED) as source, csv.CSVTarget(outfile) as target:
                for row in source:
                    target.write(row)

cols = ['remote_ip', 'dash_empty','remote_user', 'date_time', 'request', 'status', 'bytes_sent', 'http_referer', 'user_agent', 'gzip_ratio']
ndf = pd.read_csv(access_csvfile, names = cols, index_col = False )

# Keep only columns you need
ndf = ndf[['remote_ip', 'date_time', 'request', 'bytes_sent', 'status']]
ndf['date_time'] = ndf['date_time'].astype('datetime64[ns]')
ndf.head()


# Put your own "suspicious thing to look for" here, this is just one example:
hacker = ndf[ndf['request'].str.contains("php", case = False)]

headers = {'Accept': 'application/json'}
url_base = 'https://rdap.arin.net/registry/ip/'

ncc = []
addr = []

# Look up the ncc
with httpx.Client() as c:
    for idx, row in hacker.iterrows():
        url = url_base + row['remote_ip']
        res = c.get(url, headers=headers)
        smth = res.json().get('entities')[0].get('vcardArray')[1]
        n = smth[1][3]
        a = smth[2][1].get('label')
        a = a.replace('\n', ',')
        ncc.append(n)
        addr.append(a)
hacker['ncc'] = ncc
hacker['addr'] = addr