<div id="readability-page-1" class="page"><div>import&nbsp;csvimport&nbsp;loggingimport&nbsp;osimport&nbsp;sysfrom&nbsp;urllib.parse&nbsp;import&nbsp;unquote_plusfrom&nbsp;Utilities&nbsp;import&nbsp;KennyLogginsROOT_DIR&nbsp;=&nbsp;os.path.abspath(os.path.dirname(__file__))DATA_FILE&nbsp;=&nbsp;os.path.abspath(os.path.join(ROOT_DIR,&nbsp;'uap-core',&nbsp;'regexes_edit.yaml'))os.environ['UA_PARSER_YAML']&nbsp;=&nbsp;DATA_FILEfrom&nbsp;ua_parser&nbsp;import&nbsp;user_agent_parser#&nbsp;The&nbsp;following&nbsp;log&nbsp;levels&nbsp;are&nbsp;available:#&nbsp;Debug&nbsp;(NOISY!!!):#&nbsp;LOG_LEVEL&nbsp;=&nbsp;logging.DEBUG#&nbsp;Info:#&nbsp;LOG_LEVEL&nbsp;=&nbsp;logging.INFO#&nbsp;Error:LOG_LEVEL&nbsp;=&nbsp;logging.ERRORLOG_FILENAME&nbsp;=&nbsp;'TA-user-agents'kl&nbsp;=&nbsp;KennyLoggins()logger&nbsp;=&nbsp;kl.get_logger(app_name="TA-user-agents",&nbsp;file_name=LOG_FILENAME,&nbsp;log_level=logging.INFO)#&nbsp;Main&nbsp;routine&nbsp;-&nbsp;basically&nbsp;it's&nbsp;the&nbsp;standard&nbsp;python&nbsp;recipe&nbsp;for&nbsp;handling#&nbsp;Splunk&nbsp;lookups#if&nbsp;__name__&nbsp;==&nbsp;'__main__':&nbsp;&nbsp;&nbsp;&nbsp;r&nbsp;=&nbsp;csv.reader(sys.stdin)&nbsp;&nbsp;&nbsp;&nbsp;w&nbsp;=&nbsp;csv.writer(sys.stdout)&nbsp;&nbsp;&nbsp;&nbsp;have_header:&nbsp;bool&nbsp;=&nbsp;False&nbsp;&nbsp;&nbsp;&nbsp;header&nbsp;=&nbsp;[]&nbsp;&nbsp;&nbsp;&nbsp;idx&nbsp;=&nbsp;-1for&nbsp;row&nbsp;in&nbsp;r:if&nbsp;not&nbsp;have_header:&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;header&nbsp;=&nbsp;row&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('fields&nbsp;found:&nbsp;%s'&nbsp;%&nbsp;header)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;have_header&nbsp;=&nbsp;True&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;z&nbsp;=&nbsp;0for&nbsp;h&nbsp;in&nbsp;row:if&nbsp;h&nbsp;==&nbsp;"http_user_agent":&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;idx&nbsp;=&nbsp;z&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;z&nbsp;=&nbsp;z&nbsp;+&nbsp;1&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;w.writerow(row)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;continue#&nbsp;We&nbsp;only&nbsp;care&nbsp;about&nbsp;the&nbsp;cs_user_agent&nbsp;field&nbsp;-&nbsp;everything&nbsp;else&nbsp;is&nbsp;filled&nbsp;in&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;http_user_agent&nbsp;=&nbsp;row[idx]&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;useragent&nbsp;=&nbsp;unquote_plus(http_user_agent)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('found&nbsp;useragent&nbsp;%s'&nbsp;%&nbsp;http_user_agent)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('sending&nbsp;to&nbsp;ua-parser')&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;results&nbsp;=&nbsp;[]&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;try:#results&nbsp;=&nbsp;user_agent_parser.Parse(http_user_agent)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;results&nbsp;=&nbsp;user_agent_parser.ParseUserAgent(http_user_agent)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;except&nbsp;Exception&nbsp;as&nbsp;err:&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.error(err)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;continue&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('back&nbsp;from&nbsp;ua-parser')#&nbsp;create&nbsp;our&nbsp;results&nbsp;for&nbsp;Splunk#&nbsp;using&nbsp;the&nbsp;full&nbsp;results&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;forSplunk&nbsp;=&nbsp;{'ua_os_family':&nbsp;'unknown','ua_os_major':&nbsp;'unknown','ua_os_minor':&nbsp;'unknown','ua_os_patch':&nbsp;'unknown','ua_os_patch_minor':&nbsp;'unknown','ua_family':&nbsp;'unknown','ua_major':&nbsp;'unknown','ua_minor':&nbsp;'unknown','ua_patch':&nbsp;'unknown','ua_device':&nbsp;'unknown'&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;}#&nbsp;UAif&nbsp;results&nbsp;and&nbsp;'family'&nbsp;in&nbsp;results:if&nbsp;results['family']&nbsp;is&nbsp;not&nbsp;None&nbsp;and&nbsp;results['family']&nbsp;!=&nbsp;'Other':&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;forSplunk['ua_family']&nbsp;=&nbsp;results['family']&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('for&nbsp;return:&nbsp;%s'&nbsp;%&nbsp;forSplunk)#&nbsp;Now&nbsp;write&nbsp;it&nbsp;out&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('outputting')&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;orow&nbsp;=&nbsp;[]for&nbsp;header_name&nbsp;in&nbsp;header:if&nbsp;header_name&nbsp;==&nbsp;"http_user_agent":&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;orow.append(http_user_agent)else:&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;orow.append(forSplunk[header_name])&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;w.writerow(orow)&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;logger.debug('done&nbsp;output')</div></div>

케세라세라

useragent 정보를 분석해주는 스플렁크앱. 로그스태시 useragent 필터와 같은 정규표현식 검사 기반(regexes.yaml)으로 동작하며 브라우저, 운영체제, 디바이스별로 정보를 분류해준다. 그중 브라우저 정보만 출력.
고작 로그 7천여 개 분석하는 데 40초 넘게 소요. 같은 동작 기반의 로그스태시 필터와 비교하면 100배 이상 느리다. 정규표현식 검사 범위를 좁히면 좀 빨라질까? 운영체제와 디바이스 검사 조건 삭제.
원래도 빨라서 체감이 잘 안 될 뿐, 같은 조건에서 로그스태시는 30% 정도의 성능 향상을 보인다. 그런데 스플렁크는 아예 동작하지 않음. 제미나이한테 물어보니 코드(user_agents.py) 수정이 필요하다고. 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import csv
import logging
import os
import sys
from urllib.parse import unquote_plus
from Utilities import KennyLoggins
 
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
DATA_FILE = os.path.abspath(os.path.join(ROOT_DIR, 'uap-core', 'regexes_edit.yaml'))
os.environ['UA_PARSER_YAML'] = DATA_FILE
 
from ua_parser import user_agent_parser
 
# The following log levels are available:
# Debug (NOISY!!!):
# LOG_LEVEL = logging.DEBUG
# Info:
# LOG_LEVEL = logging.INFO
# Error:
LOG_LEVEL = logging.ERROR
 
LOG_FILENAME = 'TA-user-agents'
kl = KennyLoggins()
logger = kl.get_logger(app_name="TA-user-agents", file_name=LOG_FILENAME, log_level=logging.INFO)
 
# Main routine - basically it's the standard python recipe for handling
# Splunk lookups
#
if __name__ == '__main__':
    r = csv.reader(sys.stdin)
    w = csv.writer(sys.stdout)
    have_header: bool = False
 
    header = []
    idx = -1
    for row in r:
        if not have_header:
            header = row
            logger.debug('fields found: %s' % header)
            have_header = True
            z = 0
            for h in row:
                if h == "http_user_agent":
                    idx = z
                z = z + 1
            w.writerow(row)
            continue
 
        # We only care about the cs_user_agent field - everything else is filled in
        http_user_agent = row[idx]
        useragent = unquote_plus(http_user_agent)
        logger.debug('found useragent %s' % http_user_agent)
 
        logger.debug('sending to ua-parser')
        results = []
        try:
            #results = user_agent_parser.Parse(http_user_agent)
            results = user_agent_parser.ParseUserAgent(http_user_agent)
        except Exception as err:
            logger.error(err)
            continue
        logger.debug('back from ua-parser')
 
        # create our results for Splunk
        # using the full results
        forSplunk = {
            'ua_os_family': 'unknown',
            'ua_os_major': 'unknown',
            'ua_os_minor': 'unknown',
            'ua_os_patch': 'unknown',
            'ua_os_patch_minor': 'unknown',
            'ua_family': 'unknown',
            'ua_major': 'unknown',
            'ua_minor': 'unknown',
            'ua_patch': 'unknown',
            'ua_device': 'unknown'
        }
 
        # UA
        if results and 'family' in results:
            if results['family'] is not None and results['family'] != 'Other':
                forSplunk['ua_family'] = results['family']
 
 
 
 
        logger.debug('for return: %s' % forSplunk)
        # Now write it out
        logger.debug('outputting')
        orow = []
        for header_name in header:
            if header_name == "http_user_agent":
                orow.append(http_user_agent)
            else:
                orow.append(forSplunk[header_name])
        w.writerow(orow)
        logger.debug('done output')
Colored by Color Scripter
cs
수정된 코드로 실행하니 35% 정도 빨라짐. 그래도 로그스태시에 비하면 50배 이상 느려(..)
더 개선할 수 없을까?
정규표현식 검사 범위를 더 좁히기는 힘드니, 검사 데이터양을 줄여야지. 사전 집계를 통해 7천여 개의 데이터를 1,700여 개로 축소.
데이터양을 줄이니 85% 가까운 성능 개선 결과를 보여준다. 물론 여전히 로그스태시가 수십 배는 더 빠름. 같은 정규표현식 검사 기반인데 어디서 이렇게 차이가 나는 걸까(..)
그런데 useragent가 이만큼 자원 소모해가면서 분석이 필요한 정보일까?
관련 글
Logstash 필터 useragent

TA-user-agents

댓글