ELSA企业日志归档查询实战 - FreeBuf网络安全行业门户

官方公众号企业安全新浪微博

FreeBuf.COM网络安全行业门户，每日发布专业的安全资讯、技术剖析。

FreeBuf+小程序把安全装进口袋

安全管理

ELSA企业日志归档查询实战

2020-12-25 17:26:42

所属地北京

下面将要介绍的这款开源工具能够对日志进行集中采集和准实时索引处理，提供搜索、分析、监控和可视化等功能它的名字叫做ELSA，和著名的显卡厂商艾尔莎ELSA同名。

ELSA(全称：Enterprise Log Search and Archive)是一款基于Syslog-ng（新一代日志收集器，目前多数Linux发行版都不带此工具）、MySQL的开源企业日志归档查询工具，由于它和Sphinx的完美搭配，支持全文索引，可以像搜索Web一样搜索上亿条日志中的任意字符串（前提是多节点，并且服务器配置够高）。单节点ELSA日志采集系统的工作原理图如下所示：

1608887846_5fe5ae26c334cc636052c.png!small

上面这张架构图可以看出ELSA从架构上分为三层：

日志接收器，由syslog-ng完成负责接收来自本地、网络以及导入的日志文件。
日志存储索引，存储由MySQL数据库完成，索引由sphinx完成。
Web前端。

ELSA利用syslog-ng的pattern-db解析器进行有效的日志规范化，并利用Sphinx全文索引进行日志搜索。系统内部API将查询结果汇总后，发送给客户端，整个系统是异步执行，可以跑多个查询。接收器syslog-ng在接收日志时并没有进行归一化处理（类比OSSIM-Agent插件），所以对日志的正则表达式计算量不大，可在syslog-ng中保持高效的日志接收率，系统大部分有Perl脚本(/opt/elsa/node/)组成，MySQL每秒可插入100K行数据。Sphinx在索引中为新插入的行建立索引，每隔2小时会重新建立一次永久索引。整个系统最大效率发挥时每秒钟可处理100K条日志。

如果你具备ELK实战经验的话，可把ELSA理解为简版的ELK系统，结构简单，速度快。安装（感兴趣的朋友可在基于Debian（包括Ubuntu）的OS上测试，在ELSA Google Code主页上获取安装tar包）比较简单就不介绍了，下面直接切入正题。

1.采集Windows服务器日志

我们可采用Eventlog-to-Syslog工具将Windows平台的日志发送到ELSA服务器
方法：
将evtsys.exe和evtsys.dll,两个文件都复制到C:\windows\system32\目录下，进行如下操作：

net star evetsys //启动eventlog to syslog服务，成功之后才能输入下面的命令

1608887880_5fe5ae485c101fbe8ce43.png!small

evtsys.exe -i -h ELSA服务器的IP

本实验中ELSA Server IP : 192.168.109.227
将使用syslog协议发送到您的ELSA服务器，在该服务器中，日志将被解析为“ WINDOWS”类

2.采集Linux系统及相关服务的日志

Linux/Unix系统都有rsyslog 或 Syslogd进程，在其配置文件中加入下面的配置即可

*.* @ELSA服务器IP

3.配置文件概况

ELSA的主要配置文件是/etc/elsa_node.conf

{
        # 本地数据库连接信息
        "database" : {
                "db": "syslog",
                "data_db": "syslog_data",
                "dsn" : "dbi:mysql:database=syslog",
                "username" : "elsa",
                "password" : "biglog"
        },
        # 系统协调锁的目录
        "lockfile_dir": "/opt/elsa/node/tmp/locks",
        
        "num_indexes": 200,
        # 如果要归档日志，请保留此项
        "archive": {
                # Uncomment to establish a retention period in days for archive logs
                #"days": 90,
                "percentage": 33,
                "table_size": 10000000
        },
        # 日志大小限制+索引大小。设置为磁盘总空间的95-90%。
        "log_size_limit" : 8000000000,
        "sphinx" : {
                
                "indexer": "/usr/bin/indexer",
                
                "allowed_temp_percent" : 40,
              
                "allowed_mem_percent": 25
                "host" : "127.0.0.1",
                "port" : 9312,
        "mysql_port" : 9306,
                
                "config_file" : "/etc/sphinxsearch/sphinx.conf",
               
                "index_path" : "/nsm/elsa/data/sphinx",
                
                "index_interval" : 60,
                
                "perm_index_size" : 10000000,
                # Where the optional stopwords file is
                "stopwords": {
                        "file": "/etc/sphinxsearch/sphinx_stopwords.txt",
                        "top_n": 0,
                        "interval": 0,
                        "whitelist": []
                },
                
                "pid_file": "/var/run/sphinxsearch/searchd.pid"
        },
     
        "logdir" : "/nsm/elsa/data/elsa/log",
    "mysql_dir": "/nsm/elsa/data/elsa/mysql",
       
        "num_log_readers" : 1,
       #调试跟踪级别
        "debug_level" : "TRACE",
        
        "buffer_dir" : "/nsm/elsa/data/elsa/tmp/buffers/",
       
        "log_parse_errors": 1,
       
        "stats" : {
                "retention_days": 365
        },
     
        "min_expected_hosts": 2

ELSA的Web配置文件 /etc/elsa_web.conf 内容如下：

{
#定义API密钥
  "apikeys": {
    "elsa": "b7292980d34c99e2581d36681831667b"
  },
  "version": {
    "Author": "mcholste",
    "Date": "2014-07-17 15:12:58 -0700 (Thu, 17 Jul 2014)",
    "Rev": "1205",
    "Sphinx": "Sphinx 2.1.9"
  },
  "peers": {
    "127.0.0.1": {
      "url": "http://127.0.0.1:3154/",
      "username": "elsa",
      "apikey": "b7292980d34c99e2581d36681831667b"
    }
  },
  "admin_email_address": "root@localhost",
  "connectors": {
  },
  "dashboards": {
  },
  "datasources": {
  },
  "transforms": {
    "whois": {
      "known_subnets": {
        "10.0.0.0": {
          "end": "10.255.255.255",
          "org": "MyOrg"
        },
        "192.168.0.0": {
          "end": "192.168.255.255",
          "org": "MyOrg"
        },
        "172.16.0.0": {
          "end": "172.31.255.255",
          "org": "MyOrg"
        }
      },
      "known_orgs": {
        "MyOrg": {
          "name": "MyOrg",
          "org": "MyOrg",
          "descr": "MyOrg",
          "cc": "US",
          "country": "United States",
          "city": "Anytown",
          "state": "Somestate"
        }
      }
    },
    "parse": {
      "tld": [
        {
          "field": "domain",
          "pattern": "\\.([a-zA-Z]+)$",
          "extractions": [
            "tld"
          ]
        },
        {
          "field": "site",
          "pattern": "\\.([a-zA-Z]+)$",
          "extractions": [
            "tld"
          ]
        },
        {
          "field": "uri",
          "pattern": "\\.([a-zA-Z]+)(:|/|$)",
          "extractions": [
            "tld"
          ]
        }
      ],
      "url": [
        {
          "field": "uri",
          "pattern": "(?:(?<proto>[a-zA-Z]+)://)?(?:(?<username>[^/]+):(?<password>[^/]+)@)?(?<domain>\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|[^/]+\\.(?<tld>[a-zA-Z]+))(?::(?<port>\\d+))?(?<resource>/[^?]*)?(?:\\?(?<query_string>.*))?$",
          "extractions": [
            "proto",
            "username",
            "password",
            "domain",
            "tld",
            "port",
            "resource",
            "query_string"
          ]
        }
      ],
      "mimetype": [
        {
          "field": "msg",
          "pattern": "[\"'\\(\\[\\s\\|;:](?<mime>(?<type>application|audio|chemical|image|message|model|multipart|text|video)/(?<subtype>[\\w-_]+))[\"'\\)\\]\\s\\|;:]",
          "extractions": [
            "mime",
            "type",
            "subtype"
          ]
        }
      ]
    }
  },
  "plugins": {
    "SNORT": "Info::Snort",
    "WINDOWS": "Info::Windows",
    "URL": "Info::Url",
    "BRO_NOTICE": "Info::Bro"
  },
  "info": {
    "snort": {
      "url_templates": [
        "http://doc.emergingthreats.net/bin/view/Main/%d"
      ]
    },
    "url": {
      "url_templates": [
        "http://whois.domaintools.com/%s"
      ]
    },
    "windows": {
      "url_templates": [
        "http://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventid=%d"
      ]
    }
  },
  "max_concurrent_archive_queries": 4,
  "schedule_interval": 60,
  "node_info_cache_timeout": 60,
  "email": {
    "display_address": "noreply-elsa@example.com",
    "base_url": "http://elsa/",
    "subject": "ELSA Alert"
  },
  "link_key": "secret",
  "yui": {
    "local": "inc"
  },
  "data_db": {
    "db": "syslog",
    "username": "elsa",
    "password": "biglog"
  },
  "meta_db": {
    "dsn": "dbi:mysql:database=elsa_web",
    "username": "elsa",
    "password": "biglog"
  },
  "auth": {
    "method": "security_onion"
  },
  "admin_groups": [
    "system",
    "admin"
  ],
  "auth_db": {
    "dsn": "dbi:mysql:database=securityonion_db",
    "username": "root",
    "password": "",
    "auth_statement": "SELECT PASSWORD(password) FROM user_info WHERE username=?",
    "email_statement": "SELECT email FROM user_info WHERE username=?"
  },
  "peer_id_multiplier": 1000000000000,
  "query_timeout": 55,
  "pcap_url": "/capme",
  "logdir": "/nsm/elsa/data/elsa/log",
  "buffer_dir": "/nsm/elsa/data/elsa/tmp/buffers",
  "debug_level": "TRACE",
  "default_start_time_offset": 2,
  "livetail": {
    "poll_interval": 5,
    "time_limit": 3600
  }
}

4.典型应用场景

着重对ELSA软件的几个重点功能进行展示。

1.连接数 Top N

1608887968_5fe5aea049d24f6b0269f.png!small