SysmonForLinux | New integration to support Sysmon logs for Linux (el…

…astic#4531) * initial sysmon commit * remove grok processor; add dissect module * tests * run tests * add simpler logic in filestream * add code owners for sysmon * sysmon_linux instead of sysmon and PR review * update name in codeowners * add dockercompo * Drop if not sysmon process
efd6 · Dec 8, 2022 · 5c6167b · 5c6167b
1 parent f9afcc0
commit 5c6167b
Show file tree

Hide file tree

Showing 25 changed files with 4,561 additions and 0 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -172,6 +172,7 @@
 /packages/suricata @elastic/security-external-integrations
 /packages/symantec_endpoint @elastic/security-external-integrations
 /packages/synthetics @elastic/uptime
+/packages/sysmon_linux @elastic/security-external-integrations
 /packages/system @elastic/elastic-agent-data-plane
 /packages/system/kibana @elastic/elastic-agent-data-plane @elastic/kibana-visualizations
 /packages/tcp @elastic/security-external-integrations

diff --git a/packages/sysmon_linux/_dev/build/build.yml b/packages/sysmon_linux/_dev/build/build.yml
@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]
diff --git a/packages/sysmon_linux/_dev/build/docs/README.md b/packages/sysmon_linux/_dev/build/docs/README.md
@@ -0,0 +1,26 @@
+# Sysmon for Linux Integration
+
+The Sysmon for Linux integration allows you to monitor the [Sysmon for Linux](https://github.com/Sysinternals/SysmonForLinux), which is an open-source system monitor tool developed to collect security events from Linux environments.
+
+Use the Sysmon for Linux integration to collect logs from linux machine which has sysmon tool running.
+Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference data when troubleshooting an issue.
+
+NOTE: To collect Sysmon events from Windows event log, use [Windows `sysmon_operational` data stream](https://docs.elastic.co/en/integrations/windows#sysmonoperational) instead.
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+## Setup
+
+For step-by-step instructions on how to set up an integration,
+see the {{ url "getting-started-observability" "Getting started" }} guide.
+
+## Data streams
+
+The Sysmon for Linux `log` data stream provides events from logs produced by Sysmon tool running on Linux machine.
+
+{{event "log"}}
+
+{{fields "log"}}
diff --git a/packages/sysmon_linux/_dev/deploy/docker/docker-compose.yml b/packages/sysmon_linux/_dev/deploy/docker/docker-compose.yml
@@ -0,0 +1,8 @@
+version: "2.3"
+services:
+  sysmon_linux:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"
diff --git a/packages/sysmon_linux/_dev/deploy/docker/sample_logs/sysmon.log b/packages/sysmon_linux/_dev/deploy/docker/sample_logs/sysmon.log
diff --git a/packages/sysmon_linux/changelog.yml b/packages/sysmon_linux/changelog.yml
@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: initial release
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4531
diff --git a/packages/sysmon_linux/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/sysmon_linux/data_stream/log/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/sysmon_linux/data_stream/log/_dev/test/pipeline/test-sysmon.json b/packages/sysmon_linux/data_stream/log/_dev/test/pipeline/test-sysmon.json
@@ -0,0 +1,73 @@
+{
+    "events": [
+        {
+            "log": {
+                "syslog": {
+                    "hostname": "user-ubuntu",
+                    "appname": "sysmon",
+                    "procid": "3099"
+                }
+            },
+            "winlog": {
+                "record_id": "24",
+                "computer_name": "user-ubuntu",
+                "process": {
+                    "pid": 3099,
+                    "thread": {
+                        "id": 3099
+                    }
+                },
+                "event_id": "5",
+                "provider_guid": "{ff032593-a8d3-4f13-b0d6-01fc615a0f97}",
+                "level": "information",
+                "channel": "Linux-Sysmon/Operational",
+                "time_created": "2022-10-24T17:06:14.193Z",
+                "event_data": {
+                    "User": "root",
+                    "ProcessId": "783",
+                    "Image": "/opt/Elastic/Agent/data/elastic-agent-5ae799/install/osquerybeat-7.15.1-linux-x86_64/osqueryd",
+                    "RuleName": "-",
+                    "ProcessGuid": "{d65774de-8d1e-6175-0000-000000000000}",
+                    "UtcTime": "2022-10-24 17:06:14.200",
+                    "opcode": "Info",
+                    "provider_name": "Linux-Sysmon",
+                    "version": "3"
+                }
+            }
+        },
+        {
+            "log": {
+                "syslog": {
+                    "hostname": "user-ubuntu",
+                    "appname": "sysmon"
+                }
+            },
+            "winlog": {
+                "computer_name": "user-ubuntu",
+                "record_id": "22",
+                "process": {
+                    "pid": 3099,
+                    "thread": {
+                        "id": 3099
+                    }
+                },
+                "event_id": "5",
+                "level": "information",
+                "provider_guid": "{ff032593-a8d3-4f13-b0d6-01fc615a0f97}",
+                "channel": "Linux-Sysmon/Operational",
+                "time_created": "2022-10-24T17:05:48.296Z",
+                "event_data": {
+                    "User": "root",
+                    "ProcessId": "3100",
+                    "Image": "(null)",
+                    "ProcessGuid": "{d65774de-8d10-6175-0000-000000000000}",
+                    "RuleName": "-",
+                    "UtcTime": "2022-10-24 17:05:48.302",
+                    "provider_name": "Linux-Sysmon",
+                    "opcode": "Info",
+                    "version": "3"
+                }
+            }
+        }
+    ]
+}
diff --git a/packages/sysmon_linux/data_stream/log/_dev/test/pipeline/test-sysmon.json-expected.json b/packages/sysmon_linux/data_stream/log/_dev/test/pipeline/test-sysmon.json-expected.json
@@ -0,0 +1,100 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.5.0"
+            },
+            "event": {
+                "action": "log",
+                "category": [
+                    "process"
+                ],
+                "code": "5",
+                "created": "2022-10-24T17:06:14.193Z",
+                "kind": "event",
+                "provider": "Linux-Sysmon",
+                "type": [
+                    "end"
+                ]
+            },
+            "host": {
+                "hostname": "user-ubuntu"
+            },
+            "log": {
+                "level": "information"
+            },
+            "process": {
+                "entity_id": "{d65774de-8d1e-6175-0000-000000000000}",
+                "executable": "/opt/Elastic/Agent/data/elastic-agent-5ae799/install/osquerybeat-7.15.1-linux-x86_64/osqueryd",
+                "name": "sysmon",
+                "pid": 783
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "winlog": {
+                "channel": "Linux-Sysmon/Operational",
+                "computer_name": "user-ubuntu",
+                "event_id": "5",
+                "opcode": "Info",
+                "process": {
+                    "pid": 3099,
+                    "thread": {
+                        "id": 3099
+                    }
+                },
+                "provider_guid": "{ff032593-a8d3-4f13-b0d6-01fc615a0f97}",
+                "record_id": "24",
+                "version": 3
+            }
+        },
+        {
+            "ecs": {
+                "version": "8.5.0"
+            },
+            "event": {
+                "action": "log",
+                "category": [
+                    "process"
+                ],
+                "code": "5",
+                "created": "2022-10-24T17:05:48.296Z",
+                "kind": "event",
+                "provider": "Linux-Sysmon",
+                "type": [
+                    "end"
+                ]
+            },
+            "host": {
+                "hostname": "user-ubuntu"
+            },
+            "log": {
+                "level": "information"
+            },
+            "process": {
+                "entity_id": "{d65774de-8d10-6175-0000-000000000000}",
+                "executable": "(null)",
+                "name": "sysmon",
+                "pid": 3100
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "winlog": {
+                "channel": "Linux-Sysmon/Operational",
+                "computer_name": "user-ubuntu",
+                "event_id": "5",
+                "opcode": "Info",
+                "process": {
+                    "pid": 3099,
+                    "thread": {
+                        "id": 3099
+                    }
+                },
+                "provider_guid": "{ff032593-a8d3-4f13-b0d6-01fc615a0f97}",
+                "record_id": "22",
+                "version": 3
+            }
+        }
+    ]
+}
diff --git a/packages/sysmon_linux/data_stream/log/_dev/test/system/test-filestream-config.yml b/packages/sysmon_linux/data_stream/log/_dev/test/system/test-filestream-config.yml
@@ -0,0 +1,7 @@
+service: sysmon_linux
+input: filestream
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/*.log"
+    preserve_original_event: true
diff --git a/packages/sysmon_linux/data_stream/log/agent/stream/filestream.yml.hbs b/packages/sysmon_linux/data_stream/log/agent/stream/filestream.yml.hbs
@@ -0,0 +1,52 @@
+paths:
+{{#each paths as |path i|}}
+ - {{path}}
+{{/each}}
+prospector.scanner.exclude_files: [".gz$"]
+{{!-- inlcude_lines: '.* sysmon(\:|\[\d+\]).*' --}}
+{{!-- exclude_lines: '.* sysmon(\:|\[\d+\]).*' --}}
+{{!-- exclude_lines: '^(?:(?!.* sysmon(\:|\[\d+\]).*).)+$' --}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+- add_locale: ~
+- rename:
+    fields:
+      - from: message
+        to: event.original
+    ignore_missing: true
+    fail_on_error: false
+- syslog:
+    field: "event.original"
+    ignore_missing: true
+    ignore_failure: true
+- drop_event:
+    when:
+      and:
+        - has_fields: ['log.syslog.appname']
+        - not:
+            equals:
+              log.syslog.appname: 'sysmon'
+- decode_xml_wineventlog:
+    when:
+      regexp:
+        message: '^\<Event'
+    field: event.original
+    target_field: winlog
+    ignore_missing: true
+    ignore_failure: true
+    map_ecs_fields: true
+{{#if processors.length}}
+{{processors}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}