transforms.conf

transforms.conf

The following are the spec and example files for transforms.conf.

transforms.conf.spec

# Copyright (C) 2005-2010 Splunk Inc.  All Rights Reserved.  Version 4.0
#
# This file contains possible attributes and values you can use to configure transform
# and event signing in transforms.conf.
#
# There is a transforms.conf in $SPLUNK_HOME/etc/system/default/.  To set custom configurations, 
# place a transforms.conf $SPLUNK_HOME/etc/system/local/.  For examples, see transforms.conf.example.
# You can enable configurations changes made to transforms.conf by typing the following search string
# in Splunk Web:
#
# | extract reload=t 
#
# To learn more about configuration files (including precedence) please see the documentation 
# located at http://www.splunk.com/base/Documentation/latest/Admin/Aboutconfigurationfiles


[<unique_stanza_name>]
* Name your stanza. Use this name when configuring props.conf.
  For example, in a props.conf stanza, enter TRANSFORMS-<value> = <unique_stanza_name>.
* Follow this stanza name with any number of the following attribute/value pairs.  
* If you do not specify an entry for each attribute, Splunk uses the default value.

REGEX = <regular expression>
* Enter a regular expression to operate on your data.
* For transformations that occur at search time:
 * Name capturing groups in the REGEX are extracted directly to fields, meaning there is no need to specify FORMAT
 * If both field name and field value are extracted using the REGEX, the following special capturing 
   groups can be used to skip specifying the mapping in FORMAT _KEY_<string>, _VAL_<string>. 
   For example the following are equivalent:
  * using FORMAT:
   * REGEX  = ([a-z]+)=([a-z]+)
   * FORMAT = $1::$2
  * without using FORMAT
   * REGEX  = (?<_KEY_1>[a-z]+)=(?<_VAL_1>[a-z]+)
* Defaults to empty.
* NOTE: this option is valid for both index/search time KV extraction.

LOOKAHEAD = <integer>
* Specify how many characters to search into an event.
* Defaults to 256.
* NOTE: this option is valid only for index time KV extraction.

DEST_KEY = <KEY>
* Specify where to store the results of the REGEX.
* Use the KEYs listed below.
* NOTE: this option is valid only for index time KV extraction.

FORMAT = <string>
* Specify the format of the event, including any fields names or values you want to add.
* Use $n (for example $1, $2, etc) to specify the output of each REGEX match. 
* If the regex does not have n groups, the matching fails. 
* The special identifier $0 represents what was in the DEST_KEY before this regex was performed.
* Defaults to $1.
* NOTE: this option is valid for both index/search time KV extraction.

WRITE_META = <true | false>
* Automatically writes REGEX to metadata.
* Use instead of DEST_KEY = meta.
* Defaults to false.
* NOTE: this option is valid only for index time KV extraction.

DEFAULT_VALUE = <string>
* If set, and REGEX (above) fails, write this value to DEST_KEY.
* Defaults to empty.
* NOTE: this option is valid only for index time KV extraction.

SOURCE_KEY = <string>
* Set which KEY to perform the regex on.
* Defaults to _raw (the raw event).
* For index-time transformations use the KEYs listed below.
* For search-time extractions use any field that is available at the time of the 
  execution of this field extraction
* NOTE: this option is valid for both index/search time KV extraction.


REPEAT_MATCH = <true | false>
* Specify whether to run REGEX several times on the SOURCE_KEY. 
* REPEAT_MATCH starts wherever the last match stopped, and continues until no more matches are found.
* Defaults to false.
* NOTE: this option is valid only for index time KV extraction.

DELIMS = <quoted string list>
* Set delimiter characters to separate data into key-value pairs, and then to separate key from value.
* NOTE: Delimiters must be quoted with " " (to escape, use \).
* Usually, two sets of delimiter characters must be specified: 
 * The first to extract key/value pairs.
 * The second to separate the key from the value.
* If you enter only one set of delimiter characters, then the extracted tokens:
 * Are named with names from FIELDS, if FIELDS are entered (below).
 * OR even tokens are used as field names while odd tokens become field values. 
* Consecutive delimiter characters are consumed except when a list of field names is specified. 
* NOTE: this option is valid only for search time KV extraction.
  
FIELDS = <quoted string list>
* List the names of the field values extracted using DELIMS.
* NOTE: If field names contain spaces or commas they must be quoted with " " (to escape, use \).
* Defaults to "".
* NOTE: this option is valid only for search time KV extraction.

MV_ADD = <bool>
* Option controlling what the extractor does when it finds a field which already exists.
* If set to true, the extractor makes the field a multivalued field and appends the 
* newly found value, otherwise the newly found value is discarded.
* Defaults to false
* NOTE: this option is valid only for search time KV extraction.

CLEAN_KEYS = <bool>
* Option controlling whether the keys extracted at search time are cleaned. Key cleaning
  is defined as the replacement of non-alphanumeric characters with underscores. Leading 
  underscores and numbers are stripped.
* Defaults to true

CAN_OPTIMIZE = <bool>
* Option controlling whether Splunk can optimize this extraction out. An extraction is disabled
  if Splunk can determine that none of the fields extracted (by an extraction) will ever be 
  needed for the successful evaluation of a search. This option should be rarely set to false
* Defaults to true


#*******
# Lookup tables
#*******
# NOTE: lookup tables are used ONLY during search time

filename = <string>
* Name of static lookup file.  
* File should be in $SPLUNK_HOME/etc/<app_name>/lookups/ for some <app_name>, or in $SPLUNK_HOME/etc/system/lookups/
* If file is in multiple 'lookups' directories, no layering is done.  
* Standard conf file precedence is used to disambiguate.

max_matches = <integer>
* Maximum number of matching for each input lookup value
* Default = 100 if non-temporal, default = 1 if temporal (i.e. time_field is specified).
* If non-temporal, the first (in file order) <integer> entries are used.  
* If temporal, the first in descending time order <integer> are used.

min_matches = <integer>
* Minimum number of matches for each input lookup value
* Default = 0 for both temporal and non-temporal, meaning that nothing is output if no match is found
* However, if min_matches > 0, and we get less than min_matches, then we emit the default_match value provided.

default_match = <string>
* If min_matches > 0 and we have less than min_matches for any given input, we write out this default_match value 
  one or more times such that the min_matches threshold is reached 

external_cmd = <string>
* Command and arguments to invoke to perform lookups.
* This string is parsed like a shell command.
* The first argument is expected to be a python script located in $SPLUNK_HOME/etc/<app_name>/bin (or ../etc/searchscripts)
* Presence of this field indicates that lookup is external command based.

fields_list = <string>
* A comma and space delimited list of all fields that are supported by the external command.

external_type = python
* Type of external command.  
* Currently, only python is supported.

time_field = <string>
* For temporal (i.e. time bounded) lookups, specifies the field in the lookup table that represents the timestamp.
* Default = <empty string>, meaning that lookup is not temporal.

time_format = <string>
* For temporal lookups, specifies the 'strptime' format of the timestamp field.
* You can include subseconds but they will be ignored
* Default format is pure UTC time. 

max_offset_secs = <integer>
* For temporal lookups, the maximum time in seconds that the event time may be 
  ahead of lookup entry time for a match to occur
* Default is 2000000000 (no maximum).

min_offset_secs = <integer>
* For temporal lookups, the minimum time (in seconds) that the event time must be ahead of lookup entry time for a match to occur.
* Defaults to 0.


#*******
# KEYS:
#*******
* NOTE: Keys are case-sensitive.  Use the following keys exactly as they appear.

queue : Specify which queue to send the event to (can be parsingQueue, nullQueue, indexQueue).
_raw  : The raw text of the event.
_done : If set to any string, this is the last event in a stream.
_meta : A space separated list of metadata for an event.
_time : The timestamp of the event, in seconds since 1/1/1970 UTC.
MetaData:FinalType  : The event type of the event.

MetaData:Host       : The host associated with the event.
                      The value must be prefixed by "host::"

_MetaData:Index     : The index where the event should be stored.

MetaData:Source     : The source associated with the event.
                      The value must be prefixed by "source::"

MetaData:Sourcetype : The sourcetype of the event.
                      The value must be prefixed by "sourcetype::"
* NOTE: Any KEY prefixed by '_' is not indexed by Splunk, in general.

transforms.conf.example

# Copyright (C) 2005-2010 Splunk Inc.  All Rights Reserved.  Version 4.0 
#
# This is an example transforms.conf.  Use this file to create regexes and rules for transforms.
# Use this file in tandem with props.conf.
#
# To use one or more of these configurations, copy the configuration block into transforms.conf 
# in $SPLUNK_HOME/etc/system/local/. You must restart Splunk to enable configurations.
#
# To learn more about configuration files (including precedence) please see the documentation 
# located at http://www.splunk.com/base/Documentation/latest/Admin/Aboutconfigurationfiles

# Note: These are examples.  Replace the values with your own customizations.


# Indexed field:

[netscreen-error]
REGEX =  device_id=[^ ]+\s+\[w+\](.*)(?
FORMAT = err_code::$1
WRITE_META = true

# Extracted field:

[netscreen-error]
REGEX = device_id=[^ ]+\s+\[w+\](.*)(?
FORMAT = err_code::$1

# Override host:

[hostoverride]
DEST_KEY = MetaData:Host
REGEX = \s(\w*)$
FORMAT = host::$1


# Extracted fields:

[netscreen-error]
REGEX = device_id=[^ ]+\s+\[w+\](.*)(?
FORMAT = err_code::$1


# Static lookup table

[mylookuptable]
filename = mytable.csv

# one to one lookup
# guarantees that we output a single lookup value for each input value, if no match exists, 
# we use the value of "default_match", which by default is "NONE"
[mylook]
filename = mytable.csv
max_matches = 1
min_matches = 1
default_match = nothing

# external command lookup table

[myexternaltable]
external_cmd = testadapter.py blah
fields_list = foo bar

# Temporal based static lookup table

[staticwtime]
filename = mytable.csv
time_field = timestamp
time_format = %d/%m/%y %H:%M:%S


# Mask sensitive data:

[session-anonymizer]
REGEX = (?m)^(.*)SessionId=\w+(\w{4}[&"].*)$
FORMAT = $1SessionId=########$2
DEST_KEY = _raw


# Route to an alternate index:

[AppRedirect]
REGEX = Application
DEST_KEY = _MetaData:Index
FORMAT = Verbose


# Extract comma-delimited values into fields:

[extract_csv]
DELIMS = ","
FIELDS = "field1", "field2", "field3"

# This example assigns the extracted values from _raw to field1, field2 and field3 (in order of 
# extraction). If more than three values are extracted the values without a matching field name 
# are ignored.

# Extract key-value pairs
# This example extracts key-value pairs which are separated by '|'
# while the key is delimited from value by '='.

[pipe_eq]
DELIMS = "|", "="

# This example extracts key-value pairs which are separated by '|'
# while the key is delimited from value by '='.

• Was this topic useful?
• Post a comment