view release on metacpan or  search on metacpan

src_bin/suricata_extract_submit  view on Meta::CPAN

          write-fileinfo: yes
          stream-depth: 0
          force-hash: [sha1, md5]
          xff:
            enabled: no
            mode: extra-data
            deployment: reverse
            header: X-Forwarded-For

The force-hash values are optional as this script will automatically compute
those for inclusion as well SHA256.

This will use `/var/run/suricata_extract_submit.pid` as a PID file
to ensure only a single instance is running.

May be checked to see if it has hung like below. Below will alert if a PID
file with a M time of older than 5 minutes.

    /usr/local/libexec/nagios/check_file_age -i -f /var/run/suricata_extract_submit.pid

Safe to dump the output of this to dev null in cron as the output is also sent to syslog
as 'suricata_extract_submit' to the 'daemon' log.

=head1 CONFIGURATION

The config file used is '/usr/local/etc/suricata_extract_submit.ini'.

    # the API key to use if needed
    #apikey=

    # URL to find mojo_cape_submit at
    url=http://192.168.14.15:8080/

    # the group/client/whathaveya slug
    slug=foo

    # where Suricata has the file store at
    filestore=/var/log/suricata/files

    # a file of IPs or subnets to ignore SRC or DEST IPs of
    #ignore=

    # a file of regex to use for checking host names to ignore
    #ignoreHosts=

    # a file of regex to use for checking host names to ignore
    #ignoreHosts=

    # a file of regex to use for checking user agents to ignore
    #ignoreUAs=

    # a file of regex to use for checking path chunks of the URL to ignore
    #ignorePaths=

    # a file of regex to use for checking path chunks of the URL to ignore
    ignoreMaxSize=52428800

    # a JSON file to use for using with Web::ACL for checking for ignores
    #ignoreWebACL=/usr/local/etc/suricata_extract_submit_webacl.json

    # if it should use HTTPS_PROXY and HTTP_PROXY from ENV or not
    env_proxy=0

    # stats file holding only the stats for the last run
    stats_file=/var/cache/suricata_extract_submit_stats.json

    # stats dir
    stats_dir=/var/cache/suricata_extract_submit_stats/

=head2 example Web::ACL

A example Web::ACL that ignores UAs matching "foo bar" and
paths matching "/derp.exe".

    {
      "foo": {
        "ua_auth": 1,
        "path_auth": 1,
        "ua_regex_allow": [
          "^foo bar$"
        ],
        "paths_regex_allow": [
          "^\\/derp.exe$"
        ],
        "final": 1
      }
    }

Each key will be tests twice. First with the IP set to the src
IP and then set to the client IP. The var mapping is as below.

    .http.http_user_agent -> ua
    .fileinfo.filename -> path

    .dest_ip -> ip
    .src_ip -> ip

=cut

my $help;
my $version;
GetOptions(
	'h'       => \$help,
	'help'    => \$help,
	'version' => \$version,
	'v'       => \$version,
);

if ($version) {
	version;
	exit 255;
}
if ($help) {
	pod2usage( -exitval => 255, -verbose => 2, -output => \*STDOUT, );
}

my $t          = localtime;
my $today_name = $t->strftime('%F');

# sends stuff to syslog
sub log_something {

src_bin/suricata_extract_submit  view on Meta::CPAN

							my $path = '';
							if ( defined( $file_json->{http}{http_user_agent} ) ) {
								$path = $file_json->{fileinfo}{filename};
								$path =~ s/\"/\\\"/g;
							}
							log_something( 'info',
									  'Ignore Listed WebACL, "'
									. $acl
									. '": dest_ip="'
									. $file_json->{dest_ip}
									. '", ua="'
									. $ua
									. '", path="'
									. $path
									. '"' );
						} ## end if ($results)
					} ## end else [ if ($results) ]
				} ## end if ( $add_it && defined( $file_json->{http...}))
			} ## end foreach my $acl (@acls)
		} ## end if ($add_it)

		$name
			= $file_json->{src_ip} . '-'
			. $file_json->{src_port} . '-'
			. $file_json->{dest_ip} . '-'
			. $file_json->{dest_port} . '-'
			. $file_json->{proto} . '-'
			. $extUID . '-'
			. $slug . '-'
			. $t->epoch . '-'
			. $mime;

		# only add it if it is not white listed
		if ($add_it) {
			log_something( 'info', 'Name: ' . $name );

			# skip uploading it if it is zero sized
			if ( $data_size > 0 ) {

				# don't need to copy it if it is zero sized
				copy( $data_file, $name )
					or die 'Copy failed(' . $data_file . ' -> ' . $tempdir . '/' . $name . '): ' . $!;

				$file_json->{suricata_extract_submit} = {
					filename => $name,
					apikey   => $apikey,
					host     => hostname,
					to       => $config->{_}->{url},
					time     => time,
					md5      => $md5,
					sha256   => $sha256,
					sha1     => $sha1,
					slug     => $config->{_}->{slug},
				};
				my $res;
				eval {
					my $ua = LWP::UserAgent->new(
						ssl_opts => { verify_hostname => 0, SSL_verify_mode => 0 },
						timeout  => 30
					);
					# use HTTPS_PROXY/HTTP_PROXY from env if set to true
					if ($use_env_proxy) {
						$ua->env_proxy;
					}
					$res = $ua->request(
						POST $config->{_}->{url},
						Content_type => 'multipart/form-data',
						Content      => [
							apikey   => $apikey,
							filename => [$name],
							type     => 'suricata_extract',
							json     => encode_json($file_json),
						],
					);
				};

				# save error status for later
				my $sub_error = $@;

				# see if we can get a status line
				my $status_line;
				if ( defined($res) ) {
					eval { $status_line = $res->status_line; };

					# compute that status line stats
					if ( defined($status_line) ) {
						if ( $status_line =~ /^2\d\d/ ) {
							$new_stats->{sub_2xx}++;
						} elsif ( $status_line =~ /^3\d\d/ ) {
							$new_stats->{sub_3xx}++;
						} elsif ( $status_line =~ /^4\d\d/ ) {
							$new_stats->{sub_4xx}++;
						} elsif ( $status_line =~ /^5\d\d/ ) {
							$new_stats->{sub_5xx}++;
						}
					} ## end if ( defined($status_line) )
				} ## end if ( defined($res) )

				# handle submission errors
				if ($sub_error) {
					$new_stats->{sub_fail}++;

					# if this is defined, submission worked, but we got a sub error
					if ( defined($status_line) ) {
						my $error = "Failed to post... " . $res->status_line;
						push( @{ $new_stats->{last_errors} }, $error );
						die($error);
					}    # if we don't have a status line the submission eval never got that far
					else {
						$new_stats->{errors}++;
						my $error = "Failed to post... " . $sub_error;
						push( @{ $new_stats->{last_errors} }, $error );
						die($error);
					}
				} else {
					if ( $status_line =~ /^2\d\d/ ) {
						$new_stats->{sub}++;
						log_something( 'info', 'Uplodated Response Status: ' . $res->status_line );
					} else {
						$new_stats->{sub_fail}++;
						my $error = "Failed to post... " . $res->status_line;