Android-ElectricSheep-Automator
view release on metacpan or search on metacpan
lib/Android/ElectricSheep/Automator.pm view on Meta::CPAN
package Android::ElectricSheep::Automator;
# see also https://www.reddit.com/r/privacytoolsIO/comments/fit0tr/taking_almost_full_control_of_your_unrooted/
# swipe adb shell input touchscreen swipe 300 1200 100 1200 100
use 5.006;
use strict;
use warnings;
our $VERSION = '0.09';
use Mojo::Log;
use Config::JSON::Enhanced;
# it requires v0.002 (which is with my modifications)
#use Android::ADB;
# issue filed for Android::ADB :
# https://rt.cpan.org/Public/Bug/Display.html?id=163391
# until this is resolved I am copying Android::ADB
# into my distribution, fixing the issues and renaming it to
# Android::ElectricSheep::Automator::ADB
# and using that. When the issue is resolved I will go back
# using Android::ADB
# Credits for Android::ADB (now Android::ElectricSheep::Automator::ADB)
# go to Marius Gavrilescu (marius@ieval.ro)
# as seen in:
# https://metacpan.org/pod/Android::ADB
#
use Android::ElectricSheep::Automator::ADB;
use File::Temp qw/tempfile/;
use File::Basename;
use File::Spec;
use File::Path qw/make_path/;
use Cwd;
use FindBin;
use Time::HiRes qw/usleep/;
use Image::PNG;
use XML::LibXML;
use XML::LibXML::XPathContext;
use Text::ParseWords;
use Data::Roundtrip qw/perl2dump perl2json no-unicode-escape-permanently/;
use Android::ElectricSheep::Automator::DeviceProperties;
use Android::ElectricSheep::Automator::AppProperties;
use Android::ElectricSheep::Automator::ScreenLayout;
use Android::ElectricSheep::Automator::XMLParsers;
my $_DEFAULT_CONFIG = <<'EODC';
</* $VERSION = '0.09'; */>
</* comments are allowed */>
</* and <% vars %> and <% verbatim sections %> */>
{
"adb" : {
"path-to-executable" : "/usr/local/android-sdk/platform-tools/adb"
},
"debug" : {
"verbosity" : 0,
</* cleanup temp files on exit */>
"cleanup" : 1
},
"logger" : {
</* log to file if you uncomment this */>
</* "filename" : "..." */>
}
</* config for our plugins (each can go to separate file also) */>
}
EODC
# NOTE: by default, it assumes that no device is connected
# and so it does not enquire about screen size etc on startup
# In order to tell it that a device (just one)
# is connected to the desktop and that we should connect to
# it,
# use param 'device-is-connected' => 1
# if there are more than one devices and you want to connect to
# one of them, then
# use param 'device-serial' => <serial-of-device-to-connect>
# or
# use param 'device-object' => <device object>
# (of type Android::ElectricSheep::Automator::ADB::Device)
# or after instantiation with $obj->connect_device(...);
# and similarly for disconnect_device()
# NOTE: without connecting to a device you can not use
# methods which require a connected device, e.g. open_app(), swipe() etc.
sub new {
my $class = ref($_[0]) || $_[0]; # aka proto
my $params = $_[1] // {};
my $parent = ( caller(1) )[3] || "N/A";
my $whoami = ( caller(0) )[3];
my $self = {
'_private' => {
'confighash' => undef,
'configfile' => '', # this should never be undef
'Android::ADB' => undef,
'debug' => {
'verbosity' => 0,
'cleanup' => 1,
},
'log' => {
lib/Android/ElectricSheep/Automator.pm view on Meta::CPAN
if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
my $res = $self->adb->shell(@cmd);
if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening."); return unde...
if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return undef }
# we get something like "954\n995\n1217\n1372\n1392\n1642\n1741\n1856\n1898\n2549\n3236\n4456\n6570\n9245\n10115\n10746\n"
my $xx = $res->[1];
if( ! defined $xx ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, because the result got back was undef:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return undef }
my @results;
while( $xx =~ /^\s*(\d+)(\s+(.+?))?\s*$/smg ){
push @results, {
'pid' => $1,
'command' => $2 // ''
};
}
return \@results; # success
# a list of pids, can have 0, 1 or more items
}
# it takes the position on screen to tap either as a
# 'position' => [x,y]
# or
# 'bounds' => [ [topleftX,topleftY], [bottomrightX, bottomrightY] ]
# in which case, the tap position will be the mid-point of the 'bounds'
# rectangle.
# It returns 1 on failure, 0 on success
# it needs that connect_device() to have been called prior to this call
sub tap {
my ($self, $params) = @_;
$params //= {};
my $parent = ( caller(1) )[3] || "N/A";
my $whoami = ( caller(0) )[3];
my $log = $self->log();
my $verbosity = $self->verbosity();
if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'connect_device()'." before calling this."); return undef }
my (@position, $m);
if( exists($params->{'position'}) && defined($m=$params->{'position'}) ){
@position = ($m->[0], $m->[1]);
} elsif( exists($params->{'bounds'}) && defined($m=$params->{'bounds'}) ){
@position = ( int(($m->[1]->[0] + $m->[0]->[0])/2), int(($m->[1]->[1] + $m->[0]->[1])/2) );
} else { $log->error("${whoami} (via $parent), line ".__LINE__." : error, input parameter 'position' (as ['x','y']) or 'bounds' (as [lefttopX,lefttopY],[bottomrightX,bottomrighY]) was not specified."); return 1 }
my @cmd = ('input', 'tap', @position);
if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
my $res = $self->adb->shell(@cmd);
if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening."); return 1 }
if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
return 0; # success
}
# "type" the specified message into a text box expected to
# be found on the specified position.
# the message will be sanitised: spaces will be replaced with %s
# unicode in text is not supported.
# It returns 1 on failure, 0 on success
# it needs that connect_device() to have been called prior to this call
sub input_text {
my ($self, $params) = @_;
$params //= {};
my $parent = ( caller(1) )[3] || "N/A";
my $whoami = ( caller(0) )[3];
my $log = $self->log();
my $verbosity = $self->verbosity();
if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'connect_device()'." before calling this."); return undef }
my (@position, $m);
if( exists($params->{'position'}) && defined($m=$params->{'position'}) ){
@position = ($m->[0], $m->[1]);
} elsif( exists($params->{'bounds'}) && defined($m=$params->{'bounds'}) ){
@position = ( int(($m->[1]->[0] + $m->[0]->[0])/2), int(($m->[1]->[1] + $m->[0]->[1])/2) );
} else { $log->error("${whoami} (via $parent), line ".__LINE__." : error, input parameter 'position' (as ['x','y']) or 'bounds' (as [lefttopX,lefttopY],[bottomrightX,bottomrighY]) was not specified."); return 1 }
# optional text, else we send just '' (but we clicked on it)
my $text = (exists($params->{'text'}) && defined($params->{'text'})) ? $params->{'text'} : '';
# sanitise the text a bit
# replace spaces with %s,
# also newlines seem not to be supported so replaces these as well
$text =~ s/[\n \t]/%s/g;
# first tap on the text edit widget at the specified coordinates to get focus
if( $self->tap($params) ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, failed to tap on the position of the recipient of the text input"); return 1 }
usleep(0.8);
# and send the text
# adb shell input text 'hello%sworld'
# does not support unicode and also spaces must be converted to %s (already have done this)
my @cmd = ('input', 'text', $text);
if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
my $res = $self->adb->shell(@cmd);
if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening."); return 1 }
if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
return 0; # success
}
# returns 1 on failure, 0 on success
# it needs that connect_device() to have been called prior to this call
sub clear_input_field {
my ($self, $params) = @_;
$params //= {};
my $parent = ( caller(1) )[3] || "N/A";
my $whoami = ( caller(0) )[3];
my $log = $self->log();
my $verbosity = $self->verbosity();
if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'connect_device()'." before calling this."); return undef }
my (@position, $m);
if( exists($params->{'position'}) && defined($m=$params->{'position'}) ){
@position = ($m->[0], $m->[1]);
} elsif( exists($params->{'bounds'}) && defined($m=$params->{'bounds'}) ){
@position = ( int(($m->[1]->[0] + $m->[0]->[0])/2), int(($m->[1]->[1] + $m->[0]->[1])/2) );
} else { $log->error("${whoami} (via $parent), line ".__LINE__." : error, input parameter 'position' (as ['x','y']) or 'bounds' (as [lefttopX,lefttopY],[bottomrightX,bottomrighY]) was not specified."); return 1 }
# first tap on the text edit widget at the specified coordinates to get focus
if( $self->tap($params) ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, failed to tap on the position of the recipient of the text input"); return 1 }
usleep(0.8);
# from: https://stackoverflow.com/questions/32433303/clear-edit-text-adb
# the simplest way is input keycombination 113 29 && input keyevent 67
# but may not work
# then we try the lame way by erasing all chars one after the other
# part1:
my @cmd = ('input', 'keycombination', '113', '29');
if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
my $res = $self->adb->shell(@cmd);
if( ! defined($res) || ($res->[0] != 0) || ($res->[1]=~/Error: Unknown command/) || ($res->[2]=~/Error: Unknown command/) ){
$log->warn(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed which happens and now will try an alternative ...");
# alternative/part1
@cmd = ('input', 'keyevent', 'KEYCODE_MOVE_END');
if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
$res = $self->adb->shell(@cmd);
if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening. Info: this is...
if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed (Info: this is alternative part1), with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
# alternative/part2
# optional number of chars in the text-edit box, meaning how many
# times to press backspace, default is here (250)
# this is only needed for the second method (the failsafe)
my $numchars = (exists($params->{'num-characters'}) && defined($params->{'num-characters'}) && ($params->{'num-characters'}=~/^\d+$/) ) ? $params->{'num-characters'} : 250;
@cmd = ('input', 'keyevent', '--longpress', ('KEYCODE_DEL')x$numchars);
if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
$res = $self->adb->shell(@cmd);
if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening. Info: this is...
if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed (Info: this is alternative part1), with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
} else {
lib/Android/ElectricSheep/Automator.pm view on Meta::CPAN
denotes the time taken for the swipe
in milliseconds. The smaller its value the faster
the swipe. A value of C<100> is fast enough to swipe to
the next screen.
=back
It returns C<0> on success, C<1> on failure.
=head2 B<C<tap($params)>>
Emulates a "tap" at the specified location.
C<$params> is a HASH_REF which must contain one
of the following items:
=over 4
=item * B<C<position>>
should be an ARRAY_REF
as the C<X,Y> coordinates of the point to "tap".
=item * B<C<bounds>>
should be an ARRAY_REF of a bounding rectangle
of the widget to tap. Which contains two ARRAY_REFs
for the top-left and bottom-right coordinates, e.g.
C< [ [tlX,tlY], [brX,brY] ] >. This is convenient
when the widget is extracted from an XML dump of
the UI (see L</dump_current_screen_ui($params)>) which
contains exactly this bounding rectangle.
=back
It returns C<0> on success, C<1> on failure.
=head2 B<C<input_text($params)>>
It "C<types>" the specified text into the specified position,
where a text-input widget is expected to exist.
At first it taps at the widget's
location in order to get the focus. And then it enters
the text. You need to find the position of the desired
text-input widget by first getting the current screen UI
(using L</dump_current_screen_ui($params)>) and then using an XPath
selector to identify the desired widget by name/id/attributes.
See the source code of method L</send_message()> in file
C<lib/Android/ElectricSheep/Automator/Plugins/Apps/Viber.pm>
for how this is done for the message-sending text-input widget
of the Viber app.
C<$params> is a HASH_REF which must contain C<text>
and one of the two position (of the text-edit widget)
specifiers C<position> or C<bounds>:
=over 4
=item * B<C<text>>
the text to write on the text edit widget. At the
moment, this must be plain ASCII string, not unicode.
No spaces are accepted.
Each space character must be replaced with C<%s>.
=item * B<C<position>>
should be an ARRAY_REF
as the C<X,Y> coordinates of the point to "tap" in order
to get the focus of the text edit widget, preceding the
text input.
=item * B<C<bounds>>
should be an ARRAY_REF of a bounding rectangle
of the widget to tap, in order to get the focus, preceding
the text input. Which contains two ARRAY_REFs
for the top-left and bottom-right coordinates, e.g.
C< [ [tlX,tlY], [brX,brY] ] >. This is convenient
when the widget is extracted from an XML dump of
the UI (see L</dump_current_screen_ui($params)>) which
contains exactly this bounding rectangle.
=back
It returns C<0> on success, C<1> on failure.
=head2 B<C<clear_input_field($params)>>
It clears the contents of a text-input widget
at specified location.
There are several ways to do this. The simplest way
(with C<keycombination>) does not work in some
devices, in which case a failsafe way is employed
which deletes characters one after the other for
250 times.
C<$params> is a HASH_REF which must contain
one of the two position (of the text-edit widget)
specifiers C<position> or C<bounds>:
=over 4
=item <B<C<position>>
should be an ARRAY_REF
as the C<X,Y> coordinates of the point to "tap" in order
to get the focus of the text edit widget, preceding the
text input.
=item B<C<bounds>>
should be an ARRAY_REF of a bounding rectangle
of the widget to tap, in order to get the focus, preceding
the text input. Which contains two ARRAY_REFs
for the top-left and bottom-right coordinates, e.g.
C< [ [tlX,tlY], [brX,brY] ] >. This is convenient
when the widget is extracted from an XML dump of
the UI (see L</dump_current_screen_ui($params)>) which
contains exactly this bounding rectangle.
( run in 0.584 second using v1.01-cache-2.11-cpan-d7f47b0818f )