edit results from the CPAN

Android-ElectricSheep-Automator

package Android::ElectricSheep::Automator;

# see also https://www.reddit.com/r/privacytoolsIO/comments/fit0tr/taking_almost_full_control_of_your_unrooted/
# swipe adb shell input touchscreen swipe 300 1200 100 1200 100

use 5.006;
use strict;
use warnings;

our $VERSION = '0.09';

use Mojo::Log;
use Config::JSON::Enhanced;
# it requires v0.002 (which is with my modifications)
#use Android::ADB;
# issue filed for Android::ADB :
#   https://rt.cpan.org/Public/Bug/Display.html?id=163391
# until this is resolved I am copying Android::ADB
# into my distribution, fixing the issues and renaming it to
# Android::ElectricSheep::Automator::ADB
# and using that. When the issue is resolved I will go back
# using Android::ADB
# Credits for Android::ADB (now Android::ElectricSheep::Automator::ADB)
# go to Marius Gavrilescu (marius@ieval.ro)
# as seen in:
#   https://metacpan.org/pod/Android::ADB
#
use Android::ElectricSheep::Automator::ADB;
use File::Temp qw/tempfile/;
use File::Basename;
use File::Spec;
use File::Path qw/make_path/;
use Cwd;
use FindBin;
use Time::HiRes qw/usleep/;
use Image::PNG;
use XML::LibXML;
use XML::LibXML::XPathContext;
use Text::ParseWords;

use Data::Roundtrip qw/perl2dump perl2json no-unicode-escape-permanently/;

use Android::ElectricSheep::Automator::DeviceProperties;
use Android::ElectricSheep::Automator::AppProperties;
use Android::ElectricSheep::Automator::ScreenLayout;
use Android::ElectricSheep::Automator::XMLParsers;

my $_DEFAULT_CONFIG = <<'EODC';
</* $VERSION = '0.09'; */>
</* comments are allowed */>
</* and <% vars %> and <% verbatim sections %> */>
{
	"adb" : {
		"path-to-executable" : "/usr/local/android-sdk/platform-tools/adb"
	},
	"debug" : {
		"verbosity" : 0,
		</* cleanup temp files on exit */>
		"cleanup" : 1
	},
	"logger" : {
		</* log to file if you uncomment this */>
		</* "filename" : "..." */>
	}
	</* config for our plugins (each can go to separate file also) */>
}
EODC

# NOTE: by default, it assumes that no device is connected
# and so it does not enquire about screen size etc on startup
# In order to tell it that a device (just one)
# is connected to the desktop and that we should connect to
# it, 
#   use param 'device-is-connected' => 1
# if there are more than one devices and you want to connect to
# one of them, then 
#  use param 'device-serial' => <serial-of-device-to-connect>
# or
#  use param 'device-object' => <device object>
#        (of type Android::ElectricSheep::Automator::ADB::Device)
# or after instantiation with $obj->connect_device(...);
# and similarly for disconnect_device()
# NOTE: without connecting to a device you can not use

lib/Android/ElectricSheep/Automator.pm view on Meta::CPAN

# It returns 1 on failure, 0 on success
# it needs that connect_device() to have been called prior to this call
sub tap {
	my ($self, $params) = @_;
	$params //= {};

	my $parent = ( caller(1) )[3] || "N/A";
	my $whoami = ( caller(0) )[3];
	my $log = $self->log();
	my $verbosity = $self->verbosity();

	if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'connect_device()'." before calling this."); return undef }

	my (@position, $m);
	if( exists($params->{'position'}) && defined($m=$params->{'position'}) ){ 
		@position = ($m->[0], $m->[1]);
	} elsif( exists($params->{'bounds'}) && defined($m=$params->{'bounds'}) ){
		@position = ( int(($m->[1]->[0] + $m->[0]->[0])/2), int(($m->[1]->[1] + $m->[0]->[1])/2) );
	} else { $log->error("${whoami} (via $parent), line ".__LINE__." : error, input parameter 'position' (as ['x','y']) or 'bounds' (as [lefttopX,lefttopY],[bottomrightX,bottomrighY]) was not specified."); return 1 }

	my @cmd = ('input', 'tap', @position);
	if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
	my $res = $self->adb->shell(@cmd);
	if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening."); return 1 }
	if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }

	return 0; # success
}

# "type" the specified message into a text box expected to
# be found on the specified position.
# the message will be sanitised: spaces will be replaced with %s
# unicode in text is not supported.
# It returns 1 on failure, 0 on success
# it needs that connect_device() to have been called prior to this call
sub input_text {
	my ($self, $params) = @_;
	$params //= {};

	my $parent = ( caller(1) )[3] || "N/A";
	my $whoami = ( caller(0) )[3];
	my $log = $self->log();
	my $verbosity = $self->verbosity();

	if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'connect_device()'." before calling this."); return undef }

	my (@position, $m);
	if( exists($params->{'position'}) && defined($m=$params->{'position'}) ){ 
		@position = ($m->[0], $m->[1]);
	} elsif( exists($params->{'bounds'}) && defined($m=$params->{'bounds'}) ){
		@position = ( int(($m->[1]->[0] + $m->[0]->[0])/2), int(($m->[1]->[1] + $m->[0]->[1])/2) );
	} else { $log->error("${whoami} (via $parent), line ".__LINE__." : error, input parameter 'position' (as ['x','y']) or 'bounds' (as [lefttopX,lefttopY],[bottomrightX,bottomrighY]) was not specified."); return 1 }
	# optional text, else we send just '' (but we clicked on it)
	my $text = (exists($params->{'text'}) && defined($params->{'text'})) ? $params->{'text'} : '';

	# sanitise the text a bit
	# replace spaces with %s,
	# also newlines seem not to be supported so replaces these as well
	$text =~ s/[\n \t]/%s/g;

	# first tap on the text edit widget at the specified coordinates to get focus
	if( $self->tap($params) ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, failed to tap on the position of the recipient of the text input"); return 1 }
	usleep(0.8);

	# and send the text
	# adb shell input text 'hello%sworld'
	# does not support unicode and also spaces must be converted to %s (already have done this)
	my @cmd = ('input', 'text', $text);
	if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
	my $res = $self->adb->shell(@cmd);
	if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening."); return 1 }
	if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }

	return 0; # success
}

# returns 1 on failure, 0 on success
# it needs that connect_device() to have been called prior to this call
sub clear_input_field {
	my ($self, $params) = @_;
	$params //= {};

	my $parent = ( caller(1) )[3] || "N/A";
	my $whoami = ( caller(0) )[3];
	my $log = $self->log();
	my $verbosity = $self->verbosity();

	if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'connect_device()'." before calling this."); return undef }

	my (@position, $m);
	if( exists($params->{'position'}) && defined($m=$params->{'position'}) ){ 
		@position = ($m->[0], $m->[1]);
	} elsif( exists($params->{'bounds'}) && defined($m=$params->{'bounds'}) ){
		@position = ( int(($m->[1]->[0] + $m->[0]->[0])/2), int(($m->[1]->[1] + $m->[0]->[1])/2) );
	} else { $log->error("${whoami} (via $parent), line ".__LINE__." : error, input parameter 'position' (as ['x','y']) or 'bounds' (as [lefttopX,lefttopY],[bottomrightX,bottomrighY]) was not specified."); return 1 }

	# first tap on the text edit widget at the specified coordinates to get focus
	if( $self->tap($params) ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, failed to tap on the position of the recipient of the text input"); return 1 }
	usleep(0.8);

	# from: https://stackoverflow.com/questions/32433303/clear-edit-text-adb
	# the simplest way is input keycombination 113 29 && input keyevent 67
	# but may not work
	# then we try the lame way by erasing all chars one after the other
	# part1:
	my @cmd = ('input', 'keycombination', '113', '29');
	if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
	my $res = $self->adb->shell(@cmd);
	if( ! defined($res) || ($res->[0] != 0) || ($res->[1]=~/Error: Unknown command/) || ($res->[2]=~/Error: Unknown command/) ){
		$log->warn(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed which happens and now will try an alternative ...");
		# alternative/part1
		@cmd = ('input', 'keyevent', 'KEYCODE_MOVE_END');
		if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
		$res = $self->adb->shell(@cmd);
		if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening. Info: this is...
		if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed (Info: this is alternative part1), with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
		# alternative/part2
		# optional number of chars in the text-edit box, meaning how many
		# times to press backspace, default is here (250)
		# this is only needed for the second method (the failsafe)
		my $numchars = (exists($params->{'num-characters'}) && defined($params->{'num-characters'}) && ($params->{'num-characters'}=~/^\d+$/) ) ? $params->{'num-characters'} : 250;
		@cmd = ('input', 'keyevent', '--longpress', ('KEYCODE_DEL')x$numchars);
		if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
		$res = $self->adb->shell(@cmd);
		if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening. Info: this is...
		if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed (Info: this is alternative part1), with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
	} else {
		# part2
		# it worked, now on to part 2
		@cmd = ('input', 'keyevent', 'KEYCODE_DEL');
		if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }
		$res = $self->adb->shell(@cmd);
		if( ! defined $res ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed, got undefined result, most likely shell command did not run at all, this should not be happening. Info: this is...
		if( $res->[0] != 0 ){ $log->error(join(" ", @cmd)."\n${whoami} (via $parent), line ".__LINE__." : error, above shell command has failed (Info: this is part2), with:\nSTDOUT:\n".$res->[1]."\n\nSTDERR:\n".$res->[2]."\nEND."); return 1 }
	}
	return 0; # success
}

# Finds the running processes on device (using a `ps`),
# optionally can save the (parsed) `ps`
# results as JSON to the specified 'filename'.
# It needs that connect_device() to have been called prior to this call
# It returns undef on failure.
# On success, it returns a hash with keys
#   'raw' : the raw output of `ps` as a string
#   'perl': the parsed output of `ps` as a Perl hash of hashes,
#           each process
#           is represented by a hashref of items, e.g. %CPU
#           (basically all the items from the header of the `ps` command)
#           keyed on the full process command and its arguments (verbatim from `ps` output).
#   'json': the above perl data structure converted to JSON.
# NOTE: it uses _ps_parse_output() which is copied verbatim from System::Process
#       I wish they would load ps info from a string rather than running their own `ps`
sub list_running_processes {
	my ($self, $params) = @_;

	my $parent = ( caller(1) )[3] || "N/A";
	my $whoami = ( caller(0) )[3];

	my $log = $self->log();
	my $verbosity = $self->verbosity;

	if( ! $self->is_device_connected() ){ $log->error("${whoami} (via $parent), line ".__LINE__." : error, you need to connect a device to the desktop and ALSO explicitly call ".'device_connected()'." before calling this."); return undef }

	my $filename = exists($params->{'filename'}) && defined($params->{'filename'}) ? $params->{'filename'} : undef;
	my $extrafields = exists($params->{'extra-fields'}) && defined($params->{'extra-fields'}) ? $params->{'extra-fields'} : [];

	my ($FH, $tmpfilename) = tempfile(CLEANUP=>$self->cleanup);
	close $FH;

	# WARNING, you need to wake up the phone before dumping !!!!
	my $devicefile = File::Spec->catfile('/', 'data', 'local', 'tmp', $$.'.csv');

	my @cmd = ('ps', '-O', '%CPU', '-O', 'CPU');
	for my $ef (@$extrafields){ push @cmd, '-O', $ef }
	push @cmd, '-f', '-l', '>', $devicefile;

	if( $verbosity > 0 ){ $log->info("${whoami} (via $parent), line ".__LINE__." : sending command to adb: @cmd") }

lib/Android/ElectricSheep/Automator.pm view on Meta::CPAN

=item left

=item right

=back

=item * B<C<dt>>

denotes the time taken for the swipe
in milliseconds. The smaller its value the faster
the swipe. A value of C<100> is fast enough to swipe to
the next screen.

=back

It returns C<0> on success, C<1> on failure.

=head2 B<C<tap($params)>>

Emulates a "tap" at the specified location.
C<$params> is a HASH_REF which must contain one
of the following items:

=over 4

=item * B<C<position>>

should be an ARRAY_REF
as the C<X,Y> coordinates of the point to "tap".

=item * B<C<bounds>>

should be an ARRAY_REF of a bounding rectangle
of the widget to tap. Which contains two ARRAY_REFs
for the top-left and bottom-right coordinates, e.g.
C< [ [tlX,tlY], [brX,brY] ] >. This is convenient
when the widget is extracted from an XML dump of
the UI (see L</dump_current_screen_ui($params)>) which
contains exactly this bounding rectangle.

=back

It returns C<0> on success, C<1> on failure.

=head2 B<C<input_text($params)>>

It "C<types>" the specified text into the specified position,
where a text-input widget is expected to exist.
At first it taps at the widget's
location in order to get the focus. And then it enters
the text. You need to find the position of the desired
text-input widget by first getting the current screen UI
(using L</dump_current_screen_ui($params)>) and then using an XPath
selector to identify the desired widget by name/id/attributes.
See the source code of method L</send_message()> in file
C<lib/Android/ElectricSheep/Automator/Plugins/Apps/Viber.pm>
for how this is done for the message-sending text-input widget
of the Viber app.

C<$params> is a HASH_REF which must contain C<text>
and one of the two position (of the text-edit widget)
specifiers C<position> or C<bounds>:

=over 4

=item * B<C<text>>

the text to write on the text edit widget. At the
moment, this must be plain ASCII string, not unicode.
No spaces are accepted.
Each space character must be replaced with C<%s>.

=item * B<C<position>>

should be an ARRAY_REF
as the C<X,Y> coordinates of the point to "tap" in order
to get the focus of the text edit widget, preceding the
text input.

=item * B<C<bounds>>

should be an ARRAY_REF of a bounding rectangle
of the widget to tap, in order to get the focus, preceding
the text input. Which contains two ARRAY_REFs
for the top-left and bottom-right coordinates, e.g.
C< [ [tlX,tlY], [brX,brY] ] >. This is convenient
when the widget is extracted from an XML dump of
the UI (see L</dump_current_screen_ui($params)>) which
contains exactly this bounding rectangle.

=back

It returns C<0> on success, C<1> on failure.

=head2 B<C<clear_input_field($params)>>

It clears the contents of a text-input widget
at specified location.

There are several ways to do this. The simplest way
(with C<keycombination>) does not work in some
devices, in which case a failsafe way is employed
which deletes characters one after the other for
250 times. 

C<$params> is a HASH_REF which must contain
one of the two position (of the text-edit widget)
specifiers C<position> or C<bounds>:

=over 4

=item <B<C<position>>

should be an ARRAY_REF
as the C<X,Y> coordinates of the point to "tap" in order
to get the focus of the text edit widget, preceding the
text input.

=item B<C<bounds>>

should be an ARRAY_REF of a bounding rectangle
of the widget to tap, in order to get the focus, preceding
the text input. Which contains two ARRAY_REFs
for the top-left and bottom-right coordinates, e.g.
C< [ [tlX,tlY], [brX,brY] ] >. This is convenient
when the widget is extracted from an XML dump of
the UI (see L</dump_current_screen_ui($params)>) which
contains exactly this bounding rectangle.

=item B<C<num-characters>>

how many times to press the backspace? Default is 250!
But if you know the length of the text currently at
the text-edit widget then enter this here.

=back

It returns C<0> on success, C<1> on failure.

=head2 B<C<home_screen()>>

Go to the "home" screen.

It returns C<0> on success, C<1> on failure.


=head2 B<C<wake_up()>>

"Wake" up the device.

It returns C<0> on success, C<1> on failure.


=head2 B<C<next_screen()>>

Swipe to the next screen (on the right).

It returns C<0> on success, C<1> on failure.


=head2 B<C<previous_screen()>>

Swipe to the previous screen (on the left).

It returns C<0> on success, C<1> on failure.


=head2 B<C<navigation_menu_back_button()>>

Press the "back" button which is the triangular
button at the left of the navigation menu at the bottom.

It returns C<0> on success, C<1> on failure.

=head2 B<C<navigation_menu_home_button()>>

Press the "home" button which is the circular
button in the middle of the navigation menu at the bottom.

It returns C<0> on success, C<1> on failure.

=head2 B<C<navigation_menu_overview_button()>>

Press the "overview" button which is the square
button at the right of the navigation menu at the bottom.

It returns C<0> on success, C<1> on failure.

=head2 B<C<apps()>>

It returns a HASH_REF containing all the
packages (apps) installed on the device
keyed on package name (which is like C<com.android.settings>.
The list of installed apps is populated either

lib/Android/ElectricSheep/Automator.pm view on Meta::CPAN


=head2 B<C<electric-sheep-install-app>>

Install an APK file onto the device, passing extra installation
parameters C<-r> (for re-install) and C<-g> (for granting permissions),

    electric-sheep-install-app --apk-filename test.apk -p '-r' -p '-g' --configfile config/myapp.conf --device Pixel_2_API_30_x86_


=head2 B<C<electric-sheep-viber-send-message.pl>>

Send a message using the Viber app.

    electric-sheep-viber-send-message.pl --message 'hello%sthere' --recipient 'george' --configfile config/myapp.conf --device Pixel_2_API_30_x86_

This one saves a lot of debugging information to C<debug> which can be used to
deal with special cases or different versions of Viber:

    electric-sheep-viber-send-message.pl --outbase debug --verbosity 1 --message 'hello%sthere' --recipient 'george' --configfile config/myapp.conf --device Pixel_2_API_30_x86_


=head1 TESTING

The normal tests under the C<t/> directory, initiated with C<make test> command,
are quite limited in scope because they do not assume
a connected device. That is, they do not check any
functions which require interaction with a connected
device.

The I<live tests> under the C<xt/live> directory, initiated with
C<make livetest> command, require
an Android emulator or real device (the latter B<is not recommended>)
connected to your desktop computer on which you are doing the testing.
Note that testing
with your smartphone is not a good idea, please do not do this,
unless it is some phone which you do not store important data.
It is very easy to get an emulated Android device running on any OS.

So, prior to C<make livetest> make sure you have an android
emulator up and running with, for example,
C<emulator -avd Pixel_2_API_30_x86_> . See section
L<Android Emulators> for how to install, list and run them
buggers.

At least one of the I<author tests> under the C<xt/author> directory,
initiated with
C<make authortest> command, require an APK file (to be installed on the connected
device) which is quite large and it is not included in the distribution
bundle of this module. Anyway, it is not a good idea to install
an unknown APK to your device. But if you want to make this test
then pull an APK of an existing app on your connected device
with L<electric-sheep-pull-app-apk.pl> and point the test file
to this APK.

Testing will not send any messages via the device's apps.
E.g. the plugin L<Android::ElectricSheep::Automator::Plugins::Apps::Viber>
will not send a message via Viber but it will mock it.

The live tests will sometimes fail because, so far,
something unexpected happened in the device. For example,
in testing sending input text to a text-edit widget,
the calendar will be opened and a new entry will be added
and its text-edit widget will be targeted. Well, sometimes
the calendar app will give you some notification
on startup and this messes up with the focus.
Other times, the OS will detect that some app is taking too
long to launch and pops up a notification about
"I<something is not responding, shall I close it>".
This steals the focus and sometimes it causes
the tests to fail.

=head1 PREREQUISITES

=head2 Android Studio

This is not a prerequisite but it is
highly recommended to install it
(from L<https://developer.android.com/studio>)
on your desktop computer because it contains
all the executables you will need,
saved in a well documented file system hierarchy,
which can then be accessed from the command line.
You will not be using the IDE or anything, just
the accompaniying binaries and libraries it comes with.

Additionally, Android Studio offers possibly the
easiest way to create Android Virtual Devices (AVD) which emulate
an Android phone of various specifications, phone models and sizes,
API levels, etc.
I mention this because one can install apps
on an AVD and control them from your desktop
as long as you are able to receive sms verification
codes from a real phone. Perhaps you will need an Android
emulator image which comes with Google Play Services,
if you are installing apps from their store.
This is great for
experimenting without plugging in your real
smartphone on your desktop.

The bottom line is that by installing Android Studio,
you have all the executables you need for running things
from the command line and, additionally, you have
the easiest way for creating Android
Virtual Devices, which emulate Android devices: phones,
tablets, automotive displays. Once you have this set up, you
will not need to open Android Studio ever again unless you
want to update your kit. All the functionality
will be accessible from the command line.

=head2 ADB

Android Debug Bridge (ADB) is the program
which communicates with your smartphone or
an Android Virtual Device from
your desktop (Linux, osx and the unnamed C<0$>).

If you do not want to install Android Studio, the C<adb> executable
is included in the package called
"Android SDK Platform Tools" available from
the Android official site, here:
L<https://developer.android.com/tools/releases/platform-tools#downloads>

You will need the C<adb> executable to be on your path

( run in 3.028 seconds using v1.01-cache-2.11-cpan-cdf2f3d4e48 )