App-CSE

 view release on metacpan or  search on metacpan

lib/App/CSE/Command/Check.pm  view on Meta::CPAN

  }

  # The directory is there. Check it is a valid lucy index.
  my $lucy = eval{ my $l = Lucy::Search::IndexSearcher->new( index => $index_dir );
                   $l->get_reader();
                   $l->get_schema();
                   $l;
                 };
  unless( $lucy ){
    my $err = $@;
    $LOGGER->error($self->cse->colorizer->colored("The index $index_dir is not a valid lucy index.", 'red bold'));
    $LOGGER->debug("Lucy error: $err");
    return 1;
  }

  my $dirty_str = '';
  my $dirty_hash = $self->cse()->dirty_files();
  if( my $ndirty = scalar( keys %$dirty_hash ) ){
    $dirty_str  = ' '.$ndirty.' dirty files - run cse update to clean them';
  }

lib/App/CSE/Command/Check.pm  view on Meta::CPAN

All the files will be considered to be application/octet-stream at index time, making the search useless.

The shared-mime-info package is available from http://freedesktop.org/

On linux:
  Check your  package manager

On OSX:
  brew install shared-mime-info

|, 'yellow bold'));
      return 1;
  }

  # Check some watcher.
  if( my $watcher_pid = $self->cse->index_meta()->{'watcher.pid'} ){
    $LOGGER->info("Dir watcher PID=".$watcher_pid);
    ( $watcher_pid ) = ( $watcher_pid =~ /(\d+)/ );
    if( kill(0 , $watcher_pid ) ){
      $LOGGER->info("Watcher is Running");
    }else{

lib/App/CSE/Command/Index.pm  view on Meta::CPAN

  rename $index_dir , $self->cse->index_dir()->stringify();

  $self->cse->index_meta->{index_time} = DateTime->now()->iso8601();
  # Also inject the right version
  $self->cse->index_meta->{version} = $self->cse->version();;

  $self->cse->save_index_meta();

  my $END_TIME = Time::HiRes::time();

  $LOGGER->info($self->cse->colorizer->colored("Index is ".$self->cse()->index_dir()->stringify(), 'green bold'));
  $LOGGER->info("$NUM_INDEXED files ($TOTAL_SIZE Bytes) indexed in ".sprintf('%.03f', ( $END_TIME - $START_TIME )).' seconds');

  return 0;
}

__PACKAGE__->meta->make_immutable();

=head1 NAME

App::CSE::Command::Index - Indexes a directory

lib/App/CSE/Command/Search.pm  view on Meta::CPAN

    @rules = (
              Lucy::Search::SortRule->new( field => 'path' ),
              Lucy::Search::SortRule->new( type => 'score'),
             );
  }elsif( $self->sort_str() eq 'mtime' ){
    @rules = (
              Lucy::Search::SortRule->new( field => 'mtime' , reverse => 'true' ),
              Lucy::Search::SortRule->new( field => 'path' ),
             );
  }else{
    $LOGGER->error($self->cse->colorizer->colored("Unknown sort mode ".$self->sort_str().". Falling back to 'score'", 'red bold'));
  }

  return Lucy::Search::SortSpec->new(rules => \@rules );
}


sub _build_highlighter{
  my ($self) = @_;

  ## Note that this only builds a content highlighter.

lib/App/CSE/Command/Search.pm  view on Meta::CPAN

sub options_specs{
  return [ 'offset|o=i', 'num|n=i', 'sort|s=s' , 'reverse|r' ];
}

my %legit_sort = ( 'score' => 1,  'path' => 1 , 'mtime' => 1 );

sub _build_sort_str{
  my ($self) = @_;
  my $sort_str =  $self->cse()->options()->{sort} || 'score';
  unless( $legit_sort{$sort_str} ){
    $LOGGER->error($self->cse->colorizer->colored("Unknown sort mode ".$sort_str.". Falling back to 'score'", 'red bold'));
    return 'score';
  }

  my $perl_version = $];
  if( $perl_version >= 5.016 && $sort_str ne 'score' ){
    $LOGGER->warn($self->cse->colorizer->colored("A bug in Lucy doesn't allow this version of Perl($perl_version) to take sort mode (".$sort_str.") into account for now.", 'yellow bold'));
    return 'score'
  }


  return $sort_str;
}

sub _build_offset{
  my ($self) = @_;
  return $self->cse()->options->{offset} || 0;

lib/App/CSE/Command/Search.pm  view on Meta::CPAN

sub _build_hits{
  my ($self) = @_;

  $LOGGER->info("Searching for '".$self->filtered_query()->to_string()."'");


  {
    my $this_version = $self->cse()->version();
    my $index_version = $self->cse->index_meta->{version};
    unless( $this_version eq $index_version ){
      $LOGGER->warn($self->cse()->colorizer->colored("Index version is too old ($index_version) for this program ($this_version). Please consider re-indexing", 'yellow bold'));
    }
  }

  my $perl_version = $];

  my $hits = $self->searcher->hits( query => $self->filtered_query(),
                                    offset => $self->offset(),
                                    num_wanted => $self->num(),
                                    ## This segfaults on perl 16 and 18 :(
                                    ( $perl_version < 5.016 ) ? ( sort_spec => $self->sort_spec() ) : ()

lib/App/CSE/Command/Search.pm  view on Meta::CPAN

  my $colored = sub{ $colorizer->colored(@_); };

  unless( $self->query_str() ){
    $LOGGER->warn(&$colored("Missing query. Do cse help" , 'red'));
    return 1;
  }

  # Check the index.
  my $check = App::CSE::Command::Check->new({ cse => $self->cse() });
  if( $check->execute() ){
    $LOGGER->info(&$colored("Rebuilding the index..", 'green bold'));
    my $index_cmd = App::CSE::Command::Index->new( { cse => $self->cse() });
    if( $index_cmd->execute() ){
      $LOGGER->error(&$colored("Building index failed", 'red'));
      return 1;
    }
  }
  my $start_time = Time::HiRes::time();

  ## This will trigger a search. Look at _build_hits
  my $hits = $self->hits();

  my $highlighter = $self->highlighter();

  $LOGGER->info(&$colored('Hits: '. $self->offset().' - '.( $self->offset() + $self->num() - 1).' of '.$hits->total_hits().' sorted by '.$self->sort_str(), 'green bold')."\n\n");

  while ( my $hit = $hits->next ) {

    my $excerpt = $highlighter->create_excerpt($hit);

    my $star = '';
    if( my $stat = File::stat::stat( $hit->{path} ) ){
      if( $hit->{mtime} lt DateTime->from_epoch(epoch => $stat->mtime())->iso8601() ){
        $star = &$colored('*' , 'red bold');
        # Mark the file as dirty.
        $self->cse()->dirty_files()->{$hit->{'path.raw'}} = 1;
      }
    }else{
        # No stat. File is gone (consider dirty)
        $star = &$colored('-' , 'red bold');
        $self->cse()->dirty_files()->{$hit->{'path.raw'}} = 1;
    }

    $LOGGER->trace("Score: ".$hit->get_score());

    my $hit_str = &$colored($hit->{path}.'', 'magenta bold').' ('.$hit->{mime}.') ['.$hit->{mtime}.$star.']'.&$colored(':', 'cyan bold');
    $hit_str .= q|
|.( $excerpt || substr($hit->{content} || '' , 0 , 100 ) ).q|

|;


    $LOGGER->info($hit_str);
  }

  my $stop_time = Time::HiRes::time();

lib/App/CSE/Command/Unwatch.pm  view on Meta::CPAN

  my $previous_pid = $cse->index_meta()->{'watcher.pid'};
  unless( $previous_pid ){
    $LOGGER->warn("No watcher PID in ".$cse->index_meta_file()." - nothing to do");
    return 1;
  }

  # A previous pid should be a number.
  ( $previous_pid ) = ( $previous_pid =~ /(\d+)/ );
  unless( kill(0, $previous_pid ) ){
    $LOGGER->warn(&$colored("Previous watcher (PID=".$previous_pid.") is already dead. Nothing to do",
                            "yellow bold"));
    delete $self->cse->index_meta()->{'watcher.pid'};
    delete $self->cse->index_meta()->{'watcher.start'};
    $self->cse->save_index_meta();
    return 1;
  }


  kill 15, $previous_pid;
  my $n_attempts = 4;
  my $wait_time = 1;
  while( $n_attempts-- ){
    sleep($wait_time);
    unless( kill(0 , $previous_pid ) ){
      $LOGGER->info(&$colored("Watcher PID=$previous_pid has terminated gracefully" , "green bold"));
      delete $self->cse->index_meta()->{'watcher.pid'};
      delete $self->cse->index_meta()->{'watcher.start'};
      $self->cse->save_index_meta();
      return 0;
    }
    $wait_time <<= 1;
  }

  $LOGGER->error(&$colored("PID=$previous_pid refuses to terminate gracefully. Attempting to kill 9 it", "red bold"));
  kill 9 , $previous_pid;
  return 1;
}

__PACKAGE__->meta->make_immutable();

lib/App/CSE/Command/Update.pm  view on Meta::CPAN

  my ($self) = @_;

  my $colorizer = $self->cse()->colorizer();

  my $colored = sub{ $colorizer->colored(@_);};

  # Check the index.
  # Re-index if nothing is there.
  my $check = App::CSE::Command::Check->new({ cse => $self->cse() });
  if( $check->execute() ){
    $LOGGER->info(&$colored("Rebuilding the index..", 'green bold'));
    my $index_cmd = App::CSE::Command::Index->new( { cse => $self->cse() });
    if( $index_cmd->execute() ){
      $LOGGER->error(&$colored("Building index failed", 'red'));
      return 1;
    }
    # Nothing else to do.
    return 0;
  }

  # Right time to reindex dirty files.
  my @dirty_files = sort keys %{$self->cse->dirty_files()};
  unless( @dirty_files ){
    $LOGGER->info(&$colored("No dirty files", 'green bold'));
    return 0;
  }

  # Build an indexer.
  my $searcher = Lucy::Search::IndexSearcher->new( index => $self->cse()->index_dir().'' );
  my $indexer =  Lucy::Index::Indexer->new( schema => $searcher->get_schema(),
                                            index => $self->cse()->index_dir().'' );

  my $NFILES = 0;
  foreach my $dirty_file ( @dirty_files ){

lib/App/CSE/Command/Update.pm  view on Meta::CPAN

                       $content ? ( content => $content ) : ()
                      });
    # Remove from the dirty files hash
    delete $self->cse()->dirty_files()->{$dirty_file};
    $NFILES++;
  }

  # Commit and save that.
  $indexer->commit();
  $self->cse()->save_dirty_files();
  $LOGGER->info(&$colored('Re-indexed '.$NFILES.' files' ,'green bold'));
  return 0;
}

__PACKAGE__->meta->make_immutable();

lib/App/CSE/Command/Watch.pm  view on Meta::CPAN

  my $colorizer = $self->cse()->colorizer();
  my $colored = sub{ $colorizer->colored(@_);};

  my $cse = $self->cse();

  # Check the index.
  # Re-index if nothing is there.
  my $check = App::CSE::Command::Check->new({ cse => $cse });
  if( $check->execute() ){
    $LOGGER->info(&$colored("Rebuilding the index..", 'green bold'));
    my $index_cmd = App::CSE::Command::Index->new( { cse => $cse });
    if( $index_cmd->execute() ){
      $LOGGER->error(&$colored("Building index failed", 'red'));
      return 1;
    }
  }

  if( my $previous_pid = $cse->index_meta()->{'watcher.pid'} ){
    # A previous pid should be a number.
    ( $previous_pid ) = ( $previous_pid =~ /(\d+)/ );
    if( kill(0, $previous_pid ) ){
      $LOGGER->error(&$colored("Previous watcher (PID=".$previous_pid.") is still running. Try cse unwatch first",
                               "red bold"));
      return 1;
    }
  }





  my $direct_child = fork();
  confess("Cannot fork() a direct child: $!") unless defined $direct_child;

lib/App/CSE/Command/Watch.pm  view on Meta::CPAN

    return 0;
  }

  my $pid = fork();
  confess("Cannot fork() a worker child: $!") unless defined $pid;

  if( $pid ){
    $cse->index_meta()->{'watcher.pid'} = $pid;
    $cse->index_meta()->{'watcher.started'} = DateTime->now()->iso8601();
    $cse->save_index_meta();
    $LOGGER->info(&$colored("PID=$pid :  Watching for changes in ".$self->dir_index().", updating ".$cse->index_dir(), "green bold"));
    exit(0);
  }

  my $deamon_log = q|log4perl.rootLogger                = INFO, SYSLOG
log4perl.appender.SYSLOG           = Log::Dispatch::Syslog
log4perl.appender.SYSLOG.min_level = debug
log4perl.appender.SYSLOG.ident     = cse V|.$cse->version().q|[|.$$.q|]
log4perl.appender.SYSLOG.facility  = daemon
log4perl.appender.SYSLOG.layout    = Log::Log4perl::Layout::SimpleLayout
|;

lib/App/CSE/File.pm  view on Meta::CPAN

  $file_name ||= 'unknown_file_name';

  my $half_camel = $mime_type; $half_camel =~ s/\W/_/g;
  my $file_class_name = 'App::CSE::File::'.String::CamelCase::camelize($half_camel);
  # Protect against unsecure class name
  ( $file_class_name ) = ( $file_class_name =~ /^([\w:]+)$/ );
  my $file_class = eval{ Class::Load::load_class($file_class_name); };
  unless( $file_class ){
    # A bit dirty, but well..
    $LOGGER->debug(App::CSE->instance()->colorizer->colored("No class '$file_class_name' for mimetype $mime_type ($file_name)",'red bold'));
    return undef;
  }
  return $file_class;
}

__PACKAGE__->meta->make_immutable();

__END__

=head1 NAME