view release on metacpan or  search on metacpan

lib/App/CSE/Command/Index.pm  view on Meta::CPAN

         );

  # Full text types.
  my $path_type = Lucy::Plan::FullTextType->new(analyzer => $ft_anal, sortable => 1, boost => 2.0);
  my $body_type = Lucy::Plan::FullTextType->new(analyzer => $ft_anal, highlightable => 1 , boost => 1.0);
  my $declare_type = Lucy::Plan::FullTextType->new( analyzer => $decl_anal, highlightable => 1 , boost => 4.0 );
  my $call_type    = Lucy::Plan::FullTextType->new( analyzer => $decl_anal, highlightable => 1 , boost => 3.0 );


  # String type
  my $plain_sortable_string = Lucy::Plan::StringType->new( sortable => 1 );

  # Plain string type
  my $plain_string = Lucy::Plan::StringType->new();


  $schema->spec_field( name => 'content' , type => $body_type );
  $schema->spec_field( name => 'decl' , type => $declare_type );
  $schema->spec_field( name => 'call' , type => $call_type );
  $schema->spec_field( name => 'dir'  , type => $plain_sortable_string );
  $schema->spec_field( name => 'mtime' , type => $plain_sortable_string);
  $schema->spec_field( name => 'mime' , type => $plain_sortable_string );
  $schema->spec_field( name => 'path' , type => $path_type );
  $schema->spec_field( name => 'path.raw' , type => $plain_string );


  ## Ok Schema has been built
  $LOGGER->info("Building index ".$index_dir);
  my $indexer = Lucy::Index::Indexer->new(schema => $schema,
                                          index => $index_dir,
                                          create => 1,
                                         );

  $LOGGER->info("Indexing files from ".$self->dir_index());

  my $dir_index = $self->dir_index();

  my $cse = $self->cse();

  ## Wrapper to build File::find wanter subs
  my $wanted_wrapper = sub{
    my ($wrapped) = @_;

    return sub{
      my $file_name = $File::Find::name;

      my $valid = $cse->is_file_valid($file_name , {
                                                    on_hidden => sub{
                                                      $File::Find::prune = 1;
                                                      return undef;
                                                    },
                                                    on_skip => sub{
                                                      $File::Find::prune = 1;
                                                      return undef;
                                                    }
                                                   });
      unless( $valid ){
        return;
      }

      my $stat = File::stat::stat($file_name.'');
      &$wrapped($file_name, $stat);
    };
  };



  $LOGGER->info("Estimating size. Please wait");
  my $ESTIMATED_SIZE = 0;
  File::Find::find({ wanted => &$wanted_wrapper(sub{
                                                  my ($file_name, $stat) = @_;
                                                  unless( -d $file_name ){
                                                    $ESTIMATED_SIZE += $stat->size();
                                                  }
                                                }),
                     no_chdir => 1,
                     follow => 0,
                   }, $dir_index );

  my $PROGRESS_BAR =  Term::ProgressBar->new({name  => 'Indexing',
                                              count => $ESTIMATED_SIZE,
                                              ETA   => 'linear',
                                              silent => ! $self->cse()->interactive(),
                                             });

  my $START_TIME = Time::HiRes::time();
  my $NUM_INDEXED = 0;
  my $SIZE_LOOKEDAT = 0;
  my $TOTAL_SIZE = 0;


  my $wanted = sub{
    my ($file_name, $stat) = @_;


    unless( -d $file_name.'' ){
      $SIZE_LOOKEDAT += $stat->size();
      $PROGRESS_BAR->update($SIZE_LOOKEDAT);
    }


    my $mime_type = $cse->valid_mime_type($file_name);
    unless( $mime_type ){
      return;
    }

    my $file_class = App::CSE::File->class_for_mime($mime_type, $file_name.'');
    unless( $file_class ){
      return;
    }

    ## Build a file instance.
    my $file = $file_class->new({cse => $cse,
                                 mime_type => $mime_type,
                                 stat => $stat,
                                 file_path => $file_name.'' })->effective_object();


    $LOGGER->debug("Indexing ".$file->file_path().' as '.$file->mime_type());

    my $content = $file->content();