view release on metacpan or  search on metacpan

createdatabase.PL  view on Meta::CPAN

			flush_queue($dbh, $redis, $mongodb, $berkeley_db);
			$inserts = 0;
		}
	}

	$fh->close();

	# Import the counties
	$filename = 'lib/Geo/Coder/Free/OpenAddresses/databases/counties.txt';
	$| = 1;
	printf "%-70s\r", $filename;
	print "\n" if(DEBUG);
	$| = 0;

	$csv = Text::CSV->new({
		sep_char => "\t",
		allow_loose_quotes => 1,
		blank_is_undef => 1,
		empty_is_undef => 1,
		binary => 1,
		# Workaround for https://github.com/openaddresses/openaddresses/issues/3905
		# escape_char => '\\',	# Put back once issue 3905 has been fixed
	});

	$fh = File::Open::NoCache::ReadOnly->new($filename);

	$fin = $fh->fd();

	# open($fin, '<', $filename);

	$csv->column_names($csv->getline($fin));

	while(my $row = $csv->getline_hr($fin)) {
		# print __LINE__, ': ', Data::Dumper->new([\$row])->Dump();
		my $state = $state_fips{$row->{'State FIPS'}};
		die $row->{'Name'} unless(defined($state));
		my $county = uc($row->{'Name'});
		$county =~ s/\s+COUNTY$//;
		$county =~ s/'/''/g;	# O'Brien County, IA
		my %columns = (
			'COUNTRY' => 'US',
			'STATE' => $state,
			'COUNTY' => $county,
			'LAT' => $row->{'Latitude'},
			'LON' => $row->{'Longitude'},
		);
		# print "$zip => $query\n";
		$inserts += insert($dbh, $redis, $mongodb, $berkeley_db, 1, %columns);
		if($inserts >= MAX_INSERT_COUNT) {
			flush_queue($dbh, $redis, $mongodb, $berkeley_db);
			$inserts = 0;
		}
	}
	$fh->close();
}

my $current_state;
my $current_country;
# my %whosonfirst;

my $ua = LWP::UserAgent::Throttled->new(keep_alive => 1);
$ua->conn_cache(LWP::ConnCache->new());
$ua->throttle({ 'api.zippopotam.us' => 1 });
$ua->env_proxy(1);

$filename = 'downloads/allCountries.txt';

# Import the Maxmind databases
if(($force_flag || ((!(DEBUG&DEBUG_KNOWN_PLACES)) && (-r $filename))) &&
   # open(my $fin, '<', 'lib/Geo/Coder/Free/GeoNames/databases/allCountries.txt') &&
   # open(my $fin1, '<', 'lib/Geo/Coder/Free/MaxMind/databases/admin1.db') &&
   # open(my $fin2, '<', 'lib/Geo/Coder/Free/MaxMind/databases/admin2.db')) {
   (my $all_countries = File::Open::NoCache::ReadOnly->new($filename)) &&
   (!(DEBUG&(DEBUG_OSM|DEBUG_WOF|DEBUG_GET_WOF|DEBUG_DR5HN))) &&
   (my $admin1 = File::Open::NoCache::ReadOnly->new('lib/Geo/Coder/Free/MaxMind/databases/admin1.db')) &&
   (my $admin2 = File::Open::NoCache::ReadOnly->new('lib/Geo/Coder/Free/MaxMind/databases/admin2.db'))) {
	my $fin = $all_countries->fd();
	aio_readahead($fin, 0, AIO_READAHEAD_SIZE) if($^O ne 'solaris');
	my $fin1 = $admin1->fd();
	aio_readahead($fin1, 0, AIO_READAHEAD_SIZE) if($^O ne 'solaris');
	my $fin2 = $admin2->fd();
	aio_readahead($fin2, 0, AIO_READAHEAD_SIZE) if($^O ne 'solaris');

	$| = 1;
	printf "%-70s\r", $filename;
	$| = 0;
	print "\n" if(MAX_INSERT_COUNT == 1);

	# $csv->column_names(['key', 'name', 'asciiname', 'lat', 'long', 'skip1', 'skip2', 'country', 'state', 'county', 'skip3', 'skip4', 'skip5', 'skip6', 'skip7', 'skip8']);

	my $csv = Text::CSV->new({
		sep_char => "\t",
		allow_loose_quotes => 1,
		blank_is_undef => 1,
		empty_is_undef => 1,
		binary => 1,
		# Workaround for https://github.com/openaddresses/openaddresses/issues/3905
		# escape_char => '\\',	# Put back once issue 3905 has been fixed
	});

	$csv->column_names(['concatenated_codes', 'name', 'asciiname', 'geonameId']);

	my %admin1;

	while(my $row = $csv->getline_hr($fin1)) {
		$admin1{$row->{'concatenated_codes'}} = $row->{'asciiname'};
	}

	$admin1->close();

	my %admin2;

	while(my $row = $csv->getline_hr($fin2)) {
		$admin2{$row->{'concatenated_codes'}} = $row->{'asciiname'};
	}

	$admin2->close();

	my $inserts = 0;
	my $offset = AIO_READAHEAD_SIZE;