From: Llewellyn Marshall Date: Fri, 5 Aug 2022 18:37:44 +0000 (-0400) Subject: bulk distance calculation X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=3e3a44ceb332e07d9ba4b6d6e70298948d7f6b0a;p=working%2FEvergreen.git bulk distance calculation --- diff --git a/Open-ILS/src/eg2/src/app/staff/admin/server/org-unit-shipping-hub-distance.component.ts b/Open-ILS/src/eg2/src/app/staff/admin/server/org-unit-shipping-hub-distance.component.ts index 17d0050966..b7fbcdd861 100644 --- a/Open-ILS/src/eg2/src/app/staff/admin/server/org-unit-shipping-hub-distance.component.ts +++ b/Open-ILS/src/eg2/src/app/staff/admin/server/org-unit-shipping-hub-distance.component.ts @@ -108,7 +108,8 @@ export class OrgUnitShippingHubDistanceComponent implements OnInit { this.calculating = true; this.net.request( 'open-ils.vicinity-calculator', - 'open-ils.vicinity-calculator.build-distance-matrix' + 'open-ils.vicinity-calculator.build-distance-matrix', + this.auth.token() ).subscribe( n => {this.calculating = false; location.reload();}, err => {alert('API failed to calculate ' + err);this.calculating = false;} diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Geo.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Geo.pm index 4b1e28da49..ddf2a16fa4 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Geo.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Geo.pm @@ -6,12 +6,16 @@ use warnings; use OpenSRF::AppSession; use OpenILS::Application; use base qw/OpenILS::Application/; +use List::MoreUtils qw(natatime); +use List::Util qw(min); use OpenSRF::Utils::SettingsClient; use OpenILS::Utils::CStoreEditor qw/:funcs/; use OpenILS::Utils::Fieldmapper; use OpenSRF::Utils::Cache; use OpenILS::Application::AppUtils; +use Data::Dumper; +use JSON::XS; my $U = "OpenILS::Application::AppUtils"; use OpenSRF::Utils::Logger qw/$logger/; @@ -123,6 +127,163 @@ __PACKAGE__->register_method( } ); +sub _post_request { + my ($bing, $uri, $form, $json_coder) = @_; + my $json = $json_coder->encode($form); + return unless $uri; + $logger->info($uri); + $logger->info($form); + my $res = $bing->{response} = $bing->ua->post($uri,'Content-Length' => 3500,'Content-Type' => 'application/json',Content => $json); + unless($res->is_success){ + $logger->error("API ERROR\n"); + return; + } + + my @error = split /\n/, $res->decoded_content; + foreach(@error){ + $logger->error($_); + } + + # Change the content type of the response from 'application/json' so + # HTTP::Message will decode the character encoding. + $res->content_type('text/plain'); + + my $content = $res->decoded_content; + return unless $content; + my $data= eval { $json_coder->decode($res->decoded_content) }; + return unless $data; + my @results = @{ $data->{resourceSets}[0]{resources} || [] }; + return wantarray ? @results : $results[0]; +} + +sub calculate_bulk_driving_distance { + my ($self, $conn, $auth, $origin_array, $destination_array) = @_; + + return new OpenILS::Event("BAD_PARAMS", "desc" => "Missing coordinates") unless $origin_array; + return new OpenILS::Event("BAD_PARAMS", "desc" => "Missing coordinates") unless $destination_array; + + my $e = new_editor(xact => 1, authtoken=>$auth); + return $e->die_event unless $e->checkauth; + # get the requestor's org unit + my $org = $e->requestor->ws_ou; + my $use_geo = $e->retrieve_config_global_flag('opac.use_geolocation'); + $use_geo = ($use_geo and $U->is_true($use_geo->enabled)); + return new OpenILS::Event("GEOCODING_NOT_ENABLED") unless ($U->is_true($use_geo)); + + return new OpenILS::Event("BAD_PARAMS", "desc" => "No org ID supplied") unless $org; + my $service_id = $U->ou_ancestor_setting_value($org, 'opac.geographic_location_service_for_address'); + return new OpenILS::Event("GEOCODING_NOT_ALLOWED") unless ($U->is_true($service_id)); + + my $service = $e->retrieve_config_geolocation_service($service_id); + return new OpenILS::Event("GEOCODING_NOT_ALLOWED") unless ($U->is_true($service)); + + my $geo_coder = _create_geocoder($service); + if (!$geo_coder) { + return OpenILS::Event->new('GEOCODING_LOCATION_NOT_FOUND'); + } + + my @results; + + if ($service->service_code eq 'Bing') { + my @origins; + my @destinations; + my $uri = URI->new("https://dev.virtualearth.net/REST/v1/Routes/DistanceMatrix?key=".$service->api_key); + + # get the data into the right form for our request + foreach(@{$origin_array}){ + my %ocoord; + $ocoord{'latitude'} = $_->[0]; + $ocoord{'longitude'} = $_->[1]; + push(@origins, \%ocoord); + } + $logger->info(Dumper(\@origins)); + + foreach(@{$destination_array}){ + my %dcoord; + $dcoord{'latitude'} = $_->[0]; + $dcoord{'longitude'} = $_->[1]; + push(@destinations, \%dcoord); + } + $logger->info(Dumper(\@destinations)); + + # find out how many coords we can process per request. + my $budget = int(2500 / scalar(@origins)); + $logger->debug("data being chunked into ".$budget.".\n"); + my $it = natatime $budget, @origins; + my $real_index = 0; + my $json_coder = JSON::XS->new->convert_blessed; + + while (my @coords = $it->()) + { + my %content = ( + origins => \@coords, + destinations => \@destinations, + travelMode => "driving", + timeUnit => "minute", + distanceUnit => "km" + ); + $logger->info("Hash for JSON: ".Dumper(\%content)); + my $rest_req = _post_request($geo_coder,$uri,\%content,$json_coder); + # print Dumper $rest_req; + #calculate the distance matrix for this chunk of origins. + $logger->info("results from post: ".Dumper($rest_req)); + my @distance_matrix = eval{$rest_req->{results}}; + if(@distance_matrix){ + for my $ref (@distance_matrix) { + for (@$ref){ + my %dist; + $dist{origin} = $_->{originIndex} + $real_index; + $dist{destination} = $_->{destinationIndex}; + $dist{distance} = $_->{travelDistance}; + push(@results,\%dist); + } + } + + $logger->info("Information from server: ".Dumper(\@results)); + $real_index += min($budget,scalar(@coords)); + print("setting index to ".$real_index."\n"); + } + } + + return \@results; + } else { + $logger->info($service->service_code." can not get driving distance. Reverting to as-the-crow-flies."); + # if geocoder can't do driving distance just get as-the-crow-flies + my $index = 0; + foreach(@{$origin_array}){ + my $pointA = $_; + my $dindex = 0; + foreach(@{$destination_array}){ + my $pointB = $_; + my $d = calculate_distance($self, $conn, $pointA, $pointB); + my %dist; + $dist{origin} = $index; + $dist{destination} = $dindex; + $dist{distance} = $d; + push(@results,\%dist); + $dindex++; + } + $index++; + } + + return \@results; + } + return []; +} + +__PACKAGE__->register_method( + method => "calculate_bulk_driving_distance", + api_name => "open-ils.geo.calculate_bulk_driving_distance", + signature => { + params => [ + {type => 'string', desc => 'User\'s authorization token'}, + {type => 'array', desc => 'An array containing latitude and longitude origin points as an array.'}, + {type => 'array', desc => 'An array containing latitude and longitude destination points as an array.'} + ], + return => { desc => 'Driving distance between origin points and destinations in kilometers'} + } +); + sub sort_orgs_by_distance_from_coordinate { my ($self, $conn, $pointA, $orgs) = @_; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/VicinityCalculator.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/VicinityCalculator.pm index b50f64c093..40c900e57f 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/VicinityCalculator.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/VicinityCalculator.pm @@ -7,14 +7,6 @@ use OpenILS::Utils::VicinityCalculator; use OpenSRF::Utils::SettingsClient; use OpenSRF::Utils::Logger qw(:logger); -sub get_api_key { - my $config = OpenSRF::Utils::SettingsClient->new(); - my $key = $config->config_value( - apps => 'open-ils.vicinity-calculator' => app_settings => 'key' - ); - return $key; -} - __PACKAGE__->register_method( method => 'build_distance_matrix', api_name => 'open-ils.vicinity-calculator.build-distance-matrix', @@ -25,34 +17,12 @@ __PACKAGE__->register_method( ); sub build_distance_matrix{ - my ($self) = @_; - my $key = get_api_key(); - if(!defined($key) || $key eq ''){ - $logger->error("No Maps API key has been set up in opensrf xml."); - return undef; - } - else{ - my $calculator = OpenILS::Utils::VicinityCalculator->new($key); - $calculator->calculate_distance_matrix(); + my ($self, $conn, $auth) = @_; + my $calculator = OpenILS::Utils::VicinityCalculator->new($auth); + $calculator->calculate_distance_matrix(); return 1; - } } -__PACKAGE__->register_method( - method => 'set_coords', - api_name => 'open-ils.vicinity-calculator.set-coords', - signature => { - desc => q/Calculate the latitude and longitude of an address/, - } -); - -sub set_coords{ - my ($self, $client, $addr) = @_; - my $key = get_api_key(); - my $calculator = OpenILS::Utils::VicinityCalculator->new($key); - $logger->info("calculating address coordinates"); - return $calculator->set_coord_for_addr($addr); -} __PACKAGE__->register_method( method => 'get_all_hubs', @@ -63,13 +33,8 @@ __PACKAGE__->register_method( ); sub get_all_hubs{ - my ($self) = @_; + my ($self, $conn, $auth) = @_; my $calculator = OpenILS::Utils::VicinityCalculator->new(); - my $key = get_api_key(); - if(!defined($key) || $key eq ''){ - $logger->error("No Maps API key has been set up in opensrf xml."); - return undef; - } $logger->info("retreiving org unit shipping hubs"); return $calculator->get_all_hubs(); } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Utils/VicinityCalculator.pm b/Open-ILS/src/perlmods/lib/OpenILS/Utils/VicinityCalculator.pm index 162f4c7084..d9ee04e0c5 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Utils/VicinityCalculator.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Utils/VicinityCalculator.pm @@ -17,10 +17,10 @@ our $U = "OpenILS::Application::AppUtils"; my $actor; sub new { - my ($class, $api_key) = @_; + my ($class, $auth) = @_; my $self = { - editor => new_editor(), - bing => Geo::Coder::Bing->new(key => $api_key), + editor => new_editor(authtoken => $auth), + auth => $auth, hub_cache => {}, coord_cache => {}, @@ -45,8 +45,11 @@ sub calculate_distance_matrix { my @origins = values(%hub_coord); my @destinations = values(%hub_coord); my @hub_ids = keys(%hub_coord); - # make one giant request to bing to calculate our distance matrix - my @distance_matrix = $self->vicinity_between_coords(\@origins,\@destinations); + # make one giant request to our geolocation service to calculate our distance matrix + my $geo = OpenSRF::AppSession->create('open-ils.geo'); + my $geo_request = $geo->request('open-ils.geo.calculate_bulk_driving_distance', + $self->{auth}, \@origins, \@destinations); + my @distance_matrix = @{$geo_request}; if(@distance_matrix){ $self->{editor}->xact_begin; # clear out existing matrix @@ -70,80 +73,6 @@ sub calculate_distance_matrix { } } -sub get_addr_from_ou { -my($self,@org_ids) = @_; - my @ma = $self->{editor}->json_query({ - select => { - aou => [ - { - column => 'id', - } - ], - aoa => [ - { - column => 'city', - },{ - column => 'state', - },{ - column => 'county', - },{ - column => 'street1', - },{ - column => 'street2', - },{ - column => 'post_code', - } - ] - }, - from => {aou => 'aoa'}, - where => {id=>[@org_ids]} - }); - my %addrs; - - for my $ref (@ma) { - for (@$ref){ - $addrs{$_->{id}} = $self->format_street_address($_->{street1},$_->{street2},$_->{city},$_->{county},$_->{state},$_->{post_code}); - } - } - return %addrs; -} - -sub get_coord_from_ou { -my($self,@org_ids) = @_; - my @ma = $self->{editor}->json_query({ - select => { - aoa => [ - { - column => 'org_unit', - }, - { - column => 'latitude', - },{ - column => 'longitude', - },{ - column => 'address_type', - } - ] - }, - from => 'aoa', - where => {org_unit=>[@org_ids], address_type=>['MAILING']} - }); - my %coords; - - for my $ref (@ma) { - for (@$ref){ - $coords{$_->{org_unit}} = $_->{latitude}.",".$_->{longitude}; - } - } - return %coords; -} - -# gets the address into the proper format for API -sub format_street_address{ -shift; -return join(', ',grep(defined, @_)); -} - # remove all existing distance calculations. # TODO make this all happen in one query # what could the analog to DELETE FROM TABLE be? @@ -189,162 +118,6 @@ my @sh = $self->{editor}->json_query({ return @hubs; } -sub get_coord_from_address{ - my( $self, $addr ) = @_; - my $org1geo = $self->{bing}->geocode(location => $addr); - return $org1geo->{point}{coordinates}[0].",".$org1geo->{point}{coordinates}[1]; -} - -# set the latitude and longitude for all addresses associated with an org unit -sub set_coord_for_ou{ - my $self = shift; - my $ou = int(shift); - $logger->info("using API to retrieve Long/Lat for OU $ou"); - my @ma = $self->{editor}->json_query({ - select => { - aoa => [ - { - column => 'id', - }, - { - column => 'city', - }, - { - column => 'state', - }, - { - column => 'county', - }, - { - column => 'street1', - }, - { - column => 'street2', - }, - { - column => 'post_code', - } - ] - }, - from => 'aoa', - where => {org_unit => $ou} - }); - $self->{editor}->xact_begin; - for my $ref (@ma) { - for (@$ref){ - my $addr_string = $self->format_street_address($_->{street1},$_->{street2},$_->{city},$_->{county},$_->{state},$_->{post_code}); - my $org1geo = $self->{bing}->geocode($addr_string); - my $lat = $org1geo->{point}{coordinates}[0]; - my $long = $org1geo->{point}{coordinates}[1]; - my $addr = $self->{editor}->retrieve_actor_org_address($_->{id}); - $addr->latitude($lat); - $addr->longitude($long); - $logger->info("Got $lat $long for OU $ou"); - $self->{editor}->update_actor_org_address($addr) or return $self->{editor}->die_event; - } - } - $self->{editor}->xact_commit; - return 1; -} - -sub set_coord_for_addr{ - my $self = shift; - my $addr = int(shift); - $logger->info("using API to retrieve Long/Lat for address with ID $addr"); - my @ma = $self->{editor}->json_query({ - select => { - aoa => [ - { - column => 'id', - }, - { - column => 'city', - }, - { - column => 'state', - }, - { - column => 'county', - }, - { - column => 'street1', - }, - { - column => 'street2', - }, - { - column => 'post_code', - } - ] - }, - from => 'aoa', - where => {id => $addr} - }); - - for my $ref (@ma) { - for (@$ref){ - $self->{editor}->xact_begin; - my $addr_string = $self->format_street_address($_->{street1},$_->{street2},$_->{city},$_->{county},$_->{state},$_->{post_code}); - my $org1geo = $self->{bing}->geocode($addr_string); - my $lat = $org1geo->{point}{coordinates}[0]; - my $long = $org1geo->{point}{coordinates}[1]; - my $address = $self->{editor}->retrieve_actor_org_address($addr); - $address->latitude($lat); - $address->longitude($long); - $logger->info("Got $lat $long for address $addr"); - $self->{editor}->update_actor_org_address($address) or return $self->{editor}->die_event; - $self->{editor}->xact_commit; - my @val = ($lat,$long); - return \@val; - } - } - return $self->{editor}->die_event; -} - -sub vicinity_between_coord{ -my( $self, $origin_coord, $dest_coord ) = @_; - my $b = $self->{bing}; - return _geo_request($origin_coord,$dest_coord)->[0]->{travelDistance}; -} - -sub _geo_request{ -my( $self, $origin_coord, $dest_coord ) = @_; - my $b = $self->{bing}; - unless( $b->{key} ){ - $logger->error("API key was not found"); - return undef; - } - my $uri = URI->new("https://dev.virtualearth.net/REST/v1/Routes/DistanceMatrix?origins=$origin_coord&destinations=$dest_coord&distanceUnit=mi&travelMode=driving&key=".$b->{key}); - return eval{$b->_rest_request($uri)->{results}}; -} - -sub vicinity_between_coords{ -my( $self, $origin_ref, $dest_ref ) = @_; - my @origins = @{ $origin_ref }; - my @destinations = @{ $dest_ref }; - return $self->_geo_request(join(';',@origins),join(';',@destinations)); -} - - -sub vicinity_between_ou { - my( $self, $org1, $org2 ) = @_; - my @addrs = $self->get_addr_from_ou($org1,$org2); - print("Calculating route between ".$addrs[0]." and ".$addrs[1]); - return $self->vicinity_between_coord($self->get_coord_from_address($addrs[0]),$self->get_coord_from_address($addrs[1])); -} - -sub vicinity_between_hub { - my( $self, $org1, $org2 ) = @_; - my %hubs = $self->get_hub_from_ou($org1,$org2); - if($hubs{$org1} == 0 || $hubs{$org2} == 0){ - print("Requested OU does not have a shipping hub!"); - die; - } - - return $self->vicinity_between_ou($hubs{$org1},$hubs{$org2}); -} - - package OpenILS::Utils::VicinityCalculator::Matrix; use OpenSRF::System; use OpenILS::Application::Actor; @@ -451,99 +224,4 @@ my @sh = $self->{editor}->json_query({ }); return $sh[0][0]->{'hub'}; } - - - - - -=begin work zone - -OpenSRF::System->bootstrap_client(config_file =>'/openils/conf/opensrf_core.xml'); - my $idl = OpenSRF::Utils::SettingsClient->new->config_value("IDL"); - Fieldmapper->import(IDL => $idl); -my $pc = OpenILS::Utils::VicinityCalculator->new("VbVe1thIFfqm2ghCuREV~3BmYd1kV23t34b_u1DXhQw~AvpRMvRV37o03fEBAq24KnW_R7I7M9CqwzezfKINgNG-LcwMuk7u7ihsBWZCPFE4"); -print Dumper($pc->set_coord_for_addr(2)); - -OpenSRF::System->bootstrap_client(config_file =>'/openils/conf/opensrf_core.xml'); - my $idl = OpenSRF::Utils::SettingsClient->new->config_value("IDL"); - Fieldmapper->import(IDL => $idl); -my $pc = OpenILS::Utils::VicinityCalculator::Matrix->new(); -my @hubs = (7,11,4); -print Dumper($pc->hub_matrix(13,\@hubs)); - - - -my @copy_id = (4007,3507,3807,3307,3707,3207,3607,3107, 4819); - -OpenSRF::System->bootstrap_client(config_file =>'/openils/conf/opensrf_core.xml'); - my $idl = OpenSRF::Utils::SettingsClient->new->config_value("IDL"); - Fieldmapper->import(IDL => $idl); -my $pc = OpenILS::Utils::VicinityCalculator->new("AosM-K7Hdbk-OMZ1jcJC1boNDGRpoYRL_bzgK6pqKNNVAc2-z0qbOVtc3itjfWj5"); -#my $pc = OpenILS::Utils::VicinityCalculator->new(); -#$pc->calculate_distance_matrix(); -#my @dest_hubs = (226,182,393,4,208); -#my @matrix = $pc->hub_matrix(180,\@dest_hubs); -#my @targets = (17024825,14189348,5952821,17056866,15214541,14074994 ); -#my %hubs = $pc->get_target_hubs(\@targets); -#print(Dumper(\%hubs)); -print($pc->get_hub_from_ou(2)); - - -OpenSRF::System->bootstrap_client(config_file =>'/openils/conf/opensrf_core.xml'); - my $idl = OpenSRF::Utils::SettingsClient->new->config_value("IDL"); - Fieldmapper->import(IDL => $idl); -my $pc = OpenILS::Utils::VicinityCalculator->new("AosM-K7Hdbk-OMZ1jcJC1boNDGRpoYRL_bzgK6pqKNNVAc2-z0qbOVtc3itjfWj5"); -#my $prox = $pc->vicinity_between_hub(102,109); -#print "\n\nDistance is $prox miles\n"; -#my @origins = ("35.778774,-78.685422", "36.280466,-76.214402"); -#my @destinations = ("34.694165,-76.551269", "35.595012,-82.551707"); -#print Dumper($pc->vicinity_between_coords(\@origins,\@destinations)); -# all this stuff below is gonna be a function that dumps the distance matrix into the database, it'll be run for every hub and we'll probably only need to run it once a year. Each iteration of the function will produce less data since the x runs before it would have calculated data we can use again. -my $hold_id = 6832841; -my $request_ou = $pc->ou_from_hold($hold_id); -my %proxmap = $pc->compile_weighted_vicinity_map($hold_id); -print("\n$request_ou\n"); -my @OU; - -push @OU, $request_ou; -while( my($k,$v) = each %proxmap){ - push @OU, $v->{ou}; -} -my %hubs = $pc->get_hub_from_ou(@OU); - -#my %hub_addr = $pc->get_addr_from_ou(uniq(values(%hubs))); - -# save coords so I don't blow through my queries on bing -my %hub_coord = ('260' => '36.4941,-79.73601','102' => '35.240596,-81.342891','314' => '35.511453,-78.3456','182' => '36.404213,-79.333114','325' => '34.775483,-79.465872','310' => '35.9207,-81.17589','4' => '35.293008,-81.555723','161' => '35.426298,-83.444665','189' => '35.92217133,-81.523353','142' => '36.109843,-78.296138','208' => '35.055522,-78.881343','237' => '35.304749,-76.789123','112' => '35.596714,-82.554788','370' => '36.32762667,-78.40572167','180' => '35.59727,-77.58532333','343' => '36.309471,-78.587604','269' => '35.40015517,-78.814922','177' => '35.487138,-82.9919','277' => '36.244018,-80.854557','291' => '35.266071,-77.581526','298' => '35.315485,-82.462982','196' => '35.68377417,-82.0106725','107' => '35.81924333,-80.25970833','367' => '35.240179,-82.216521','187' => '35.195678,-78.068236','166' => '35.897794,-80.559671','306' => '35.787559,-80.887852','226' => '36.098649,-80.252405','137' => '36.15954,-81.14848','238' => '35.543145,-77.05459333'); -# get coords for each hub -#while( my($k,$v) = each %hub_addr){ -# $hub_coord{$k} = $pc->get_coord_from_address($v); -#} - - - -# get distance matrix between my hub and every other hub -my $origin_hub = $hubs{$request_ou}; -my @origins = ($hub_coord{$origin_hub}); -my @destinations = values(%hub_coord); -my @hub_ids = keys(%hub_coord); -my @distance_matrix = $pc->vicinity_between_coords(\@origins,\@destinations); -my %hub_distance_matrix; - -# break down distance matrix into hash - $pc->{editor}->xact_begin; - for my $ref (@distance_matrix) { - for (@$ref){ - $hub_distance_matrix{$hub_ids[$_->{destinationIndex}]} = $_->{travelDistance}; - # put them in the database from here - my $dist = Fieldmapper::actor::org_unit_shipping_hub_distance->new; - $dist->orig_hub($origin_hub); - $dist->dest_hub($hub_ids[$_->{destinationIndex}]); - $dist->distance($_->{travelDistance}); - $pc->{editor}->runmethod('create', 'actor.org_unit_shipping_hub_distance', 'aoushd', $dist); - } - } - $pc->{editor}->xact_commit; -print Dumper(\%hub_distance_matrix); -=cut 1; \ No newline at end of file