From 7fe0ddb835633bd630a0aab32c9bc5a0c11f0b77 Mon Sep 17 00:00:00 2001 From: Lebbeous Fogle-Weekley Date: Tue, 20 Mar 2012 18:44:42 -0400 Subject: [PATCH] a work in progress. process_map() works really nicely to fill out the fleshing and the joins, and I think overall that's really the hard part. we'll see. Signed-off-by: Lebbeous Fogle-Weekley --- Open-ILS/scratch-recur.pl | 63 +++++ .../perlmods/lib/OpenILS/Application/Fielder.pm | 45 ++++ .../perlmods/lib/OpenILS/Application/Flattener.pm | 257 +++++++++++++++++++++ 3 files changed, 365 insertions(+) create mode 100644 Open-ILS/scratch-recur.pl create mode 100644 Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm diff --git a/Open-ILS/scratch-recur.pl b/Open-ILS/scratch-recur.pl new file mode 100644 index 0000000000..fd37de7160 --- /dev/null +++ b/Open-ILS/scratch-recur.pl @@ -0,0 +1,63 @@ +#!/usr/bin/perl -w + +use strict; + +use Data::Dumper; + +my $one = { + -and => [ + {foo => "bar"}, + {baz => {">=" => 3}} + ] +}; + +my $two = { + "xact_start" => {">" => "today"}, + "circ_lib" => "BR1" +}; + +my $three = { + -not => [ + {abc => {between => [1,32]}}, + {def => {">=" => 1}}, + {"ghi" => [1,2,3,4], "jkl" => 5} + ] +}; + +sub newkey { + return "XXX" . shift; +} + +sub recursively_mark_column_names { + my ($o, $state) = @_; + + $state ||= {}; + + if (ref $o eq 'HASH') { + foreach my $key (keys %$o) { + if (not $state->{in_expr} and $key =~ /^[a-z]/) { + $state->{in_expr} = 1; + + my $newkey = newkey($key); + print "attempting replacement of $key with $newkey\n"; + + $o->{$newkey} = $o->{$key}; + delete $o->{$key}; + + recursively_mark_column_names($o->{$newkey}, $state); + + $state->{in_expr} = 0; + } else { + recursively_mark_column_names($o->{$key}, $state); + } + } + } elsif (ref $o eq 'ARRAY') { + recursively_mark_column_names($_, $state) foreach @$o; + } # else scalar, nothing to do? +} + +recursively_mark_column_names($one); +recursively_mark_column_names($two); +recursively_mark_column_names($three); + +print Dumper($one, $two, $three), "\n"; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Fielder.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Fielder.pm index 341b569ed3..2cfc979462 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Fielder.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Fielder.pm @@ -23,6 +23,8 @@ use XML::LibXML; use XML::LibXML::XPathContext; use XML::LibXSLT; +use OpenILS::Application::Flattener; + our %namespace_map = ( oils_persist=> {ns => 'http://open-ils.org/spec/opensrf/IDL/persistence/v1'}, oils_obj => {ns => 'http://open-ils.org/spec/opensrf/IDL/objects/v1'}, @@ -153,6 +155,49 @@ sub generate_methods { }; } +sub flattened_search { + my ($self, $conn, $auth, $hint, $map, $where, $slo) = @_; + + # Process the map to normalize it, and to get all our joins and fleshing + # structure into the jffolo. + my ($map, $jffolo) = + OpenILS::Application::Flattener::process_map($hint, $map); + + # Process the suppied where clause, using our map, to make the + # filter. + my $filter = OpenILS::Application::Flattener::prepare_filter($map, $where); + + # Process the supplied sort/limit/offset clause and use it to finish the + # jffolo. + $jffolo = OpenILS::Application::Flattener::finish_jffolo( + $map, $jffolo, $slo + ); + + # Reach out and touch some service. + my $pcrud = create OpenSRF::AppSession("open-ils.pcrud"); + my $req = $pcrud->request( + "open-ils.pcrud.search.$hint", $auth, $filter, $jffolo + ); + + # Stream back flattened results. + while (my $resp = $req->recv(timeout => 60)) { + $conn->respond( + OpenILS::Application::Flattener::process_result($map, $resp) + ); + } + + # Clean up. + $pcrud->kill_me; + + return; +} + +__PACKAGE__->register_method( + method => 'flattened_search', + api_name => 'open-ils.fielder.flattened_search', + stream => 1, + argc => 5 +); 1; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm new file mode 100644 index 0000000000..c984ab4abf --- /dev/null +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Flattener.pm @@ -0,0 +1,257 @@ +package OpenILS::Application::Flattener; + +# This package is not meant to be registered as a stand-alone OpenSRF +# application, but to be mixed into other services + +use base qw/OpenILS::Application/; + +use strict; +use warnings; + +use OpenSRF::EX qw/:try/; +use OpenSRF::Utils::Logger qw/:logger/; +use OpenILS::Utils::CStoreEditor q/:funcs/; + + +sub _fm_class_by_hint { + my ($hint) = @_; + + my ($class) = grep { + Fieldmapper->publish_fieldmapper->{$_}{hint} eq $hint + } keys %{ Fieldmapper->publish_fieldmapper }; + + return $class; + +} + +sub _fm_identity_from_class { + my ($class) = @_; + + return Fieldmapper->publish_fieldmapper->{$class}{identity}; +} + +sub _fm_link_from_class { + my ($class, $piece) = @_; + + return Fieldmapper->publish_fieldmapper->{$class}{links}{$piece}; +} + +sub _flattened_search_single_flesh_wad { + my ($hint, $path) = @_; + + $path = [ @$path ]; # clone for processing here + my $class = _fm_class_by_hint($hint); + + my $flesh_depth = 0; + my $flesh_fields = {}; + + pop @$path; # last part is just field + + my $piece; + + while ($piece = shift @$path) { + $logger->debug("_flattened_search_single_flesh_wad() dealing with $piece"); + my $link = _fm_link_from_class($class, $piece); + if ($link) { + $flesh_fields->{$hint} ||= []; + push @{ $flesh_fields->{$hint} }, $piece; + $hint = $link->{class}; + $class = _fm_class_by_hint($hint); + $flesh_depth++; + } else { + throw OpenSRF::EX::ERROR("no link $piece on $class"); + } + } + + return { + flesh => $flesh_depth, + flesh_fields => $flesh_fields + }; +} + +# returns a join clause AND a string representing the deepest join alias +# generated. +sub _flattened_search_single_join_clause { + my ($column_name, $hint, $path) = @_; + + my $class = _fm_class_by_hint($hint); + my $last_ident = _fm_identity_from_class($class); + + $path = [ @$path ]; # clone for processing here + + pop @$path; # last part is just field + + my $core_join = {}; + my $last_join; + my $piece; + my $alias; # yes, we need it out at this scope. + + while ($piece = shift @$path) { + $logger->debug("_flattened_search_single_join_clause() dealing with $piece"); + + my $link = _fm_link_from_class($class, $piece); + if ($link) { + $hint = $link->{class}; + $class = _fm_class_by_hint($hint); + + my $reltype = $link->{reltype}; + my $field = $link->{key}; + if ($link->{map}) { + # XXX having a non-blank value for map means we'll need + # an additional level of join. TODO. + throw OpenSRF::EX::ERROR( + "support not yet implemented for links like '$piece' with" . + " non-blank 'map' IDL attribute" + ); + } + + $alias = "__${column_name}_${hint}"; + my $new_join; + if ($reltype eq 'has_a') { + $new_join = { + class => $hint, + fkey => $piece, + field => $field + }; + } elsif ($reltype eq 'has_many' or $reltype eq 'might_have') { + $new_join = { + class => $hint, + fkey => $last_ident, + field => $field + }; + } else { + throw OpenSRF::EX::ERROR("unexpected reltype for link $piece"); + } + + if ($last_join) { + $last_join->{join}{$alias} = $new_join; + } else { + $core_join->{$alias} = $new_join; + } + + $last_ident = _fm_identity_from_class($class); + $last_join = $new_join; + } else { + throw new OpenSRF::EX::ERROR("no link '$piece' on $class"); + } + } + + return ($core_join, $alias); +} + +# When $value is a string (short form of a column definition), it is assumed to +# be a dot-delimited path. This will be normalized into a hash (long form) +# containing and path key, whose value will be made into an array, and true +# values for sort/filter/display. +# +# When $value is already a hash (long form), just make an array of the path key +# and explicity set any sort/filter/display values not present to 0. +# +sub _flattened_search_normalize_map_column { + my ($value) = @_; + + if (ref $value eq "HASH") { + foreach (qw/sort filter display/) { + $value->{$_} = 0 unless exists $value->{$_}; + } + $value->{path} = [split /\./, $value->{path}]; + } else { + $value = { + path => [split /\./, $value], + sort => 1, + filter => 1, + display => 1 + }; + } + + return $value; +} + +sub _flattened_search_merge_flesh_wad { + my ($old, $new) = @_; + + $old->{flesh} ||= 0; + $old->{flesh} = $old->{flesh} > $new->{flesh} ? $old->{flesh} : $new->{flesh}; + + $old->{flesh_fields} ||= {}; + foreach my $key (keys %{$new->{flesh_fields}}) { + if ($old->{flesh_fields}{$key}) { + # For easy bonus points, somebody could take the following block + # and make it use Set::Scalar so it's more semantic, which would + # mean a new Evergreen dependency. + # + # The nonobvious point of the block is to merge the arrays at + # $old->{flesh_fields}{$key} and $new->{flesh_fields}{$key}, + # treating the arrays as sets. + + my %hash = map { $_ => 1 } ( + @{ $old->{flesh_fields}{$key} }, + @{ $new->{flesh_fields}{$key} } + ); + $old->{flesh_fields}{$key} = [ keys(%hash) ]; + } else { + $old->{flesh_fields}{$key} = $new->{flesh_fields}{$key}; + } + } +} + +sub _flattened_search_merge_join_clause { + my ($old, $new) = @_; + + %$old = ( %$old, %$new ); +} + +# returns a normalized version of the map, and the jffolo (see below) +sub process_map { + my ($hint, $map) = @_; + + $map = { %$map }; # clone map, to work on new copy + + my $jffolo = { # jffolo: join/flesh/flesh_fields/order_by/limit/offset + join => {} + }; + + foreach my $k (keys %$map) { + my $column = $map->{$k} = + _flattened_search_normalize_map_column($map->{$k}); + + # For display columns, we'll need fleshing. + if ($column->{display}) { + _flattened_search_merge_flesh_wad( + $jffolo, + _flattened_search_single_flesh_wad($hint, $column->{path}) + ); + } + + # For filter or sort columns, we'll need joining. + if ($column->{filter} or $column->{sort}) { + my ($clause, $last_join_alias) = + _flattened_search_single_join_clause($k,$hint,$column->{path}); + + $map->{$k}{last_join_alias} = $last_join_alias; + _flattened_search_merge_join_clause($jffolo->{join}, $clause); + } + } + + return ($map, $jffolo); +} + +# return a filter clause for PCRUD or cstore, by processing the supplied +# simplifed $where clause using $map. +sub prepare_filter { + my ($map, $where) = @_; + + my $filter = {}; + + # XXX find scratch-recur.pl and do something like that. +# foreach my $key (keys(%$where)) { +# next unless exists $map->{$key}; +# +# my $field = $map->{$key}{path}[-1]; +# my $last_join_alias = $map->{$key}{last_join_alias}; +# +# +# } +} + +1; -- 2.11.0