use XML::LibXML::XPathContext;
use XML::LibXSLT;
+use OpenILS::Application::Flattener;
+
our %namespace_map = (
oils_persist=> {ns => 'http://open-ils.org/spec/opensrf/IDL/persistence/v1'},
oils_obj => {ns => 'http://open-ils.org/spec/opensrf/IDL/objects/v1'},
};
}
+sub register_map {
+ my ($self, $conn, $auth, $hint, $map) = @_;
+
+ $key = 'flat_search_' . md5_hex(
+ $hint .
+ OpenSRF::Utils::JSON->perl2JSON( $map )
+ );
+
+ $cache->put_cache( $key => { hint => $hint, map => $map } => $cache_timeout );
+}
+
+__PACKAGE__->register_method(
+ method => 'register_map',
+ api_name => 'open-ils.fielder.flattened_search.prepare',
+ argc => 2,
+ signature => {
+ params => [
+ {name => "auth", type => "string", desc => "auth token"},
+ {name => "hint", type => "string",
+ desc => "fieldmapper class hint of core object"},
+ {name => "map", type => "object", desc => q{
+ path-field mapping structure. See documentation under
+ docs/TechRef/Flattener in the Evergreen source tree.} }
+ ],
+ return => {
+ desc => q{
+ A key used to reference a prepared flattened search on subsequent
+ calls to open-ils.fielder.flattened_search.execute},
+ type => "string"
+ }
+ }
+);
+
+sub execute_registered_flattened_search {
+ my $self = shift;
+ my $conn = shift;
+ my $auth = shift;
+ my $key = shift;
+
+ my $blob = $cache->get_cache( $key );
+
+ flattened_search( $self, $conn, $auth, $blob->{hint}, $blob->{map}, @_ )
+ if (ref($blob) and $blob->{hint} and $blob->{map});
+}
+
+__PACKAGE__->register_method(
+ method => 'execute_registered_flattened_search',
+ api_name => 'open-ils.fielder.flattened_search.execute',
+ stream => 1,
+ argc => 5,
+ signature => {
+ params => [
+ {name => "auth", type => "string", desc => "auth token"},
+ {name => "key", type => "string",
+ desc => "Key for a registered map provided by open-ils.fielder.flattened_search.prepare"},
+ {name => "where", type => "object", desc => q{
+ simplified query clause (like the 'where' clause of a
+ json_query, but different). See documentation under
+ docs/TechRef/Flattener in the Evergreen source tree.} },
+ {name => "slo", type => "object", desc => q{
+ simplified sort/limit/offset object. See documentation under
+ docs/TechRef/Flattener in the Evergreen source tree.} }
+ ],
+ return => {
+ desc => q{
+ A stream of objects flattened to your specifications. See
+ documentation under docs/TechRef/Flattener in the Evergreen
+ source tree.},
+ type => "object"
+ }
+ }
+);
+
+sub flattened_search {
+ my ($self, $conn, $auth, $hint, $map, $where, $slo) = @_;
+
+ # All but the last argument really are necessary.
+ $slo ||= {};
+
+ # Process the map to normalize it, and to get all our joins and fleshing
+ # structure into the jffolo.
+ my $jffolo;
+ ($map, $jffolo) =
+ OpenILS::Application::Flattener::process_map($hint, $map);
+
+ # Process the suppied where clause, using our map, to make the
+ # filter.
+ my $filter = OpenILS::Application::Flattener::prepare_filter($map, $where);
+
+ # Process the supplied sort/limit/offset clause and use it to finish the
+ # jffolo.
+ $jffolo = OpenILS::Application::Flattener::finish_jffolo(
+ $hint, $map, $jffolo, $slo
+ );
+
+ # Reach out and touch pcrud (could be cstore, if we wanted to offer
+ # this as a private service).
+ my $pcrud = create OpenSRF::AppSession("open-ils.pcrud");
+ my $req = $pcrud->request(
+ "open-ils.pcrud.search.$hint", $auth, $filter, $jffolo
+ );
+
+ # Stream back flattened results.
+ while (my $resp = $req->recv(timeout => 60)) {
+ $conn->respond(
+ OpenILS::Application::Flattener::process_result(
+ $map, $resp->content
+ )
+ );
+ }
+
+ # Clean up.
+ $pcrud->kill_me;
+
+ return;
+}
+
+__PACKAGE__->register_method(
+ method => 'flattened_search',
+ api_name => 'open-ils.fielder.flattened_search',
+ stream => 1,
+ argc => 5,
+ signature => {
+ params => [
+ {name => "auth", type => "string", desc => "auth token"},
+ {name => "hint", type => "string",
+ desc => "fieldmapper class hint of core object"},
+ {name => "map", type => "object", desc => q{
+ path-field mapping structure. See documentation under
+ docs/TechRef/Flattener in the Evergreen source tree.} },
+ {name => "where", type => "object", desc => q{
+ simplified query clause (like the 'where' clause of a
+ json_query, but different). See documentation under
+ docs/TechRef/Flattener in the Evergreen source tree.} },
+ {name => "slo", type => "object", desc => q{
+ simplified sort/limit/offset object. See documentation under
+ docs/TechRef/Flattener in the Evergreen source tree.} }
+ ],
+ return => {
+ desc => q{
+ A stream of objects flattened to your specifications. See
+ documentation under docs/TechRef/Flattener in the Evergreen
+ source tree.},
+ type => "object"
+ }
+ }
+);
1;
--- /dev/null
+package OpenILS::Application::Flattener;
+
+# This package is not meant to be registered as a stand-alone OpenSRF
+# application, but to be used by high level methods in other services.
+
+use base qw/OpenILS::Application/;
+
+use strict;
+use warnings;
+
+use OpenSRF::EX qw/:try/;
+use OpenSRF::Utils::Logger qw/:logger/;
+use OpenILS::Utils::CStoreEditor q/:funcs/;
+use OpenSRF::Utils::JSON;
+
+sub _fm_link_from_class {
+ my ($class, $field) = @_;
+
+ return Fieldmapper->publish_fieldmapper->{$class}{links}{$field};
+}
+
+sub _flattened_search_single_flesh_wad {
+ my ($hint, $path) = @_;
+
+ $path = [ @$path ]; # clone for processing here
+ my $class = OpenSRF::Utils::JSON->lookup_class($hint);
+
+ my $flesh_depth = 0;
+ my $flesh_fields = {};
+
+ pop @$path; # last part is just field
+
+ my $piece;
+
+ while ($piece = shift @$path) {
+ my $link = _fm_link_from_class($class, $piece);
+ if ($link) {
+ $flesh_fields->{$hint} ||= [];
+ push @{ $flesh_fields->{$hint} }, $piece;
+ $hint = $link->{class};
+ $class = OpenSRF::Utils::JSON->lookup_class($hint);
+ $flesh_depth++;
+ } else {
+ throw OpenSRF::EX::ERROR("no link $piece on $class");
+ }
+ }
+
+ return {
+ flesh => $flesh_depth,
+ flesh_fields => $flesh_fields
+ };
+}
+
+# returns a join clause AND a string representing the deepest join alias
+# generated.
+sub _flattened_search_single_join_clause {
+ my ($column_name, $hint, $path) = @_;
+
+ my $class = OpenSRF::Utils::JSON->lookup_class($hint);
+ my $last_ident = $class->Identity;
+
+ $path = [ @$path ]; # clone for processing here
+
+ pop @$path; # last part is just field
+
+ my $core_join = {};
+ my $last_join;
+ my $piece;
+ my $alias; # yes, we need it out at this scope.
+
+ while ($piece = shift @$path) {
+ my $link = _fm_link_from_class($class, $piece);
+ if ($link) {
+ $hint = $link->{class};
+ $class = OpenSRF::Utils::JSON->lookup_class($hint);
+
+ my $reltype = $link->{reltype};
+ my $field = $link->{key};
+ if ($link->{map}) {
+ # XXX having a non-blank value for map means we'll need
+ # an additional level of join. TODO.
+ throw OpenSRF::EX::ERROR(
+ "support not yet implemented for links like '$piece' with" .
+ " non-blank 'map' IDL attribute"
+ );
+ }
+
+ $alias = "__${column_name}_${hint}";
+ my $new_join;
+ if ($reltype eq "has_a") {
+ $new_join = {
+ class => $hint,
+ fkey => $piece,
+ field => $field
+ };
+ } elsif ($reltype eq "has_many" or $reltype eq "might_have") {
+ $new_join = {
+ class => $hint,
+ fkey => $last_ident,
+ field => $field
+ };
+ } else {
+ throw OpenSRF::EX::ERROR("unexpected reltype for link $piece");
+ }
+
+ if ($last_join) {
+ $last_join->{join}{$alias} = $new_join;
+ } else {
+ $core_join->{$alias} = $new_join;
+ }
+
+ $last_ident = $class->Identity;
+ $last_join = $new_join;
+ } else {
+ throw new OpenSRF::EX::ERROR("no link '$piece' on $class");
+ }
+ }
+
+ return ($core_join, $alias);
+}
+
+# When $value is a string (short form of a column definition), it is assumed to
+# be a dot-delimited path. This will be normalized into a hash (long form)
+# containing and path key, whose value will be made into an array, and true
+# values for sort/filter/display.
+#
+# When $value is already a hash (long form), just make an array of the path key
+# and explicity set any sort/filter/display values not present to 0.
+#
+sub _flattened_search_normalize_map_column {
+ my ($value) = @_;
+
+ if (ref $value eq "HASH") {
+ foreach (qw/sort filter display/) {
+ $value->{$_} = 0 unless exists $value->{$_};
+ }
+ $value->{path} = [split /\./, $value->{path}];
+ } else {
+ $value = {
+ path => [split /\./, $value],
+ sort => 1,
+ filter => 1,
+ display => 1
+ };
+ }
+
+ return $value;
+}
+
+sub _flattened_search_merge_flesh_wad {
+ my ($old, $new) = @_;
+
+ $old->{flesh} ||= 0;
+ $old->{flesh} = $old->{flesh} > $new->{flesh} ? $old->{flesh} : $new->{flesh};
+
+ $old->{flesh_fields} ||= {};
+ foreach my $key (keys %{$new->{flesh_fields}}) {
+ if ($old->{flesh_fields}{$key}) {
+ # For easy bonus points, somebody could take the following block
+ # and make it use Set::Scalar so it's more semantic, which would
+ # mean a new Evergreen dependency.
+ #
+ # The nonobvious point of the following code is to merge the
+ # arrays at $old->{flesh_fields}{$key} and
+ # $new->{flesh_fields}{$key}, treating the arrays as sets.
+
+ my %hash = map { $_ => 1 } (
+ @{ $old->{flesh_fields}{$key} },
+ @{ $new->{flesh_fields}{$key} }
+ );
+ $old->{flesh_fields}{$key} = [ keys(%hash) ];
+ } else {
+ $old->{flesh_fields}{$key} = $new->{flesh_fields}{$key};
+ }
+ }
+}
+
+sub _flattened_search_merge_join_clause {
+ my ($old, $new) = @_;
+
+ %$old = ( %$old, %$new );
+}
+
+sub _flattened_search_expand_filter_column {
+ my ($o, $key, $map) = @_;
+
+ if ($map->{$key}) {
+ my $table = $map->{$key}{last_join_alias};
+ my $column = $map->{$key}{path}[-1];
+
+ if ($table) {
+ $table = "+" . $table;
+ $o->{$table} ||= {};
+
+ $o->{$table}{$column} = $o->{$key};
+ delete $o->{$key};
+
+ return $o->{$table}{$column};
+ } else { # field must be on core class
+ if ($column ne $key) {
+ $o->{$column} = $o->{$key};
+ delete $o->{$key};
+ }
+ return $o->{$column};
+ }
+ } else {
+ return $o->{$key};
+ }
+}
+
+sub _flattened_search_recursively_apply_map_to_filter {
+ my ($o, $map, $state) = @_;
+
+ $state ||= {};
+
+ if (ref $o eq "HASH") {
+ foreach my $key (keys %$o) {
+ # XXX this business about "in_expr" may prove inadequate, but it's
+ # intended to avoid trying to map things like "between" in
+ # constructs like:
+ # {"somecolumn": {"between": [1,10]}}
+ # and to that extent, it works.
+
+ if (not $state->{in_expr} and $key =~ /^[a-z]/) {
+ $state->{in_expr} = 1;
+
+ _flattened_search_recursively_apply_map_to_filter(
+ _flattened_search_expand_filter_column($o, $key, $map),
+ $map, $state
+ );
+
+ $state->{in_expr} = 0;
+ } else {
+ _flattened_search_recursively_apply_map_to_filter(
+ $o->{$key}, $map, $state
+ );
+ }
+ }
+ } elsif (ref $o eq "ARRAY") {
+ _flattened_search_recursively_apply_map_to_filter(
+ $_, $map, $state
+ ) foreach @$o;
+ } # else scalar, nothing to do?
+}
+
+# returns a normalized version of the map, and the jffolo (see below)
+sub process_map {
+ my ($hint, $map) = @_;
+
+ $map = { %$map }; # clone map, to work on new copy
+
+ my $jffolo = { # jffolo: join/flesh/flesh_fields/order_by/limit/offset
+ join => {}
+ };
+
+ foreach my $k (keys %$map) {
+ my $column = $map->{$k} =
+ _flattened_search_normalize_map_column($map->{$k});
+
+ # For display columns, we'll need fleshing.
+ if ($column->{display}) {
+ _flattened_search_merge_flesh_wad(
+ $jffolo,
+ _flattened_search_single_flesh_wad($hint, $column->{path})
+ );
+ }
+
+ # For filter or sort columns, we'll need joining.
+ if ($column->{filter} or $column->{sort}) {
+ my ($clause, $last_join_alias) =
+ _flattened_search_single_join_clause($k,$hint,$column->{path});
+
+ $map->{$k}{last_join_alias} = $last_join_alias;
+ _flattened_search_merge_join_clause($jffolo->{join}, $clause);
+ }
+ }
+
+ return ($map, $jffolo);
+}
+
+# return a filter clause for PCRUD or cstore, by processing the supplied
+# simplifed $where clause using $map.
+sub prepare_filter {
+ my ($map, $where) = @_;
+
+ my $filter = {%$where};
+
+ _flattened_search_recursively_apply_map_to_filter($filter, $map);
+
+ return $filter;
+}
+
+# Return a jffolo with sort/limit/offset from the simplified sort hash (slo)
+# mixed in. limit and offset are copied as-is. sort is translated into
+# an order_by that calls simplified column named by their real names by checking
+# the map.
+sub finish_jffolo {
+ my ($core_hint, $map, $jffolo, $slo) = @_;
+
+ $jffolo = { %$jffolo }; # clone
+ $slo = { %$slo }; # clone
+
+ $jffolo->{limit} = $slo->{limit} if exists $slo->{limit};
+ $jffolo->{offset} = $slo->{offset} if exists $slo->{offset};
+
+ return $jffolo unless $slo->{sort};
+
+ # The slo has a special format for 'sort' that gives callers what they
+ # need, but isn't as flexible as json_query's 'order_by'.
+ #
+ # "sort": [{"column1": "asc"}, {"column2": "desc"}]
+ # or
+ # "sort": ["column1", {"column2": "desc"}]
+ # or
+ # "sort": {"onlycolumn": "asc"}
+ # or
+ # "sort": "onlycolumn"
+
+ $jffolo->{order_by} = [];
+
+ # coerce from optional simpler format (see comment blob above)
+ $slo->{sort} = [ $slo->{sort} ] unless ref $slo->{sort} eq "ARRAY";
+
+ foreach my $exp (@{ $slo->{sort} }) {
+ $exp = { $exp => "asc" } unless ref $exp;
+
+ # XXX By assuming that each sort expression is (at most) a single
+ # key/value pair, we preclude the ability to use transforms and the
+ # like for now.
+
+ my ($key) = keys(%$exp);
+
+ if ($map->{$key}) {
+ my $class = $map->{$key}{last_join_alias} || $core_hint;
+ push @{ $jffolo->{order_by} }, {
+ class => $class,
+ field => $map->{$key}{path}[-1],
+ direction => $exp->{$key}
+ };
+ }
+
+ # If the key wasn't defined in the map, we'll leave it out of our
+ # order_by clause.
+ }
+
+ return $jffolo;
+}
+
+# Given a map and a fieldmapper object, return a flat representation as
+# specified by the map's display fields
+sub process_result {
+ my ($map, $fmobj) = @_;
+
+ if (not ref $fmobj) {
+ throw OpenSRF::EX::ERROR(
+ "process_result() was passed an inappropriate second argument"
+ );
+ }
+
+ my $flatrow = {};
+
+ while (my ($key, $mapping) = each %$map) {
+ next unless $mapping->{display};
+
+ my @path = @{ $mapping->{path} };
+ my $field = pop @path;
+
+ my $obj = $fmobj;
+ while (my $step = shift @path) {
+ $obj = $obj->$step;
+ last unless ref $obj;
+ }
+
+ $flatrow->{$key} = ref $obj ? $obj->$field : undef;
+ }
+
+ return $flatrow;
+}
+
+1;
--- /dev/null
+Deep-data Flattening Service
+============================
+Mike Rylander
+with Lebbeous Fogle-Weekley
+
+[abstract]
+Purpose
+-------
+Evergreen supplies a broad API for accessing, processing and interacting with library data. Because of the relatively complex nature of the underlying database schema, and the flexibility required by clients when, in the simplest case, performing CRUD operations, focus has been given to providing a nearly direct view of various data source. When, however, the verbosity or complexity of full object access gets in the way of performant or straight-forward UIs, there has been a tendency to create one-off data simplification calls targetting specific use cases.
+
+A generalized API which accepts a simplified query structure and field mapping, calculates the required backend query, and flattens the hierarchical data returned for each top level row into a would facilitate the simplification of existing UIs and provide for new UIs based on simple, reusable components.
+
+Overview
+--------
+The existing, public open-ils.fielder server will be extended with two new OpenSRF methods, contained in a separate package so that they will be reusable in a private service which does not require authentication.
+
+These methods will be supported by code which takes simplifed cstore/pcrud search and order-by hashes and computes the full data structure required for the query. The simplification will leverage the IDL to discover links between classes.
+
+Underlying the simplified search grammar will be a path-field mapping structure. This will describe the layout of fields, how they are to collapse from fleshed objects, and how the shape of both the query and result data structures should be interpreted by and presented to the caller.
+
+Mapping Structure
+-----------------
+Implemented as a JSON object, each property name will represent a data element that can be displayed, filtered or sorted, and each property value will represent either a simple path (in which case it is usable for display, filtering or sorting), or an object providing the path and available uses.
+
+Example Map
+~~~~~~~~~~~
+Assuming a core class of circ:
+
+--------------------------------------------------------------------------------
+{
+ "patron_barcode": "usr.card.barcode",
+ "circ_lib_name": "circ_lib.name",
+ "circ_lib": "circ_lib.shortname",
+ "xact_start": { "path": "xact_start", "sort": true, "display": true },
+ "id": "id",
+ "checkin_time": { "path": "checkin_time", "filter": true, "sort": true }
+}
+--------------------------------------------------------------------------------
+
+Based on this mapping structure simplified queries can be constructed by the caller, which will then be expanded in the flattening service to produce join and where clauses that can be used by open-ils.pcrud.
+
+Example Query
+~~~~~~~~~~~~~
+Assuming the above example map:
+
+-------------------------------------
+{ "xact_start": { ">": "today" },
+ "circ_lib": "BR1"
+}
+-------------------------------------
+
+This example would expand to a PCrud query based on the map provided above, containing not only the complex where clause, but a join tree and the necessary fleshing structure.
+
+
+Expanded PCrud Query
+~~~~~~~~~~~~~~~~~~~~
+
+---------------------------------------
+{ "xact_start": {">": "today"},
+ "+circ_lib_aou": {"name": "BR1"}
+}, {
+ "join": {
+ "circ_lib_aou": {
+ "class": "aou",
+ "fkey": "circ_lib"
+ },
+ "usr_au": {
+ "class": "au",
+ "fkey": "usr",
+ "join": { "usr_card_ac": {
+ "class": "ac",
+ "fkey": "card"
+ }
+ }
+ }
+ },
+ "flesh_fields": {
+ "circ" : ["circ_lib","usr"],
+ "aou" : ["card"]
+ },
+ "flesh": 2
+}
+---------------------------------------
+
+
+API
+---
+
+OpenSRF Method name: open-ils.fielder.flattened_search
+
+Parameters:
+
+- Authentication token (as for pcrud)
+- IDL class
+- "circ"
+- Path map hash
+ * e.g. {"patron_barcode":"usr.card.barcode","circ_lib_name":"circ_lib.name", "circ_lib":"circ_lib.shortname","xact_start":{ "path": "xact_start", "sort": true, "display": true},"id":"id","checkin_time": { "path": "checkin_time", "filter": true, "sort": true }}
+- Simplified query hash
+ * e.g. {"xact_start": { ">": "today" }, "circ_lib": "BR1"}
+- Simplified sort/limit/offset hash
+ * e.g. { "sort":[{"checkin_time":"desc"},{"circ_lib":"asc"}],"limit":10}
+ * or {"sort":{"checkin_time":"desc"}}
+ * or {"sort": "circ_lib"}
+ * or {"sort": ["circ_lib", {"checkin_time": "desc"}]}
+
+Returns:
+
+- stream (or array, for .atomic) of hashes having the shape described in the path map
+ * e.g. {"patron_barcode":"3123456789012","circ_lib_name":"Branch 1", "circ_lib":"BR1","xact_start":"2011-10-17T12:00:00-05:00,"id":"123"}
projects / working / Evergreen.git / commitdiff