From 3ef95f51163cec16e0f05d3ca9853e5b8e2716c5 Mon Sep 17 00:00:00 2001 From: thread Date: Sun, 26 Feb 2023 12:49:10 -0500 Subject: [PATCH] v0.4.0 unsorted mode --- CHANGELOG.md | 15 +++++++++++ lib/indexed.ex | 35 +++++++++++++++----------- lib/indexed/actions/create_view.ex | 7 ------ lib/indexed/actions/put.ex | 21 ++++++---------- lib/indexed/actions/warm.ex | 30 ++++++---------------- lib/indexed/entity.ex | 14 ++++++----- lib/indexed/managed.ex | 2 +- lib/indexed/managed/prepare.ex | 11 -------- lib/indexed/uniques_bundle.ex | 10 -------- mix.exs | 2 +- test/indexed/indexed_test.exs | 20 +++++++++++++++ test/indexed/managed_test.exs | 9 +++++-- test/support/managed/blog_server.ex | 2 +- test/support/managed/blog_server_nt.ex | 1 + 14 files changed, 88 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac4be13..b1b67ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2023-02-27 + +### Changed +- `fields` list was optional, but the behavior when none declared has changed. + - Before, the `id_key` (default `:id`) was automatically used in the + background. This meant that presorted indexes (ascending and descending) + were auto-maintained and used as the default when calling `get_records` etc. + - Now, no such presorted indexes will be maintained. When querying, + `:ets.tab_to_list/2` will be used, with no particular order being promised. + +## [0.3.3] - 2023-02-07 + +### Changed +- Minor dialyzer fix. + ## [0.3.2] - 2023-02-03 ### Added diff --git a/lib/indexed.ex b/lib/indexed.ex index a030532..10f179d 100644 --- a/lib/indexed.ex +++ b/lib/indexed.ex @@ -8,7 +8,7 @@ defmodule Indexed do @moduledoc """ Tools for creating an index. """ - alias Indexed.View + alias Indexed.{Entity, View} alias __MODULE__ # Baseline opts. Others such as :named_table may be added. @@ -21,7 +21,7 @@ defmodule Indexed do * `:index_ref` - ETS table reference for all indexed data. """ @type t :: %Indexed{ - entities: %{atom => Indexed.Entity.t()}, + entities: %{atom => Entity.t()}, index_ref: table_ref } @@ -83,7 +83,7 @@ defmodule Indexed do @doc "Get a record by id from the index." @spec get(t, atom, id, any) :: any def get(index, entity_name, id, default \\ nil) do - case :ets.lookup(Map.fetch!(index.entities, entity_name).ref, id) do + case :ets.lookup(ref(index, entity_name), id) do [{_, val}] -> val [] -> default end @@ -170,17 +170,17 @@ defmodule Indexed do sub-section of the data should be queried. Default is `nil` - no prefilter. """ @spec get_records(t, atom, prefilter, order_hint | nil) :: [record] - def get_records(index, entity_name, prefilter, order_hint \\ nil) do - default_order_hint = fn -> - path = [Access.key(:entities), entity_name, Access.key(:fields)] - index |> get_in(path) |> hd() |> elem(0) + def get_records(index, entity_name, prefilter \\ nil, order_hint \\ nil) do + if ord = order_hint || default_order_hint(index, entity_name) do + index + |> get_index(entity_name, prefilter, ord) + |> Enum.map(&get(index, entity_name, &1)) + else + index + |> ref(entity_name) + |> :ets.tab2list() + |> Enum.map(&elem(&1, 1)) end - - order_hint = order_hint || default_order_hint.() - - index - |> get_index(entity_name, prefilter, order_hint) - |> Enum.map(&get(index, entity_name, &1)) end @doc "Cache key for a given entity, field and direction." @@ -243,12 +243,17 @@ defmodule Indexed do @doc """ Get the name of the first indexed field for an entity. + If none, return `nil`, meaning there are no ordering indexes. Good order_hint default. """ @spec default_order_hint(t, atom) :: atom def default_order_hint(index, entity_name) do - k = &Access.key(&1) - index |> get_in([k.(:entities), entity_name, k.(:fields)]) |> hd() |> elem(0) + path = [Access.key(:entities), entity_name, Access.key(:fields)] + + case get_in(index, path) do + [{name, _} | _] -> name + _ -> nil + end end @doc "Delete all ETS tables associated with the given index." diff --git a/lib/indexed/actions/create_view.ex b/lib/indexed/actions/create_view.ex index ee81816..c3ae808 100644 --- a/lib/indexed/actions/create_view.ex +++ b/lib/indexed/actions/create_view.ex @@ -4,7 +4,6 @@ defmodule Indexed.Actions.CreateView do """ alias Indexed.Actions.Warm alias Indexed.View - require Logger @typep id :: any @@ -29,8 +28,6 @@ defmodule Indexed.Actions.CreateView do """ @spec run(Indexed.t(), atom, View.fingerprint(), keyword) :: {:ok, View.t()} | :error def run(index, entity_name, fingerprint, opts \\ []) do - Logger.debug("Creating #{entity_name} view: #{fingerprint}") - entity = Map.fetch!(index.entities, entity_name) prefilter = opts[:prefilter] @@ -113,8 +110,6 @@ defmodule Indexed.Actions.CreateView do map_key = Indexed.uniques_map_key(entity_name, fingerprint, field_name) list_key = Indexed.uniques_list_key(entity_name, fingerprint, field_name) - Logger.debug(" * Saving #{field_name} uniques with #{map_size(counts_map)} values.") - :ets.insert(index.index_ref, {map_key, counts_map}) :ets.insert(index.index_ref, {list_key, list}) end @@ -141,8 +136,6 @@ defmodule Indexed.Actions.CreateView do asc_key = Indexed.index_key(entity_name, fingerprint, {:asc, field_name}) desc_key = Indexed.index_key(entity_name, fingerprint, {:desc, field_name}) - Logger.debug(" * Saving #{field_name} index with #{length(sorted_ids)} ids.") - :ets.insert(index.index_ref, {asc_key, sorted_ids}) :ets.insert(index.index_ref, {desc_key, Enum.reverse(sorted_ids)}) end diff --git a/lib/indexed/actions/put.ex b/lib/indexed/actions/put.ex index 217d342..2719fbd 100644 --- a/lib/indexed/actions/put.ex +++ b/lib/indexed/actions/put.ex @@ -3,19 +3,18 @@ defmodule Indexed.Actions.Put do import Indexed.Helpers, only: [add_to_lookup: 4, id: 1] alias Indexed.{Entity, UniquesBundle, View} alias __MODULE__ - require Logger defstruct [:current_view, :entity_name, :index, :previous, :pubsub, :record] @typedoc """ - * `:current_view` - View struct currently being updated. - * `:entity_name` - Entity name being operated on. - * `:index` - See `t:Indexed.t/0`. - * `:previous` - The previous version of the record. `nil` if none. - * `:pubsub` - If configured, a Phoenix.PubSub module to send view updates. - * `:record` - The new record being added in the put operation. + - `current_view` : View struct currently being updated. + - `entity_name` : Entity name being operated on. + - `index` : See `t:Indexed.t/0`. + - `previous` : The previous version of the record. `nil` if none. + - `pubsub` : If configured, a Phoenix.PubSub module to send view updates. + - `record` : The new record being added in the put operation. """ - @type t :: %__MODULE__{ + @type t :: %Put{ current_view: View.t() | nil, entity_name: atom, index: Indexed.t(), @@ -48,8 +47,6 @@ defmodule Indexed.Actions.Put do end defp do_run(%{entity_name: name, index: index} = put, id) do - Logger.debug("Putting into #{put.entity_name}: id #{id}") - %{fields: fields, lookups: lookups, prefilters: prefilters, ref: ref} = Map.fetch!(index.entities, name) @@ -63,8 +60,6 @@ defmodule Indexed.Actions.Put do update_all_uniques(put, pf_opts[:maintain_unique] || [], nil, false) {pf_key, pf_opts} -> - Logger.debug("--> Getting UB for #{name} prefilter nil, field: #{pf_key}") - handle_prefilter_value = fn pnb, value, new_value? -> prefilter = {pf_key, value} update_index_for_fields(put, prefilter, fields, new_value?) @@ -199,8 +194,6 @@ defmodule Indexed.Actions.Put do %{previous: previous, record: record} = put Enum.each(fields, fn {field_name, _} = field -> - Logger.debug("--> Updating index for PF #{inspect(prefilter)}, field: #{field_name}") - this_under_pf = under_prefilter?(put, record, prefilter) prev_under_pf = previous && under_prefilter?(put, previous, prefilter) record_value = Map.get(record, field_name) diff --git a/lib/indexed/actions/warm.ex b/lib/indexed/actions/warm.ex index b31e13d..ff41805 100644 --- a/lib/indexed/actions/warm.ex +++ b/lib/indexed/actions/warm.ex @@ -3,7 +3,6 @@ defmodule Indexed.Actions.Warm do import Indexed.Helpers, only: [add_to_lookup: 4, id: 2] alias Indexed.{Entity, UniquesBundle} alias __MODULE__ - require Logger defstruct [:data_tuple, :entity_name, :id_key, :index_ref] @@ -55,9 +54,10 @@ defmodule Indexed.Actions.Warm do as keys and keyword lists of options are values. Allowed options are as follows: - * `:fields` - List of field name atoms to index by. At least one required. + * `:fields` - List of field name atoms to index by. * If field is a DateTime, use sort: `{:my_field, sort: :datetime}`. * Ascending and descending will be indexed for each field. + * If none, sorted results will not be available. * `:id_key` - Primary key to use in indexes and for accessing the records of this entity. See `t:Indexed.Entity.t/0`. Default: `:id`. * `:lookups` - See `t:Indexed.Entity.t/0`. @@ -145,13 +145,11 @@ defmodule Indexed.Actions.Warm do index_ref: index.index_ref } - Logger.debug("Warming #{entity_name}...") - # Create and insert the caches for this entity: for each prefilter # configured, build & store indexes for each indexed field. - # Internally, a `t:prefilter/0` refers to a `{:my_field, "possible - # value"}` tuple or `nil` which we implicitly include, where no - # prefilter is applied. + # Internally, a `t:prefilter/0` refers to a `{:my_field, "some value"}` + # tuple or `nil` which we implicitly include, + # where no prefilter is applied. for prefilter_config <- entity.prefilters, do: warm_prefilter(warm, prefilter_config, entity.fields) @@ -190,11 +188,6 @@ defmodule Indexed.Actions.Warm do warm_sorted(warm, prefilter, field, data_tuple) end - Logger.debug(""" - * Putting index (PF #{pf_key || "nil"}) for \ - #{inspect(Enum.map(fields, &elem(&1, 0)))}\ - """) - if nil == pf_key do Enum.each(fields, &warm_sorted.(nil, &1, full_data)) @@ -216,8 +209,6 @@ defmodule Indexed.Actions.Warm do bundle = UniquesBundle.new(counts_map, Enum.sort(Enum.uniq(list))) UniquesBundle.put(bundle, index_ref, entity_name, nil, pf_key, new?: true) - Logger.debug("--> Putting UB (for #{pf_key}) with #{map_size(counts_map)} elements.") - # For each value found for the prefilter, create a set of indexes. Enum.each(grouped, fn {pf_val, data} -> prefilter = {pf_key, pf_val} @@ -276,21 +267,14 @@ defmodule Indexed.Actions.Warm do list = counts_map |> Map.keys() |> Enum.sort() bundle = UniquesBundle.new(counts_map, list) - Logger.debug(""" - --> Putting UB (PF #{inspect(prefilter)}, #{field_name}) \ - with #{map_size(counts_map)} elements."\ - """) - UniquesBundle.put(bundle, index_ref, entity_name, prefilter, field_name, new?: true) end) end @doc "Normalize fields." - @spec resolve_fields_opt([atom | Entity.field()], atom) :: [Entity.field()] + @spec resolve_fields_opt([atom | Entity.field()] | nil, atom) :: [Entity.field()] def resolve_fields_opt(fields, _entity_name) do - # match?([_ | _], fields) || raise "At least one field to index is required on #{entity_name}." - - Enum.map(fields, fn + Enum.map(List.wrap(fields), fn {_name, _opts} = f -> f name when is_atom(name) -> {name, []} end) diff --git a/lib/indexed/entity.ex b/lib/indexed/entity.ex index fa0b378..517b617 100644 --- a/lib/indexed/entity.ex +++ b/lib/indexed/entity.ex @@ -4,7 +4,9 @@ defmodule Indexed.Entity do @typedoc """ - `fields` : List of `t:field/0`s for which pairs of lists should be - maintained with the ID sorted ascending and descending. + maintained with the ID sorted ascending and descending. If none, then no + such indexes will be maintained and queries will default to + `order_hint: nil`, meaning that result ordering is not needed. - `id_key` : Specifies how to find the id for a record. It can be an atom field name to access, a function, or a tuple in the form `{module, function_name}`. In the latter two cases, the record will be passed in. @@ -20,16 +22,16 @@ defmodule Indexed.Entity do unique values under the prefilter will be managed. These lists can be fetched via `Indexed.get_uniques_list/4` and `Indexed.get_uniques_map/4`. - - `ref` : ETS table reference where records of this entity type are - stored, keyed by id. This will be nil in the version compiled into a managed - module. + - `ref` : ETS table reference where records of + this entity type are stored, keyed by id. + (`nil` in the instances compiled into a managed module.) """ @type t :: %__MODULE__{ - fields: [field], + fields: [field] | nil, id_key: any, lookups: [field_name], prefilters: [prefilter_config], - ref: :ets.tid() | atom | nil + ref: Indexed.table_ref() | nil } @typedoc """ diff --git a/lib/indexed/managed.ex b/lib/indexed/managed.ex index 38c64a7..f85ece1 100644 --- a/lib/indexed/managed.ex +++ b/lib/indexed/managed.ex @@ -146,7 +146,7 @@ defmodule Indexed.Managed do """ @type t :: %Managed{ children: children, - fields: [atom | Entity.field()], + fields: [atom | Entity.field()] | nil, id_key: id_key, lookups: [Entity.field_name()], query: (Ecto.Queryable.t() -> Ecto.Queryable.t()) | nil, diff --git a/lib/indexed/managed/prepare.ex b/lib/indexed/managed/prepare.ex index a6c037c..e430f83 100644 --- a/lib/indexed/managed/prepare.ex +++ b/lib/indexed/managed/prepare.ex @@ -22,7 +22,6 @@ defmodule Indexed.Managed.Prepare do manageds |> map_put.(:children, &do_rewrite_children/2) |> map_put.(:prefilters, &do_rewrite_prefilters/2) - |> map_put.(:fields, &do_rewrite_fields/2) |> map_put.(:tracked, &do_rewrite_tracked/2) end @@ -84,16 +83,6 @@ defmodule Indexed.Managed.Prepare do else: finish.(required) end - # If :fields is empty, use the id key or the first field given by Ecto. - @spec do_rewrite_fields(Managed.t(), [Managed.t()]) :: [atom | Entity.field()] - defp do_rewrite_fields(%{fields: [], id_key: id_key}, _) when is_atom(id_key), - do: [id_key] - - defp do_rewrite_fields(%{fields: [], module: mod}, _), - do: [hd(mod.__schema__(:fields))] - - defp do_rewrite_fields(%{fields: fields}, _), do: fields - # Return true for tracked if another entity has a :one association to us. @spec do_rewrite_tracked(Managed.t(), [Managed.t()]) :: boolean defp do_rewrite_tracked(%{name: name}, manageds) do diff --git a/lib/indexed/uniques_bundle.ex b/lib/indexed/uniques_bundle.ex index 9a92897..42e35ea 100644 --- a/lib/indexed/uniques_bundle.ex +++ b/lib/indexed/uniques_bundle.ex @@ -9,7 +9,6 @@ defmodule Indexed.UniquesBundle do of these values. """ alias __MODULE__ - require Logger @typedoc """ A 3-element tuple defines unique values under a prefilter: @@ -66,11 +65,6 @@ defmodule Indexed.UniquesBundle do map = Indexed.get_uniques_map(index, entity_name, prefilter, field_name) list = Indexed.get_uniques_list(index, entity_name, prefilter, field_name) - Logger.debug(fn -> - key = Indexed.uniques_map_key(entity_name, prefilter, field_name) - "UB: Getting #{key}: #{inspect(map)}" - end) - {map, list, [], false} end @@ -121,8 +115,6 @@ defmodule Indexed.UniquesBundle do field_pf? = is_tuple(prefilter) if not new? and field_pf? and Enum.empty?(list) do - Logger.debug("UB: Dropping #{counts_key}") - # This prefilter has ran out of records -- delete the ETS table. # Note that for views (binary prefilter) and the global prefilter (nil) # we allow the uniques to remain in the empty state. They go when the @@ -130,8 +122,6 @@ defmodule Indexed.UniquesBundle do :ets.delete(index_ref, list_key) :ets.delete(index_ref, counts_key) else - Logger.debug("UB: Putting #{counts_key}: #{inspect(counts_map)}") - if new? or list_updated?, do: :ets.insert(index_ref, {list_key, list}), else: list diff --git a/mix.exs b/mix.exs index 937fa25..7d59555 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule Indexed.MixProject do use Mix.Project @source_url "https://github.com/djthread/indexed" - @version "0.3.3" + @version "0.3.4" def project do [ diff --git a/test/indexed/indexed_test.exs b/test/indexed/indexed_test.exs index c0ab2d6..ff69e9f 100644 --- a/test/indexed/indexed_test.exs +++ b/test/indexed/indexed_test.exs @@ -285,4 +285,24 @@ defmodule IndexedTest do assert %Car{id: 1, make: "Lamborghini"} == Indexed.get(index, :cars, 1) end + + test "unsorted mode" do + index = Indexed.warm(cars: [data: @cars]) + + has? = fn makes -> + recs = Indexed.get_records(index, :cars) + Enum.all?(makes, fn m -> Enum.any?(recs, &(&1.make == m)) end) + end + + assert has?.(~w(Lamborghini Mazda)) + + Indexed.put(index, :cars, %Car{id: 3, make: "Batmobile"}) + + assert has?.(~w(Lamborghini Mazda Batmobile)) + + Indexed.drop(index, :cars, 3) + + refute has?.(~w(Batmobile)) + assert has?.(~w(Lamborghini Mazda)) + end end diff --git a/test/indexed/managed_test.exs b/test/indexed/managed_test.exs index 2f91891..320fc1f 100644 --- a/test/indexed/managed_test.exs +++ b/test/indexed/managed_test.exs @@ -87,14 +87,19 @@ defmodule Indexed.ManagedTest do } ] = entries() - assert [%{name: "fred"}, %{name: "jill"}, %{name: "lee"}, %{name: "lucy"}] = records(:users) + has? = fn recs, names, length -> + Enum.all?(names, fn n -> Enum.any?(recs, &(&1.name == n)) end) and length == length(recs) + end + + assert has?.(records(:users), ~w(fred jill lee lucy), 4) + assert %{^bob_id => 5, ^jill_id => 2, ^lee_id => 1, ^lucy_id => 1} = tracking(bs_pid, :users) {:ok, _} = Blog.delete_comment(comment_id) msg = "user-#{jill_id}" assert_receive [:unsubscribe, ^msg] - assert [%{name: "fred"}, %{name: "lee"}, %{name: "lucy"}] = records(:users) + assert has?.(records(:users), ~w(fred lee lucy), 3) assert %{bob_id => 5, lee_id => 2, lucy_id => 1} == tracking(bs_pid, :users) assert [%{comments: [%{content: "woah"}]}, %{comments: [_, _]}] = entries() diff --git a/test/support/managed/blog_server.ex b/test/support/managed/blog_server.ex index 76d62e4..d3711af 100644 --- a/test/support/managed/blog_server.ex +++ b/test/support/managed/blog_server.ex @@ -21,9 +21,9 @@ defmodule BlogServer do fields: [:inserted_at], manage_path: [author: :flare_pieces] + # No fields declared: Unsorted mode! managed :users, User, children: [:best_friend, :flare_pieces], - indexes: [:name], prefilters: [:name], subscribe: &Blog.subscribe_to_user/1, unsubscribe: &Blog.unsubscribe_from_user/1 diff --git a/test/support/managed/blog_server_nt.ex b/test/support/managed/blog_server_nt.ex index a1fd1e4..bc841cd 100644 --- a/test/support/managed/blog_server_nt.ex +++ b/test/support/managed/blog_server_nt.ex @@ -26,6 +26,7 @@ defmodule BlogServerNT do manage_path: [author: :flare_pieces] managed :users, User, + fields: [:id], children: [:best_friend, :flare_pieces], prefilters: [:name], lookups: [:name],