diff --git a/migration/20150803-add_custom_list_entry_licensepool.sql b/migration/20150803-add_custom_list_entry_licensepool.sql deleted file mode 100644 index f58fdaa0..00000000 --- a/migration/20150803-add_custom_list_entry_licensepool.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE customlistentries ADD COLUMN license_pool_id integer; -CREATE INDEX "ix_customlistentries_license_pool_id" ON customlistentries (license_pool_id); -ALTER TABLE customlistentries ADD CONSTRAINT customlistentries_license_pool_id_fkey FOREIGN KEY (license_pool_id) REFERENCES licensepools(id); diff --git a/migration/20150813_set_open_access_download_url.py b/migration/20150813_set_open_access_download_url.py deleted file mode 100644 index 391a8242..00000000 --- a/migration/20150813_set_open_access_download_url.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -"""Set Edition.open_access_download_url for all Project Gutenberg books.""" -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) -from core.model import ( - DataSource, - DeliveryMechanism, - Edition, - Representation, - production_session, -) -from core.monitor import EditionSweepMonitor -from core.scripts import RunMonitorScript - -set_delivery_mechanism = len(sys.argv) > 1 and sys.argv[1] == "delivery" - - -class OpenAccessDownloadSetMonitor(EditionSweepMonitor): - """Set the open-access link f.""" - - def __init__(self, _db, interval_seconds=None): - super(OpenAccessDownloadSetMonitor, self).__init__( - _db, "Open Access Download link set", interval_seconds, batch_size=100 - ) - - def edition_query(self): - gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) - return self._db.query(Edition).filter(Edition.data_source == gutenberg) - - def process_edition(self, edition): - edition.set_open_access_link() - if set_delivery_mechanism: - link = edition.best_open_access_link - if link: - print(edition.id, edition.title, link.url) - edition.license_pool.set_delivery_mechanism( - Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.NO_DRM, link - ) - else: - print(edition.id, edition.title, "[no link]") - return True - - -RunMonitorScript(OpenAccessDownloadSetMonitor).run() diff --git a/migration/20150826-1-change_target_age_from_int_to_range.sql b/migration/20150826-1-change_target_age_from_int_to_range.sql deleted file mode 100644 index da516bd5..00000000 --- a/migration/20150826-1-change_target_age_from_int_to_range.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP MATERIALIZED VIEW mv_works_editions_datasources_identifiers; -DROP MATERIALIZED VIEW mv_works_editions_workgenres_datasources_identifiers; - -ALTER TABLE works add COLUMN target_age_2 NUMRANGE; -ALTER TABLE works DROP COLUMN target_age; -ALTER TABLE works RENAME COLUMN target_age_2 TO target_age; -CREATE INDEX "ix_works_target_age" ON works (target_age); - -ALTER TABLE subjects add COLUMN target_age_2 NUMRANGE; -ALTER TABLE subjects DROP COLUMN target_age; -ALTER TABLE subjects RENAME COLUMN target_age_2 TO target_age; -CREATE INDEX "ix_subjects_target_age" ON subjects (target_age); diff --git a/migration/20150826-2-recreate-views.sql b/migration/20150826-2-recreate-views.sql deleted file mode 100644 index 5e190a15..00000000 --- a/migration/20150826-2-recreate-views.sql +++ /dev/null @@ -1,71 +0,0 @@ -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.id AS works_id, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND editions.medium = 'Book'::medium AND works.simple_opds_entry IS NOT NULL - ORDER BY editions.sort_title, editions.sort_author; - -create materialized view mv_works_editions_datasources_identifiers -as - SELECT editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.id AS works_id, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND editions.medium = 'Book'::medium AND works.simple_opds_entry IS NOT NULL - ORDER BY editions.sort_title, editions.sort_author; diff --git a/migration/20150826-3-recreate-view-indexes.sql b/migration/20150826-3-recreate-view-indexes.sql deleted file mode 100644 index caf564b0..00000000 --- a/migration/20150826-3-recreate-view-indexes.sql +++ /dev/null @@ -1,33 +0,0 @@ -create index mv_works_editions_adult_fiction_author_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_w_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_w_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_yachild_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -; -create index mv_works_editions_adult_fiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_nfiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_yachild_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); -create unique index mv_works_editions_workgenres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); diff --git a/migration/20150826-4-recalculate_age_range_for_childrens_books.py b/migration/20150826-4-recalculate_age_range_for_childrens_books.py deleted file mode 100644 index 79727a8e..00000000 --- a/migration/20150826-4-recalculate_age_range_for_childrens_books.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -"""Recalculate the age range for all subjects whose audience is Children or Young Adult.""" -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) -from core.classifier import Classifier -from core.model import DataSource, Edition, Subject, production_session -from core.monitor import SubjectSweepMonitor -from core.scripts import RunMonitorScript - - -class RecalculateAgeRangeMonitor(SubjectSweepMonitor): - """Recalculate the age range for every young adult or children's subject.""" - - def __init__(self, _db, interval_seconds=None): - super(RecalculateAgeRangeMonitor, self).__init__( - _db, - "20150825 migration - Recalculate age range for children's books", - interval_seconds, - batch_size=1000, - ) - - def subject_query(self): - audiences = [Classifier.AUDIENCE_YOUNG_ADULT, Classifier.AUDIENCE_CHILDREN] - return self._db.query(Subject).filter(Subject.audience.in_(audiences)) - - def process_identifier(self, subject): - old_target_age = subject.target_age - subject.assign_to_genre() - if subject.target_age != old_target_age and subject.target_age.lower != None: - print(("%r: %r->%r" % (subject, old_target_age, subject.target_age))) - - -RunMonitorScript(RecalculateAgeRangeMonitor).run() diff --git a/migration/20150826-5-recalculate_work_presentation_for_childrens_books.py b/migration/20150826-5-recalculate_work_presentation_for_childrens_books.py deleted file mode 100644 index b0a028b1..00000000 --- a/migration/20150826-5-recalculate_work_presentation_for_childrens_books.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -"""Recalculate the age range for all subjects whose audience is Children or Young Adult.""" -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) -from core.classifier import Classifier -from core.model import ( - DataSource, - Edition, - Identifier, - Subject, - Work, - production_session, -) -from core.monitor import WorkSweepMonitor -from core.scripts import RunMonitorScript -from psycopg2.extras import NumericRange - - -class RecalculateAgeRangeMonitor(WorkSweepMonitor): - """Recalculate the age range for every young adult or children's book.""" - - def __init__(self, _db, interval_seconds=None): - super(RecalculateAgeRangeMonitor, self).__init__( - _db, - "20150825 migration - Recalculate age range for children's books (Works)", - interval_seconds, - batch_size=10, - ) - - def work_query(self): - audiences = [Classifier.AUDIENCE_YOUNG_ADULT, Classifier.AUDIENCE_CHILDREN] - return self._db.query(Work).filter(Work.audience.in_(audiences)) - - def process_work(self, work): - primary_identifier_ids = [x.primary_identifier.id for x in work.editions] - data = Identifier.recursively_equivalent_identifier_ids( - self._db, primary_identifier_ids, 5, threshold=0.5 - ) - flattened_data = Identifier.flatten_identifier_ids(data) - workgenres, work.fiction, work.audience, target_age = work.assign_genres( - flattened_data - ) - old_target_age = work.target_age - work.target_age = NumericRange(*target_age) - if work.target_age != old_target_age and work.target_age.lower != None: - print("%r: %r->%r" % (work.title, old_target_age, work.target_age)) - - -RunMonitorScript(RecalculateAgeRangeMonitor).run() diff --git a/migration/20150828-recalculate_age_range_for_childrens_books.py b/migration/20150828-recalculate_age_range_for_childrens_books.py deleted file mode 100644 index df1dcedd..00000000 --- a/migration/20150828-recalculate_age_range_for_childrens_books.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -"""Recalculate the age range for all subjects whose audience is Children or Young Adult.""" -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) -from core.classifier import Classifier -from core.model import DataSource, Edition, Subject, production_session -from core.monitor import SubjectSweepMonitor -from core.scripts import RunMonitorScript - - -class RecalculateAgeRangeMonitor(SubjectSweepMonitor): - """Recalculate the age range for every young adult or children's subject.""" - - def __init__(self, _db, interval_seconds=None): - super(RecalculateAgeRangeMonitor, self).__init__( - _db, - "20150825 migration - Recalculate age range for children's books", - interval_seconds, - batch_size=1000, - ) - - def subject_query(self): - audiences = [Classifier.AUDIENCE_YOUNG_ADULT, Classifier.AUDIENCE_CHILDREN] - return self._db.query(Subject).filter(Subject.audience.in_(audiences)) - - def process_identifier(self, subject): - old_target_age = subject.target_age - subject.assign_to_genre() - if subject.target_age != old_target_age and subject.target_age.lower != None: - print("%r: %r->%r" % (subject, old_target_age, subject.target_age)) - - -RunMonitorScript(RecalculateAgeRangeMonitor).run() diff --git a/migration/20150925-add-contributor-biography.sql b/migration/20150925-add-contributor-biography.sql deleted file mode 100644 index 658c4a4c..00000000 --- a/migration/20150925-add-contributor-biography.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE contributors add COLUMN biography varchar; diff --git a/migration/20150930-1-recreate-views.sql b/migration/20150930-1-recreate-views.sql deleted file mode 100644 index 2f9b79df..00000000 --- a/migration/20150930-1-recreate-views.sql +++ /dev/null @@ -1,75 +0,0 @@ -create materialized view mv_works_editions_datasources_identifiers -as - SELECT distinct(editions.id) AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.id AS works_id, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN licensepooldeliveries on licensepools.id=licensepooldeliveries.license_pool_id - JOIN deliverymechanisms on deliverymechanisms.id=licensepooldeliveries.delivery_mechanism_id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND editions.medium = 'Book'::medium AND works.simple_opds_entry IS NOT NULL AND deliverymechanisms.default_client_can_fulfill=true - ORDER BY editions.sort_title, editions.sort_author; - -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT distinct(editions.id) AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.id AS works_id, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - JOIN licensepooldeliveries on licensepools.id=licensepooldeliveries.license_pool_id - JOIN deliverymechanisms on deliverymechanisms.id=licensepooldeliveries.delivery_mechanism_id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND editions.medium = 'Book'::medium AND works.simple_opds_entry IS NOT NULL AND deliverymechanisms.default_client_can_fulfill=true - ORDER BY editions.sort_title, editions.sort_author; diff --git a/migration/20150930-2-recreate-view-indexes.sql b/migration/20150930-2-recreate-view-indexes.sql deleted file mode 100644 index caf564b0..00000000 --- a/migration/20150930-2-recreate-view-indexes.sql +++ /dev/null @@ -1,33 +0,0 @@ -create index mv_works_editions_adult_fiction_author_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_w_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_w_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_yachild_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -; -create index mv_works_editions_adult_fiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_nfiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_yachild_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); -create unique index mv_works_editions_workgenres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); diff --git a/migration/20151002-fix-audiobooks-mislabeled-as-print-books.py b/migration/20151002-fix-audiobooks-mislabeled-as-print-books.py deleted file mode 100644 index 495131e7..00000000 --- a/migration/20151002-fix-audiobooks-mislabeled-as-print-books.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -"""Fix audiobooks mislabeled as print books.""" - -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..") -sys.path.append(os.path.abspath(package_dir)) -from threem import ThreeMAPI - -from model import ( - DeliveryMechanism, - Edition, - Identifier, - LicensePool, - LicensePoolDeliveryMechanism, -) -from monitor import IdentifierSweepMonitor -from overdrive import OverdriveAPI, OverdriveRepresentationExtractor -from scripts import RunMonitorScript - - -class SetDeliveryMechanismMonitor(IdentifierSweepMonitor): - def __init__(self, _db, interval_seconds=None): - super(SetDeliveryMechanismMonitor, self).__init__( - _db, - "20151002 migration - Correct medium of mislabeled audiobooks", - interval_seconds, - batch_size=100, - ) - self.overdrive = OverdriveAPI(_db) - self.threem = ThreeMAPI(_db) - - types = [Identifier.THREEM_ID, Identifier.OVERDRIVE_ID, Identifier.AXIS_360_ID] - - content_types = [ - "application/epub+zip", - "application/pdf", - "Kindle via Amazon", - "Streaming Text", - ] - - def identifier_query(self): - qu = ( - self._db.query(Identifier) - .join(Identifier.licensed_through) - .join(LicensePool.delivery_mechanisms) - .join(LicensePoolDeliveryMechanism.delivery_mechanism) - .filter(Identifier.type.in_(self.types)) - .filter(~DeliveryMechanism.content_type.in_(self.content_types)) - ) - return qu - - def process_identifier(self, identifier): - # What is the correct medium? - correct_medium = None - lp = identifier.licensed_through - for lpdm in lp.delivery_mechanisms: - correct_medium = lpdm.delivery_mechanism.implicit_medium - if correct_medium: - break - if not correct_medium and identifier.type == Identifier.OVERDRIVE_ID: - content = self.overdrive.metadata_lookup(identifier) - metadata = OverdriveRepresentationExtractor.book_info_to_metadata(content) - correct_medium = metadata.medium - - if not correct_medium and identifier.type == Identifier.THREEM_ID: - metadata = self.threem.bibliographic_lookup(identifier) - correct_medium = metadata.medium - - if not correct_medium: - set_trace() - - if lp.edition.medium != correct_medium: - print( - ( - "%s is actually %s, not %s" - % (lp.edition.title, correct_medium, lp.edition.medium) - ) - ) - lp.edition.medium = correct_medium or Edition.BOOK_MEDIUM - - -RunMonitorScript(SetDeliveryMechanismMonitor).run() diff --git a/migration/20151019-recalculate-fiction-status-for-overdrive-subjects.py b/migration/20151019-recalculate-fiction-status-for-overdrive-subjects.py deleted file mode 100644 index 9859a59a..00000000 --- a/migration/20151019-recalculate-fiction-status-for-overdrive-subjects.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -"""Recalculate the age range for all subjects whose audience is Children or Young Adult.""" -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) -from core.classifier import Classifier -from core.model import DataSource, Edition, Subject, production_session -from core.monitor import SubjectSweepMonitor -from core.scripts import RunMonitorScript - - -class RecalculateFictionStatusMonitor(SubjectSweepMonitor): - """Recalculate the age range for every young adult or children's subject.""" - - def __init__(self, _db, interval_seconds=None): - super(RecalculateFictionStatusMonitor, self).__init__( - _db, - "20150825 migration - Recalculate age range for children's books", - interval_seconds, - batch_size=1000, - ) - - def subject_query(self): - return self._db.query(Subject).filter(Subject.type == Subject.OVERDRIVE) - - def process_identifier(self, subject): - old_fiction = subject.fiction - old_audience = subject.audience - subject.assign_to_genre() - print("%s %s %s" % (subject.identifier, subject.fiction, subject.audience)) - - -RunMonitorScript(RecalculateFictionStatusMonitor).run() diff --git a/migration/20151020-fix-dewey-304.py b/migration/20151020-fix-dewey-304.py deleted file mode 100644 index ac9f4441..00000000 --- a/migration/20151020-fix-dewey-304.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -"""DDC 304 was incorrectly classified under Periodicals. 304 is Social Sciences, 305 is Periodicals. - -That has now been fixed. - -Correct all 304.* and 305.* subjects, and reclassify every work classified under those IDs. -""" -import logging -import os -import sys -from pdb import set_trace - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) -from core.model import ( - Edition, - Genre, - Identifier, - Subject, - Work, - WorkGenre, - production_session, -) - -_db = production_session() - - -def reclassify(ddc): - log = logging.getLogger("Migration script - Fix Dewey %s" % ddc) - for subject in ( - _db.query(Subject) - .filter(Subject.type == Subject.DDC) - .filter(Subject.identifier.like(ddc + "%")) - ): - log.info("Considering subject %s/%s", subject.identifier, subject.name) - subject.assign_to_genre() - for cl in subject.classifications: - ids = cl.identifier.equivalent_identifier_ids() - log.info("Looking for editions associated with %d ids.", len(ids)) - editions = ( - _db.query(Edition).filter(Edition.primary_identifier_id.in_(ids)).all() - ) - for edition in editions: - if edition.work: - old_genres = set(edition.work.genres) - edition.work.calculate_presentation() - if old_genres != set(edition.work.genres): - log.info( - "%s GENRE CHANGE: %r -> %r", - edition.title, - old_genres, - edition.work.genres, - ) - else: - edition.calculate_presentation() - _db.commit() - - -reclassify("205") -reclassify("304") -reclassify("305") diff --git a/migration/20151021-set-license-pool-for-list-membership.py b/migration/20151021-set-license-pool-for-list-membership.py deleted file mode 100644 index 6848f5f1..00000000 --- a/migration/20151021-set-license-pool-for-list-membership.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -"""Make sure every CustomListEntry has a LicensePool set. -""" -import logging -import os -import sys -from pdb import set_trace - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) - - -from core.model import CustomListEntry, production_session - -_db = production_session() - -qu = _db.query(CustomListEntry).filter(CustomListEntry.license_pool == None) -print("Fixing %d custom list entries with no licensepool." % qu.count()) - -for cle in qu: - cle.set_license_pool() diff --git a/migration/20151104-add-loan-delivery-mechanism.sql b/migration/20151104-add-loan-delivery-mechanism.sql deleted file mode 100644 index eabefa73..00000000 --- a/migration/20151104-add-loan-delivery-mechanism.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE loans ADD COLUMN fulfillment_id integer; -ALTER TABLE loans ADD CONSTRAINT loans_fulfillment_id_fkey FOREIGN KEY (fulfillment_id) REFERENCES licensepooldeliveries(id); diff --git a/migration/20151207-1-recreate-views.sql b/migration/20151207-1-recreate-views.sql deleted file mode 100644 index 72bc3715..00000000 --- a/migration/20151207-1-recreate-views.sql +++ /dev/null @@ -1,111 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY editions.sort_title, editions.sort_author; - -create index mv_works_editions_adult_fiction_author_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_w_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_w_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_yachild_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -; -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY (editions.sort_title, editions.sort_author); - -create index mv_works_editions_adult_fiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_nfiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_yachild_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create unique index mv_works_editions_workgenres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); diff --git a/migration/20151210-1-add-patron-username.sql b/migration/20151210-1-add-patron-username.sql deleted file mode 100644 index 73cdc26f..00000000 --- a/migration/20151210-1-add-patron-username.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE patrons add COLUMN username varchar; diff --git a/migration/20151210-2-add-patron-fines.sql b/migration/20151210-2-add-patron-fines.sql deleted file mode 100644 index 299864ce..00000000 --- a/migration/20151210-2-add-patron-fines.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE patrons add COLUMN fines varchar; diff --git a/migration/20160105-recreate-views.sql b/migration/20160105-recreate-views.sql deleted file mode 100644 index c6699f2a..00000000 --- a/migration/20160105-recreate-views.sql +++ /dev/null @@ -1,113 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY editions.sort_title, editions.sort_author; - -create index mv_works_editions_adult_fiction_author_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_w_ds_id on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_fiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_w_ds_id on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_author_other_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; -create index mv_works_editions_adult_nfiction_title_other_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_yachild_fiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_iden on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_iden on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -; -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY (editions.sort_title, editions.sort_author); - -create index mv_works_editions_adult_fiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_fiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; -create index mv_works_editions_adult_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language='eng'; -create index mv_works_editions_adult_nfiction_author_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_g_ds_id on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_adult_nfiction_title_other_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; -create index mv_works_editions_adult_nfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, license_pool_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language='eng'; -create index mv_works_editions_yachild_fiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_fiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; -create index mv_works_editions_yachild_nonfiction_author_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create index mv_works_editions_yachild_nonfiction_title_wg_iden on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; -create unique index mv_works_editions_workgenres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); diff --git a/migration/20160121-recreate-materialized-views.sql b/migration/20160121-recreate-materialized-views.sql deleted file mode 100644 index 46fbb610..00000000 --- a/migration/20160121-recreate-materialized-views.sql +++ /dev/null @@ -1,207 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.was_merged_into_id IS NULL - AND works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -------------------------------------------------- - - ---------------------------------- - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.was_merged_into_id IS NULL AND works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20160209-materialized-view-indexes.sql b/migration/20160209-materialized-view-indexes.sql deleted file mode 100644 index fc7b051e..00000000 --- a/migration/20160209-materialized-view-indexes.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); diff --git a/migration/20160229-suppress-licensepool.sql b/migration/20160229-suppress-licensepool.sql deleted file mode 100644 index b0897fad..00000000 --- a/migration/20160229-suppress-licensepool.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE licensepools ADD COLUMN suppressed boolean default false; -create index "ix_licensepools_suppressed" on licensepools (suppressed); diff --git a/migration/20160324-coverage-record-operation.sql b/migration/20160324-coverage-record-operation.sql deleted file mode 100644 index bb542f8c..00000000 --- a/migration/20160324-coverage-record-operation.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE coveragerecords RENAME date TO timestamp; -ALTER TABLE coveragerecords ALTER COLUMN timestamp SET DATA TYPE timestamp; -ALTER TABLE coveragerecords ADD COLUMN operation varchar(255) DEFAULT NULL; -CREATE UNIQUE INDEX "ix_coveragerecords_data_source_id_operation_identifier_id" ON coveragerecords (data_source_id, operation, identifier_id); -ALTER TABLE ONLY coveragerecords ADD CONSTRAINT coveragerecords_identifier_id_data_source_id_operation_key UNIQUE (identifier_id, data_source_id, operation); diff --git a/migration/20160408-add-complaint-resolved.sql b/migration/20160408-add-complaint-resolved.sql deleted file mode 100644 index d2c4a505..00000000 --- a/migration/20160408-add-complaint-resolved.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE complaints ADD COLUMN resolved timestamp; \ No newline at end of file diff --git a/migration/20160412-add-licensepool-superceded.sql b/migration/20160412-add-licensepool-superceded.sql deleted file mode 100644 index 3c16a165..00000000 --- a/migration/20160412-add-licensepool-superceded.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE licensepools ADD COLUMN superceded boolean DEFAULT false; diff --git a/migration/20160504-add-licensepool-exception.sql b/migration/20160504-add-licensepool-exception.sql deleted file mode 100644 index 51b5f5e9..00000000 --- a/migration/20160504-add-licensepool-exception.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE licensepools ADD COLUMN license_exception varchar; \ No newline at end of file diff --git a/migration/20160510-add-licensepool-presentation-edition-id.sql b/migration/20160510-add-licensepool-presentation-edition-id.sql deleted file mode 100644 index 7e541dc1..00000000 --- a/migration/20160510-add-licensepool-presentation-edition-id.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE licensepools ADD COLUMN presentation_edition_id integer; -CREATE INDEX ix_licensepools_presentation_edition_id ON licensepools USING btree (presentation_edition_id); -ALTER TABLE ONLY licensepools ADD CONSTRAINT licensepools_presentation_edition_id_fkey FOREIGN KEY (presentation_edition_id) REFERENCES editions(id); -update licensepools set presentation_edition_id = (select id from editions e where licensepools.data_source_id=e.data_source_id and licensepools.identifier_id=e.primary_identifier_id); diff --git a/migration/20160511-add-edition-series-position.sql b/migration/20160511-add-edition-series-position.sql deleted file mode 100644 index 62b40056..00000000 --- a/migration/20160511-add-edition-series-position.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE editions ADD COLUMN series_position integer; diff --git a/migration/20160512-remove-redundant-equivalencies.sql b/migration/20160512-remove-redundant-equivalencies.sql deleted file mode 100644 index daa3de94..00000000 --- a/migration/20160512-remove-redundant-equivalencies.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Over 70K equivalents in the Wrangler db point an identifier back to itself. --- This SQL query will delete those rows. -delete from equivalents -where id in ( - select e.id from equivalents e - join identifiers inputs on e.input_id = inputs.id - join identifiers outputs on e.output_id = outputs.id - where inputs.id = outputs.id -); diff --git a/migration/20160516-remove-work-was-merged-into.sql b/migration/20160516-remove-work-was-merged-into.sql deleted file mode 100644 index ec7b02f5..00000000 --- a/migration/20160516-remove-work-was-merged-into.sql +++ /dev/null @@ -1,219 +0,0 @@ -DELETE FROM works where was_merged_into_id is not null; -ALTER TABLE works DROP COLUMN was_merged_into_id CASCADE; - -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - - -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - FROM works - JOIN editions ON editions.work_id = works.id AND editions.is_primary_for_work = true - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20160517-remove-edition-is-primary-for-work.sql b/migration/20160517-remove-edition-is-primary-for-work.sql deleted file mode 100644 index 3d49480e..00000000 --- a/migration/20160517-remove-edition-is-primary-for-work.sql +++ /dev/null @@ -1,224 +0,0 @@ -ALTER TABLE works ADD COLUMN presentation_edition_id integer; -CREATE INDEX ix_works_presentation_edition_id on works USING btree (presentation_edition_id); -UPDATE works as w SET presentation_edition_id = e.id from editions as e where e.work_id = w.id and e.is_primary_for_work = true; -ALTER TABLE editions DROP COLUMN work_id CASCADE; -ALTER TABLE editions DROP COLUMN is_primary_for_work CASCADE; - - -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - - - -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.data_source_id = licensepools.data_source_id AND editions.primary_identifier_id = licensepools.identifier_id - JOIN datasources ON editions.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true AND works.simple_opds_entry IS NOT NULL - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20160602-1-move-rights-status-to-licensepooldeliverymechanisms.sql b/migration/20160602-1-move-rights-status-to-licensepooldeliverymechanisms.sql deleted file mode 100644 index 0eba0a37..00000000 --- a/migration/20160602-1-move-rights-status-to-licensepooldeliverymechanisms.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE licensepooldeliveries ADD COLUMN rightsstatus_id integer; -CREATE INDEX ix_licensepooldeliveries_rightsstatus_id on licensepooldeliveries USING btree (rightsstatus_id); -UPDATE licensepooldeliveries as lpd SET rightsstatus_id = lp.rightsstatus_id from licensepools as lp where lp.id = lpd.license_pool_id; -ALTER TABLE licensepools DROP COLUMN rightsstatus_id; \ No newline at end of file diff --git a/migration/20160602-2-add-license-pool-id-to-cachedfeeds.sql b/migration/20160602-2-add-license-pool-id-to-cachedfeeds.sql deleted file mode 100644 index c607a04a..00000000 --- a/migration/20160602-2-add-license-pool-id-to-cachedfeeds.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE cachedfeeds add COLUMN license_pool_id integer; diff --git a/migration/20160607-add-index-on-cachedfeeds-license-pool-id b/migration/20160607-add-index-on-cachedfeeds-license-pool-id deleted file mode 100644 index 380ab3e1..00000000 --- a/migration/20160607-add-index-on-cachedfeeds-license-pool-id +++ /dev/null @@ -1,2 +0,0 @@ -CREATE INDEX ix_cachedfeeds_license_pool_id ON cachedfeeds USING btree (license_pool_id); -ALTER TABLE ONLY cachedfeeds ADD CONSTRAINT cachedfeeds_license_pool_id_fkey FOREIGN KEY (license_pool_id) REFERENCES licensepools(id); diff --git a/migration/20160614-add-coverage-status.sql b/migration/20160614-add-coverage-status.sql deleted file mode 100644 index 95be9937..00000000 --- a/migration/20160614-add-coverage-status.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE TYPE coverage_status AS ENUM ( - 'success', - 'transient failure', - 'persistent failure' -); - -alter table coveragerecords add column status coverage_status; -alter table workcoveragerecords add column status coverage_status; - -CREATE INDEX ix_coveragerecords_status ON coveragerecords USING btree (status); -CREATE INDEX ix_workcoveragerecords_status ON workcoveragerecords USING btree (status); - -update coveragerecords set status='success' where exception is null and status != 'success'; -update coveragerecords set status='persistent failure' where exception is not null and status != 'persistent failure'; - -update workcoveragerecords set status='success' where exception is null and status != 'success'; -update workcoveragerecords set status='persistent failure' where exception is not null and status != 'persistent failure'; diff --git a/migration/20160615-remove-data-source-primary-identifier-types.sql b/migration/20160615-remove-data-source-primary-identifier-types.sql deleted file mode 100644 index 7cdf1efd..00000000 --- a/migration/20160615-remove-data-source-primary-identifier-types.sql +++ /dev/null @@ -1,10 +0,0 @@ -update datasources set primary_identifier_type=NULL where name in ( - 'Library Simplified Open Access Content Server', - 'OCLC Classify', - 'OCLC Linked Data', - 'Amazon', - 'Library Simplified metadata wrangler', - 'Library Staff' -); -update datasources set primary_identifier_type='NoveList ID' where name='NoveList Select'; -update datasources set primary_identifier_type='ISBN' where name='Content Cafe'; diff --git a/migration/20160616-recreate-materialized-views.sql b/migration/20160616-recreate-materialized-views.sql deleted file mode 100644 index 5c35b03c..00000000 --- a/migration/20160616-recreate-materialized-views.sql +++ /dev/null @@ -1,222 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - - - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20160706-repair-duplicate-and-ambiguous-contributor-records.py b/migration/20160706-repair-duplicate-and-ambiguous-contributor-records.py deleted file mode 100644 index 8222f808..00000000 --- a/migration/20160706-repair-duplicate-and-ambiguous-contributor-records.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python -"""Fix Editions that list the same contributor as both 'Primary Author' -and 'Author', and Editions that list the same contributor in an -'Unknown' role plus some more specific role. -""" -import logging -import os -import sys -from pdb import set_trace - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) - -import time - -from core.model import Contribution, Contributor, Edition, production_session -from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import and_, or_ - - -def dedupe(edition): - print("Deduping edition %s (%s)" % (edition.id, edition.title)) - primary_author = [ - x for x in edition.contributions if x.role == Contributor.PRIMARY_AUTHOR_ROLE - ] - seen = set() - contributors_with_roles = set() - unresolved_mysteries = {} - resolved_mysteries = set() - - if primary_author: - primary_author_contribution = primary_author[0] - print(" Primary author: %s" % primary_author_contribution.contributor.name) - seen.add((primary_author_contribution.contributor, Contributor.AUTHOR_ROLE)) - contributors_with_roles.add(primary_author_contribution.contributor) - - for contribution in list(edition.contributions): - contributor = contribution.contributor - role = contribution.role - if role == Contributor.PRIMARY_AUTHOR_ROLE: - # Already handled. - continue - key = (contributor, role) - if key in seen: - print(" Removing duplicate %s %s" % (role, contributor.name)) - _db.delete(contribution) - continue - seen.add(key) - if role == "Unknown": - if contributor in contributors_with_roles: - print( - " Found unknown role for %s, but mystery already resolved." - % contributor.name - ) - _db.delete(contribution) - else: - print(" The role of %s is a mystery." % contributor.name) - unresolved_mysteries[contributor] = contribution - else: - print(" Found %s %s" % (role, contributor.name)) - contributors_with_roles.add(contributor) - if contributor in unresolved_mysteries: - print(" Deleting now-resolved mystery.") - now_resolved = unresolved_mysteries[contributor] - resolved_mysteries.add(now_resolved) - del unresolved_mysteries[contributor] - _db.delete(now_resolved) - - -_db = production_session() -contribution2 = aliased(Contribution) - - -# Find Editions where one Contributor is listed both in an 'Unknown' role -# and some other role. Also find editions where one Contributor is listed -# twice in author roles. -unknown_role_or_duplicate_author_role = or_( - and_( - Contribution.role == Contributor.UNKNOWN_ROLE, - contribution2.role != Contributor.UNKNOWN_ROLE, - ), - and_( - Contribution.role.in_(Contributor.AUTHOR_ROLES), - contribution2.role.in_(Contributor.AUTHOR_ROLES), - ), -) - -qu = ( - _db.query(Edition) - .join(Edition.contributions) - .join(contribution2, contribution2.edition_id == Edition.id) - .filter(contribution2.id != Contribution.id) - .filter(contribution2.contributor_id == Contribution.contributor_id) - .filter(unknown_role_or_duplicate_author_role) -) - -print("Fixing %s Editions." % qu.count()) -qu = qu.limit(1000) -results = True -while results: - a = time.time() - results = qu.all() - for ed in qu: - # for contribution in ed.contributions: - # print contribution.contributor, contribution.role - dedupe(ed) - _db.commit() - b = time.time() - print("Batch processed in %.2f sec" % (b - a)) diff --git a/migration/20160719-fix-incorrectly-encoded-work-descriptions.py b/migration/20160719-fix-incorrectly-encoded-work-descriptions.py deleted file mode 100644 index 14710da1..00000000 --- a/migration/20160719-fix-incorrectly-encoded-work-descriptions.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -"""Fix work descriptions that were originally UTF-8 but were incorrectly -encoded as Windows-1252. -""" -import logging -import os -import sys -from pdb import set_trace - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) - -import time - -from core.external_search import ExternalSearchIndex -from core.model import Work, production_session - -_db = production_session() -client = ExternalSearchIndex() -base = _db.query(Work).filter(Work.summary_text != None).order_by(Work.id) -results = True -offset = 0 -print("Looking at %d works." % base.count()) -while results: - fixed = 0 - qu = base.offset(offset).limit(1000) - results = qu.all() - for work in results: - possibly_bad = work.summary_text - try: - windows_1252_from_unicode = work.summary_text.encode("windows-1252") - # If we get to this point, the Unicode summary can be - # encoded as Windows-1252. - try: - final = windows_1252_from_unicode.decode("utf8") - # If we get to this point, it's UTF-8 that was incorrectly - # encoded as Windows-1252. - except UnicodeDecodeError as e: - # It was Windows-1252 all along. - final = windows_1252_from_unicode.decode("windows-1252") - except UnicodeEncodeError as e: - # This description can't be encoded as Windows-1252, an - # indication that it was originally UTF-8 and is not - # subject to this problem. - final = possibly_bad - - if possibly_bad != final: - work.summary_text = final - print("%s\n =>\n %s" % (possibly_bad.encode("utf8"), final.encode("utf8"))) - work.calculate_opds_entries() - work.update_external_index(client) - print() - fixed += 1 - pass - offset += 1000 - print("At %s, %s/%s needed fixing." % (offset, fixed, len(results))) - _db.commit() diff --git a/migration/20160721-kick-out-licensepools-without-pwid.py b/migration/20160721-kick-out-licensepools-without-pwid.py deleted file mode 100644 index c3b366cb..00000000 --- a/migration/20160721-kick-out-licensepools-without-pwid.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -"""Find LicensePools that have a work but no permanent work ID and -call calculate_work() on them. This will fix the problem, either by -calculating the appropriate permanent work ID or kicking them out of -their Works. -""" - -import logging -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..") -sys.path.append(os.path.abspath(package_dir)) - - -from core.model import Edition, LicensePool, Work, production_session - -_db = production_session() - - -def fix(_db, description, qu): - a = 0 - print("%s: %s" % (description, qu.count())) - for lp in qu: - lp.calculate_work() - a += 1 - if not a % 10: - print("Committing") - _db.commit() - - -no_presentation_edition = ( - _db.query(LicensePool) - .outerjoin(LicensePool.presentation_edition) - .filter(Edition.id == None) - .filter(LicensePool.work_id != None) -) - -no_permanent_work_id = ( - _db.query(LicensePool) - .join(LicensePool.presentation_edition) - .filter(Edition.permanent_work_id == None) - .filter(LicensePool.work_id != None) -) - -no_title = ( - _db.query(LicensePool) - .join(LicensePool.presentation_edition) - .filter(Edition.title == None) - .filter(LicensePool.work_id != None) -) - -licensepools_in_same_work_as_another_licensepool_with_different_pwid = _db.execute( - "select lp1.id from licensepools lp1 join works w on lp1.work_id=w.id join editions e1 on lp1.presentation_edition_id=e1.id join licensepools lp2 on lp2.work_id=w.id join editions e2 on e2.id=lp2.presentation_edition_id and e2.permanent_work_id != e1.permanent_work_id;" -) -ids = [ - x[0] for x in licensepools_in_same_work_as_another_licensepool_with_different_pwid -] -in_same_work = _db.query(LicensePool).filter(LicensePool.id.in_(ids)) - -fix(_db, "Pools in the same work as another pool with a different pwid", in_same_work) -_db.commit() -fix(_db, "Pools with work but no presentation edition", no_presentation_edition) -_db.commit() -fix(_db, "Pools with work but no permanent work ID", no_permanent_work_id) -_db.commit() -fix(_db, "Pools with work but no title", no_title) -_db.commit() - -qu = _db.query(Work).filter(~Work.license_pools.any()) -print("Deleting %d works with no license pools." % qu.count()) diff --git a/migration/20160728-1-add-rights-status-names.py b/migration/20160728-1-add-rights-status-names.py deleted file mode 100755 index 10ceed4d..00000000 --- a/migration/20160728-1-add-rights-status-names.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -"""Add names to rightsstatus table.""" - -import logging -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..", "..", "..") -sys.path.append(os.path.abspath(package_dir)) - - -from core.model import RightsStatus, production_session - -_db = production_session() - -for uri in list(RightsStatus.NAMES.keys()): - status = RightsStatus.lookup(_db, uri) - status.name = RightsStatus.NAMES.get(uri) - -_db.commit() diff --git a/migration/20160728-2-use-unknown-rights-instead-of-empty-string.sql b/migration/20160728-2-use-unknown-rights-instead-of-empty-string.sql deleted file mode 100644 index 49ee6231..00000000 --- a/migration/20160728-2-use-unknown-rights-instead-of-empty-string.sql +++ /dev/null @@ -1,25 +0,0 @@ -UPDATE licensepooldeliveries - SET rightsstatus_id = (SELECT id from rightsstatus WHERE uri = 'http://librarysimplified.org/terms/rights-status/in-copyright') - FROM licensepools AS lp - WHERE lp.data_source_id = (SELECT id from datasources where name = 'Overdrive') - AND lp.id = licensepooldeliveries.license_pool_id; - -UPDATE licensepooldeliveries - SET rightsstatus_id = (SELECT id from rightsstatus WHERE uri = 'http://librarysimplified.org/terms/rights-status/in-copyright') - FROM licensepools AS lp - WHERE lp.data_source_id = (SELECT id from datasources where name = '3M') - AND lp.id = licensepooldeliveries.license_pool_id; - -UPDATE licensepooldeliveries - SET rightsstatus_id = (SELECT id from rightsstatus WHERE uri = 'http://librarysimplified.org/terms/rights-status/in-copyright') - FROM licensepools AS lp - WHERE lp.data_source_id = (SELECT id from datasources where name = 'Axis 360') - AND lp.id = licensepooldeliveries.license_pool_id; - -UPDATE licensepooldeliveries - SET rightsstatus_id = (SELECT id from rightsstatus WHERE uri = 'http://librarysimplified.org/terms/rights-status/unknown') - WHERE rightsstatus_id is null; - -UPDATE licensepooldeliveries - SET rightsstatus_id = (SELECT id from rightsstatus WHERE uri = 'http://librarysimplified.org/terms/rights-status/unknown') - WHERE rightsstatus_id = (SELECT id from rightsstatus WHERE uri is null); \ No newline at end of file diff --git a/migration/20160804-circulation-event-index.sql b/migration/20160804-circulation-event-index.sql deleted file mode 100644 index e4d7efe6..00000000 --- a/migration/20160804-circulation-event-index.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX ix_circulationevents_start_desc_nullslast ON circulationevents USING btree (start DESC NULLS LAST); -CREATE INDEX ix_circulationevents_license_pool_id ON circulationevents USING btree (license_pool_id); -CREATE INDEX ix_circulationevents_type ON circulationevents USING btree (type); diff --git a/migration/20160902-change-contributors-name-to-sortname.sql b/migration/20160902-change-contributors-name-to-sortname.sql deleted file mode 100644 index b33fda4a..00000000 --- a/migration/20160902-change-contributors-name-to-sortname.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE contributors RENAME COLUMN name TO sort_name; diff --git a/migration/20160907-rename-3m-to-bibliotheca.sql b/migration/20160907-rename-3m-to-bibliotheca.sql deleted file mode 100644 index ed0a8ff6..00000000 --- a/migration/20160907-rename-3m-to-bibliotheca.sql +++ /dev/null @@ -1,2 +0,0 @@ -update datasources set name='Bibliotheca' where name='3M'; -update identifiers set type='Bibliotheca ID' where type='3M ID'; diff --git a/migration/20160913-1-offer-licenses-on-internal-processing-source.sql b/migration/20160913-1-offer-licenses-on-internal-processing-source.sql deleted file mode 100644 index b3b3ef9e..00000000 --- a/migration/20160913-1-offer-licenses-on-internal-processing-source.sql +++ /dev/null @@ -1 +0,0 @@ -UPDATE datasources SET offers_licenses = true where name = 'Library Simplified Internal Process'; diff --git a/migration/20160928-rename-bibliotheca-primary-identifier-type.sql b/migration/20160928-rename-bibliotheca-primary-identifier-type.sql deleted file mode 100644 index 58a44c44..00000000 --- a/migration/20160928-rename-bibliotheca-primary-identifier-type.sql +++ /dev/null @@ -1 +0,0 @@ -update datasources set primary_identifier_type='Bibliotheca ID' where name='Bibliotheca'; diff --git a/migration/20161018-rename-distributor-circulation-event-types.sql b/migration/20161018-rename-distributor-circulation-event-types.sql deleted file mode 100644 index 69a9bd5d..00000000 --- a/migration/20161018-rename-distributor-circulation-event-types.sql +++ /dev/null @@ -1,9 +0,0 @@ -update circulationevents set type='distributor_check_out' where type='check_out'; -update circulationevents set type='distributor_check_in' where type='check_in'; -update circulationevents set type='distributor_hold_place' where type='hold_place'; -update circulationevents set type='distributor_hold_release' where type='hold_release'; -update circulationevents set type='distributor_license_add' where type='license_add'; -update circulationevents set type='distributor_license_remove' where type='license_remove'; -update circulationevents set type='distributor_availability_notify' where type='availability_notify'; -update circulationevents set type='distributor_title_add' where type='title_add'; -update circulationevents set type='distributor_title_remove' where type='title_remove'; \ No newline at end of file diff --git a/migration/20161024-numeric-patron-fines.sql b/migration/20161024-numeric-patron-fines.sql deleted file mode 100644 index d6c49f95..00000000 --- a/migration/20161024-numeric-patron-fines.sql +++ /dev/null @@ -1,4 +0,0 @@ -alter table patrons add column fines2 numeric; -update patrons set fines2=to_number(substr(fines, 2, 100), '9999.99'); -alter table patrons drop column fines; -alter table patrons rename column fines2 to fines; diff --git a/migration/20161101-remove-exceptions-from-successful-coverage-records.sql b/migration/20161101-remove-exceptions-from-successful-coverage-records.sql deleted file mode 100644 index b0fb708e..00000000 --- a/migration/20161101-remove-exceptions-from-successful-coverage-records.sql +++ /dev/null @@ -1,2 +0,0 @@ -update coveragerecords set exception = null where status = 'success' and exception is not null; -update workcoveragerecords set exception = null where status = 'success' and exception is not null; diff --git a/migration/20161115-recreate-materialized-views.sql b/migration/20161115-recreate-materialized-views.sql deleted file mode 100644 index d59be9db..00000000 --- a/migration/20161115-recreate-materialized-views.sql +++ /dev/null @@ -1,224 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.open_access_download_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20161201-add-patron-block-reason.sql b/migration/20161201-add-patron-block-reason.sql deleted file mode 100644 index 1fbdc9a7..00000000 --- a/migration/20161201-add-patron-block-reason.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE patrons ADD COLUMN block_reason varchar(255) DEFAULT NULL; diff --git a/migration/20161215-rightsstatus-uri-is-unique.sql b/migration/20161215-rightsstatus-uri-is-unique.sql deleted file mode 100644 index 3b27d0a4..00000000 --- a/migration/20161215-rightsstatus-uri-is-unique.sql +++ /dev/null @@ -1 +0,0 @@ -alter table rightsstatus add constraint rightsstatus_uri_key UNIQUE (uri); diff --git a/migration/20170110-set-voted-quality-default-values.sql b/migration/20170110-set-voted-quality-default-values.sql deleted file mode 100644 index a36bd077..00000000 --- a/migration/20170110-set-voted-quality-default-values.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE resources ALTER COLUMN voted_quality SET DEFAULT 0.0; -ALTER TABLE resources ALTER COLUMN votes_for_quality SET DEFAULT 0; diff --git a/migration/20170111-make-customlists-foreign-identifier-unique.sql b/migration/20170111-make-customlists-foreign-identifier-unique.sql deleted file mode 100644 index af1ceb58..00000000 --- a/migration/20170111-make-customlists-foreign-identifier-unique.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX ix_customlists_name ON customlists(name); -ALTER TABLE customlists ADD CONSTRAINT customlists_foreign_identifier_data_source_id_key UNIQUE (foreign_identifier, data_source_id); -ALTER TABLE customlists ADD CONSTRAINT customlists_name_data_source_id_key UNIQUE (name, data_source_id); diff --git a/migration/20170117-add-featured-to-customlistentries.sql b/migration/20170117-add-featured-to-customlistentries.sql deleted file mode 100644 index 374ea9e3..00000000 --- a/migration/20170117-add-featured-to-customlistentries.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE customlistentries ADD COLUMN featured boolean; -UPDATE customlistentries SET featured = 'f'; -ALTER TABLE customlistentries ALTER COLUMN featured SET NOT NULL; -ALTER TABLE customlistentries ALTER COLUMN featured SET DEFAULT FALSE; diff --git a/migration/20170220-patron-sync-reading.sql b/migration/20170220-patron-sync-reading.sql deleted file mode 100644 index 5708ad5a..00000000 --- a/migration/20170220-patron-sync-reading.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE patrons ADD COLUMN synchronize_annotations boolean default null; diff --git a/migration/20170224-1-rename-collections-to-catalogs.sql b/migration/20170224-1-rename-collections-to-catalogs.sql deleted file mode 100644 index c3da507f..00000000 --- a/migration/20170224-1-rename-collections-to-catalogs.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Since these tables are not yet used for anything, it's simpler --- to drop them and recreate them than to rename the fields. -drop table collectionsidentifiers; -drop table collections cascade; diff --git a/migration/20170303-1-move-open-access-download-url-to-licensepool.sql b/migration/20170303-1-move-open-access-download-url-to-licensepool.sql deleted file mode 100644 index 9b36acf6..00000000 --- a/migration/20170303-1-move-open-access-download-url-to-licensepool.sql +++ /dev/null @@ -1,12 +0,0 @@ -alter table licensepools add open_access_download_url character varying; -update licensepools set open_access_download_url = subq.url from ( - select e.id as eid, e.open_access_download_url as url - from editions e - join licensepools lp on lp.identifier_id=e.primary_identifier_id - where e.open_access_download_url is not null -) -as subq where licensepools.presentation_edition_id=subq.eid; - --- It doesn't hurt anything to keep editions.open_access_download_url --- around for a while, in case there's been a mistake. --- alter table editions drop open_access_download_url; diff --git a/migration/20170303-2-recreate-materialize-views.sql b/migration/20170303-2-recreate-materialize-views.sql deleted file mode 100644 index 30125dd6..00000000 --- a/migration/20170303-2-recreate-materialize-views.sql +++ /dev/null @@ -1,226 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.open_access_download_url, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - - - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - editions.data_source_id, - editions.primary_identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.open_access_download_url, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on editions.primary_identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20170303-3-customlistentries-work-id.sql b/migration/20170303-3-customlistentries-work-id.sql deleted file mode 100644 index f3a8a983..00000000 --- a/migration/20170303-3-customlistentries-work-id.sql +++ /dev/null @@ -1,15 +0,0 @@ -alter table customlistentries add column work_id integer; - -alter table customlistentries - add constraint customlistentries_work_id_fkey - foreign key (work_id) - references works(id); - -update customlistentries set work_id = subquery.work_id from ( - select cle.id as entry_id, works.id as work_id - from works - join licensepools lp on lp.work_id=works.id - join customlistentries cle on cle.license_pool_id = lp.id - where lp.work_id is not null -) -as subquery where customlistentries.id=subquery.entry_id; diff --git a/migration/20170321-remove-catalogs.sql b/migration/20170321-remove-catalogs.sql deleted file mode 100644 index f8ca8878..00000000 --- a/migration/20170321-remove-catalogs.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS catalogsidentifiers CASCADE; -DROP TABLE IF EXISTS catalogs CASCADE; diff --git a/migration/20170523-add-admin-password.sql b/migration/20170523-add-admin-password.sql deleted file mode 100644 index a48e27ff..00000000 --- a/migration/20170523-add-admin-password.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE admins ADD COLUMN password_hashed varchar; -ALTER TABLE admins DROP COLUMN access_token; diff --git a/migration/20170626-no-internal-processing-licenses-by-default.sql b/migration/20170626-no-internal-processing-licenses-by-default.sql deleted file mode 100644 index b671ceaf..00000000 --- a/migration/20170626-no-internal-processing-licenses-by-default.sql +++ /dev/null @@ -1,3 +0,0 @@ -update datasources -set offers_licenses = 'f' -where name = 'Library Simplified Internal Process'; diff --git a/migration/20170627-remove-unresolved-identifiers.sql b/migration/20170627-remove-unresolved-identifiers.sql deleted file mode 100644 index 2a911c33..00000000 --- a/migration/20170627-remove-unresolved-identifiers.sql +++ /dev/null @@ -1 +0,0 @@ -drop table if exists unresolvedidentifiers; diff --git a/migration/20170713-1-timestamp-has-numeric-primary-key.sql b/migration/20170713-1-timestamp-has-numeric-primary-key.sql deleted file mode 100644 index e9ed1a0f..00000000 --- a/migration/20170713-1-timestamp-has-numeric-primary-key.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Move the old table out of the way. - -ALTER TABLE timestamps RENAME TO timestamps_old; - --- Create the new table. - -CREATE TABLE timestamps ( - id bigserial NOT NULL, - service character varying(255) NOT NULL, - collection_id integer, - "timestamp" timestamp without time zone, - counter integer -); - --- Copy preexisting timestamps from the old table to the new, giving --- each one an ascending value for the new primary key. - -insert into timestamps (service, timestamp, counter) select t.service, t.timestamp, t.counter from timestamps_old as t order by t.service; - --- Drop the old table now that the new one exists. - -drop table timestamps_old; diff --git a/migration/20170713-10-add-default-to-library.sql b/migration/20170713-10-add-default-to-library.sql deleted file mode 100644 index 3434a36b..00000000 --- a/migration/20170713-10-add-default-to-library.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Add libraries.is_default -alter table libraries add column is_default boolean default false; - -create index "ix_libraries_default" on libraries (is_default); - --- The first library in the system is set as the default. -update libraries set is_default = False; -update libraries set is_default = True where id in (select min(id) from libraries); diff --git a/migration/20170713-12-add-name-protocol-goal-to-externalintegration.sql b/migration/20170713-12-add-name-protocol-goal-to-externalintegration.sql deleted file mode 100644 index ffd772a4..00000000 --- a/migration/20170713-12-add-name-protocol-goal-to-externalintegration.sql +++ /dev/null @@ -1,8 +0,0 @@ -alter table externalintegrations add column name character varying; -create index "ix_externalintegrations_name" on externalintegrations (name); - -alter table externalintegrations add column protocol varchar; -create index "ix_externalintegrations_protocol" on externalintegrations (protocol); - -alter table externalintegrations add column goal varchar; -create index "ix_externalintegrations_goal" on externalintegrations (goal); diff --git a/migration/20170713-14-recreate-materialized-views.sql b/migration/20170713-14-recreate-materialized-views.sql deleted file mode 100644 index f0357ab4..00000000 --- a/migration/20170713-14-recreate-materialized-views.sql +++ /dev/null @@ -1,226 +0,0 @@ -drop materialized view mv_works_editions_datasources_identifiers; -create materialized view mv_works_editions_datasources_identifiers -as - SELECT - distinct works.id AS works_id, - editions.id AS editions_id, - licensepools.data_source_id, - licensepools.identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.open_access_download_url, - licensepools.availability_time, - licensepools.collection_id - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on licensepools.identifier_id = identifiers.id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY editions.sort_title, editions.sort_author, licensepools.availability_time; - --- Create a unique index so that searches can look up books by work ID. - -create unique index mv_works_editions_work_id on mv_works_editions_datasources_identifiers (works_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_editions_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_editions_by_modification on mv_works_editions_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_editions_english_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_editions_english_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_editions_english_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_editions_english_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_editions_nonenglish_adult_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_editions_nonenglish_adult_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_editions_ya_fiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_editions_ya_fiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_editions_ya_nonfiction_by_author on mv_works_editions_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_title on mv_works_editions_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_editions_ya_nonfiction_by_availability on mv_works_editions_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - - -drop materialized view mv_works_editions_workgenres_datasources_identifiers; -create materialized view mv_works_editions_workgenres_datasources_identifiers -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - licensepools.data_source_id, - licensepools.identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.open_access_download_url, - licensepools.availability_time - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on licensepools.identifier_id = identifiers.id - JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time); - --- Create a work/genre lookup. -create unique index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_genres_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_genres_by_modification on mv_works_editions_workgenres_datasources_identifiers (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_genres_english_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_genres_english_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_genres_english_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_genres_english_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_genres_nonenglish_adult_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_genres_nonenglish_adult_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_genres_ya_fiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_genres_ya_fiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_genres_ya_nonfiction_by_author on mv_works_editions_workgenres_datasources_identifiers (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_title on mv_works_editions_workgenres_datasources_identifiers (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_genres_ya_nonfiction_by_availability on mv_works_editions_workgenres_datasources_identifiers (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; diff --git a/migration/20170713-16-add-library-id-to-cachedfeed.sql b/migration/20170713-16-add-library-id-to-cachedfeed.sql deleted file mode 100644 index e8aefe82..00000000 --- a/migration/20170713-16-add-library-id-to-cachedfeed.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Remove the existing index from cachedfeeds -drop index if exists ix_cachedfeeds_lane_name_type_facets_pagination; - --- Add library_id as a foreign key -alter table cachedfeeds add column library_id integer; -alter table cachedfeeds add constraint cachedfeeds_library_id_fkey - foreign key (library_id) references libraries(id); - -create index "ix_cachedfeeds_library_id_lane_name_type_facets_pagination" - on cachedfeeds (library_id, lane_name, type, facets, pagination); - --- Set library_id to the default library for all existing cachedfeeds. -update cachedfeeds set library_id = (select id from libraries limit 1); diff --git a/migration/20170713-17-move-externalintegration-columns-to-settings.sql b/migration/20170713-17-move-externalintegration-columns-to-settings.sql deleted file mode 100644 index 23bd9ba3..00000000 --- a/migration/20170713-17-move-externalintegration-columns-to-settings.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Create a setting for each external integration url. -INSERT INTO configurationsettings (external_integration_id, key, value) -SELECT e.id, 'url', e.url -FROM externalintegrations e -WHERE e.url is not null; - --- Create a setting for each external integration username. -INSERT INTO configurationsettings (external_integration_id, key, value) -SELECT e.id, 'username', e.username -FROM externalintegrations e -WHERE e.username is not null; - --- Create a setting for each external integration password. -INSERT INTO configurationsettings (external_integration_id, key, value) -SELECT e.id, 'password', e.password -FROM externalintegrations e -WHERE e.password is not null; - -ALTER TABLE externalintegrations DROP COLUMN url; -ALTER TABLE externalintegrations DROP COLUMN username; -ALTER TABLE externalintegrations DROP COLUMN password; - --- Add the collection protocols to the external integrations. -UPDATE externalintegrations as e -SET protocol = c.protocol -FROM collections as c -WHERE e.id = c.external_integration_id; - --- Create an externalintegration for the Open Access Content Server. -INSERT INTO externalintegrations(protocol) -SELECT c.protocol -FROM collections c -WHERE c.external_integration_id is null; - --- Associate the OA Content Server collection with its integration. -UPDATE collections c -SET external_integration_id = e.id -FROM externalintegrations e -WHERE e.protocol = 'OPDS Import' and c.external_integration_id is null; - -ALTER TABLE collections DROP COLUMN IF EXISTS protocol; - -INSERT INTO configurationsettings (external_integration_id, key, value) -SELECT eis.external_integration_id, eis.key, eis.value -FROM externalintegrationsettings eis -WHERE eis.value is not null; diff --git a/migration/20170713-18-move-third-party-config-to-external-integrations.py b/migration/20170713-18-move-third-party-config-to-external-integrations.py deleted file mode 100755 index 147f5bd4..00000000 --- a/migration/20170713-18-move-third-party-config-to-external-integrations.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -"""Move integration details from the Configuration file into the -database as ExternalIntegrations -""" -import logging -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..") -sys.path.append(os.path.abspath(package_dir)) - -from config import Configuration -from external_search import ExternalSearchIndex -from model import ExternalIntegration as EI -from model import production_session -from s3 import S3Uploader - -log = logging.getLogger(name="Core configuration import") - - -def log_import(integration_or_setting): - log.info("CREATED: %r" % integration_or_setting) - - -try: - Configuration.load() - _db = production_session() - - # Import CDN configuration. - cdn_conf = Configuration.integration("CDN") - - if cdn_conf and isinstance(cdn_conf, dict): - for k, v in list(cdn_conf.items()): - cdn = EI(protocol=EI.CDN, goal=EI.CDN_GOAL) - _db.add(cdn) - cdn.url = str(v) - cdn.setting(Configuration.CDN_MIRRORED_DOMAIN_KEY).value = str(k) - log_import(cdn) - - # Import Elasticsearch configuration. - elasticsearch_conf = Configuration.integration("Elasticsearch") - if elasticsearch_conf: - url = elasticsearch_conf.get("url") - works_index = elasticsearch_conf.get(ExternalSearchIndex.WORKS_INDEX_KEY) - - integration = EI(protocol=EI.ELASTICSEARCH, goal=EI.SEARCH_GOAL) - _db.add(integration) - - if url: - integration.url = str(url) - if works_index: - integration.set_setting(ExternalSearchIndex.WORKS_INDEX_KEY, works_index) - - log_import(integration) - - # Import S3 configuration. - s3_conf = Configuration.integration("S3") - if s3_conf: - username = s3_conf.get("access_key") - password = s3_conf.get("secret_key") - del s3_conf["access_key"] - del s3_conf["secret_key"] - - integration = EI(protocol=EI.S3, goal=EI.STORAGE_GOAL) - _db.add(integration) - integration.username = username - integration.password = password - - S3_SETTINGS = [ - S3Uploader.BOOK_COVERS_BUCKET_KEY, - S3Uploader.OA_CONTENT_BUCKET_KEY, - ] - for k, v in list(s3_conf.items()): - if not k in S3_SETTINGS: - log.warn('No ExternalIntegration goal for "%s" S3 bucket' % k) - continue - integration.setting(str(k)).value = str(v) - - log_import(integration) - -finally: - _db.commit() - _db.close() diff --git a/migration/20170713-2-licensepool-identifier-and-collection-is-unique.sql b/migration/20170713-2-licensepool-identifier-and-collection-is-unique.sql deleted file mode 100644 index 2cf418fa..00000000 --- a/migration/20170713-2-licensepool-identifier-and-collection-is-unique.sql +++ /dev/null @@ -1,12 +0,0 @@ --- It used to be that LicensePool.identifier_id had to be unique. Now --- the combination of identifier_id, data_source_id, and collection_id --- must be unique. -ALTER TABLE licensepools DROP CONSTRAINT IF EXISTS licensepools_identifier_id_key; -ALTER TABLE licensepools DROP CONSTRAINT IF EXISTS licensepools_identifier_id_fkey; - -ALTER TABLE licensepools ADD COLUMN collection_id integer; -ALTER TABLE licensepools ADD CONSTRAINT licensepools_collection_id_fkey FOREIGN KEY (collection_id) REFERENCES collections(id); -ALTER TABLE licensepools ADD CONSTRAINT licensepools_identifier_id_collection_id_key UNIQUE (identifier_id, data_source_id, collection_id); - -DROP INDEX ix_licensepools_data_source_id_identifier_id; -CREATE UNIQUE INDEX ix_licensepools_collection_id_data_source_id_identifier_id on licensepools USING btree (collection_id, data_source_id, identifier_id); diff --git a/migration/20170713-20-add-goal-to-collection-external-integrations.sql b/migration/20170713-20-add-goal-to-collection-external-integrations.sql deleted file mode 100644 index 6cf5bdbd..00000000 --- a/migration/20170713-20-add-goal-to-collection-external-integrations.sql +++ /dev/null @@ -1,7 +0,0 @@ -update externalintegrations - set goal = 'licenses' - where id in ( - select ei.id from externalintegrations ei - join collections c on ei.id = c.external_integration_id - ); - diff --git a/migration/20170713-21-make-collection-external-integration-id-unique.sql b/migration/20170713-21-make-collection-external-integration-id-unique.sql deleted file mode 100644 index 7088c493..00000000 --- a/migration/20170713-21-make-collection-external-integration-id-unique.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE collections ADD CONSTRAINT collection_external_integration_id_key UNIQUE (external_integration_id); \ No newline at end of file diff --git a/migration/20170713-3-drop-hyperlink-license-pool-id.sql b/migration/20170713-3-drop-hyperlink-license-pool-id.sql deleted file mode 100644 index 5f0a9d71..00000000 --- a/migration/20170713-3-drop-hyperlink-license-pool-id.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Remove the connection between Hyperlink and LicensePool, which --- was never used and is now in the way. -DROP INDEX IF EXISTS ix_hyperlinks_license_pool_id; -ALTER TABLE hyperlinks DROP CONSTRAINT hyperlinks_license_pool_id_fkey; -ALTER TABLE hyperlinks DROP COLUMN license_pool_id; diff --git a/migration/20170713-4-modify-licensepooldeliveries.sql b/migration/20170713-4-modify-licensepooldeliveries.sql deleted file mode 100644 index 6237d1d2..00000000 --- a/migration/20170713-4-modify-licensepooldeliveries.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Create two new columns that will replace license_pool_id. -ALTER TABLE licensepooldeliveries ADD COLUMN data_source_id integer; -ALTER TABLE licensepooldeliveries ADD COLUMN identifier_id integer; - -alter table licensepooldeliveries - add constraint licensepooldeliveries_data_source_id_fkey - foreign key (data_source_id) - references datasources(id); - -alter table licensepooldeliveries - add constraint licensepooldeliveries_identifier_id_fkey - foreign key (identifier_id) - references identifiers(id); - --- Copy in appropriate values using license_pool_id. -update licensepooldeliveries set - data_source_id=subquery.data_source_id, - identifier_id=subquery.identifier_id from ( - select lpdm.id as delivery_id, - lp.identifier_id as identifier_id, - lp.data_source_id as data_source_id - from licensepooldeliveries lpdm - join licensepools lp on lpdm.license_pool_id=lp.id -) as subquery where licensepooldeliveries.id=subquery.delivery_id; - --- Now that we have the data, create a unique index. -CREATE UNIQUE INDEX ix_licensepooldeliveries_datasource_identifier_mechanism on licensepooldeliveries USING btree (data_source_id, identifier_id, delivery_mechanism_id, resource_id); - --- Finally, remove the now-unnecessary license_pool_id. -ALTER TABLE licensepooldeliveries DROP COLUMN license_pool_id; diff --git a/migration/20170713-5-add-work-id-to-cached-feeds.sql b/migration/20170713-5-add-work-id-to-cached-feeds.sql deleted file mode 100644 index 5b18efb9..00000000 --- a/migration/20170713-5-add-work-id-to-cached-feeds.sql +++ /dev/null @@ -1,7 +0,0 @@ -alter table cachedfeeds add column work_id integer; -create index ix_cachedfeeds_work_id on cachedfeeds using btree (work_id); -alter table cachedfeeds - ADD CONSTRAINT cachedfeeds_work_id_fkey - FOREIGN KEY (work_id) REFERENCES works(id); -delete from cachedfeeds where license_pool_id is not null; -alter table cachedfeeds drop column if exists license_pool_id; diff --git a/migration/20170713-6-add-library-to-patron.sql b/migration/20170713-6-add-library-to-patron.sql deleted file mode 100644 index 14d46336..00000000 --- a/migration/20170713-6-add-library-to-patron.sql +++ /dev/null @@ -1,22 +0,0 @@ --- It used to be that patrons.username, --- patrons.authorization_identifier, and patrons.external_identifier --- each had to be unique. Now, the *combination* of each of those fields --- with patrons.library_id must be unique. - -DROP INDEX IF EXISTS ix_patrons_authorization_identifier; -DROP INDEX IF EXISTS ix_patrons_external_identifier; -DROP INDEX IF EXISTS ix_patrons_username; - -ALTER TABLE patrons ADD COLUMN library_id integer; -ALTER TABLE patrons ADD CONSTRAINT patrons_library_id_fkey FOREIGN KEY (library_id) REFERENCES libraries(id); - - -ALTER TABLE patrons ADD CONSTRAINT patrons_library_id_authorization_identifier_key UNIQUE (library_id, authorization_identifier); -ALTER TABLE patrons ADD CONSTRAINT patrons_library_id_external_identifier_key UNIQUE (library_id, external_identifier); -ALTER TABLE patrons ADD CONSTRAINT patrons_library_id_username_key UNIQUE (library_id, username); - -CREATE INDEX ix_patron_library_id_authorization_identifier ON patrons USING btree (library_id, authorization_identifier); -CREATE INDEX ix_patron_library_id_external_identifier ON patrons USING btree (library_id, external_identifier); -CREATE INDEX ix_patron_library_id_username ON patrons USING btree (library_id, username); - -UPDATE patrons set library_id = (select id from libraries limit 1); diff --git a/migration/20170713-7-delete-admin-auth-services.sql b/migration/20170713-7-delete-admin-auth-services.sql deleted file mode 100644 index 7d1a2b2f..00000000 --- a/migration/20170713-7-delete-admin-auth-services.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS adminauthenticationservices; diff --git a/migration/20170713-8-add-collection-id-to-coveragerecords.sql b/migration/20170713-8-add-collection-id-to-coveragerecords.sql deleted file mode 100644 index 08607001..00000000 --- a/migration/20170713-8-add-collection-id-to-coveragerecords.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Remove the existing indices from coveragerecords -drop index if exists ix_coveragerecords_data_source_id_operation_identifier_id; -alter table coveragerecords drop constraint if exists coveragerecords_identifier_id_data_source_id_operation_key; - --- Add collection_id as a foreign key -alter table coveragerecords add column collection_id integer; -alter table coveragerecords add constraint coveragerecords_collection_id_fkey - foreign key (collection_id) references collections(id); - --- Create unique indices for coveragerecords with or without a collection_id -create unique index ix_identifier_id_data_source_id_operation - on coveragerecords (identifier_id, data_source_id, operation) - where collection_id is null; - -create unique index ix_identifier_id_data_source_id_operation_collection_id - on coveragerecords (identifier_id, data_source_id, operation, collection_id); diff --git a/migration/20170803-drop-externalintegrationsettings.sql b/migration/20170803-drop-externalintegrationsettings.sql deleted file mode 100644 index 03646f2a..00000000 --- a/migration/20170803-drop-externalintegrationsettings.sql +++ /dev/null @@ -1 +0,0 @@ -drop table if exists externalintegrationsettings; diff --git a/migration/20170809-fix-bibliotheca-import.sql b/migration/20170809-fix-bibliotheca-import.sql deleted file mode 100644 index e095d7f9..00000000 --- a/migration/20170809-fix-bibliotheca-import.sql +++ /dev/null @@ -1,11 +0,0 @@ -update identifiers - set type = 'Bibliotheca ID' - where type in ('3M ID', 'Bibliotecha ID'); - -update editions - set data_source_id = (select id from datasources where name = 'Bibliotheca') - where data_source_id in (select id from datasources where name in ('3M', 'Bibliotecha')); - -update coveragerecords - set data_source_id = (select id from datasources where name = 'Bibliotheca') - where data_source_id in (select id from datasources where name in ('3M', 'Bibliotecha')); diff --git a/migration/20170825-fix-acsm-media-type.sql b/migration/20170825-fix-acsm-media-type.sql deleted file mode 100644 index bb29dff1..00000000 --- a/migration/20170825-fix-acsm-media-type.sql +++ /dev/null @@ -1 +0,0 @@ -update deliverymechanisms set drm_scheme='application/vnd.adobe.adept+xml' where drm_scheme='vnd.adobe/adept+xml'; diff --git a/migration/20170831-add-data-source-integration-client-id.sql b/migration/20170831-add-data-source-integration-client-id.sql deleted file mode 100644 index 3e72c73a..00000000 --- a/migration/20170831-add-data-source-integration-client-id.sql +++ /dev/null @@ -1,6 +0,0 @@ -alter table datasources add column integration_client_id integer unique; -alter table datasources add constraint datasources_integration_client_id_fkey - foreign key (integration_client_id) references integrationclients(id); - -create index "ix_datasources_integration_client" - on datasources (integration_client_id); \ No newline at end of file diff --git a/migration/20170905-update-integrationclients-for-shared-secret.sql b/migration/20170905-update-integrationclients-for-shared-secret.sql deleted file mode 100644 index 15c54e66..00000000 --- a/migration/20170905-update-integrationclients-for-shared-secret.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Reset the integrationclients table. -alter table integrationclients drop column _secret; -alter table integrationclients rename column key to shared_secret; -drop index if exists ix_integrationclients_key; -delete from integrationclients; diff --git a/migration/20170907-rename-one-click.sql b/migration/20170907-rename-one-click.sql deleted file mode 100644 index f1596c80..00000000 --- a/migration/20170907-rename-one-click.sql +++ /dev/null @@ -1,5 +0,0 @@ -update datasources set name='RBdigital' where name='OneClick'; -update externalintegrations set protocol='RBdigital' where protocol='OneClick'; -update configurationsettings set value='https://api.rbdigital.com/' where key='url' and value='https://api.oneclickdigital.com/'; -update configurationsettings set value='http://api.rbdigitalstage.com/' where key='url' and value='https://api.oneclickdigital.us/'; -update identifiers set type='RBdigital ID' where type='OneClick ID'; diff --git a/migration/20170908-change-metadata-wrangler-settings.py b/migration/20170908-change-metadata-wrangler-settings.py deleted file mode 100755 index 3e97f5b8..00000000 --- a/migration/20170908-change-metadata-wrangler-settings.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -"""Delete outdated ConfigurationSettings for the metadata wrangler.""" -import logging -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..") -sys.path.append(os.path.abspath(package_dir)) - -from model import ExternalIntegration as EI -from model import production_session - -_db = production_session() -try: - integration = EI.lookup(_db, EI.METADATA_WRANGLER, EI.METADATA_GOAL) - - if integration: - for setting in integration.settings: - if setting.key == "username": - # A username (or client_id) is no longer required. - _db.delete(setting) - if setting.key == "password": - # The password (previously client_secret) must be reset to - # register for a shared_secret. - setting.value = None - _db.commit() - _db.close() -except Exception: - _db.close() - raise diff --git a/migration/20170913-bibliotheca-books-delivered-through-findaway.sql b/migration/20170913-bibliotheca-books-delivered-through-findaway.sql deleted file mode 100644 index 20d2bc43..00000000 --- a/migration/20170913-bibliotheca-books-delivered-through-findaway.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Create a new delivery mechanism for audiobooks delivered --- through Findaway -insert into deliverymechanisms (content_type, drm_scheme) values ( - 'audio/mpeg', - 'application/vnd.librarysimplified.findaway.license+json' -); - --- Update all audiobooks delivered through Bibliotheca to use --- the new delivery mechanism. -update licensepooldeliveries set delivery_mechanism_id = ( - select id from deliverymechanisms where - content_type='audio/mpeg' and - drm_scheme='application/vnd.librarysimplified.findaway.license+json' -) - -where id in ( - select lpd.id from licensepooldeliveries lpd - join deliverymechanisms dm on lpd.delivery_mechanism_id=dm.id - join datasources ds on lpd.data_source_id=ds.id - where ds.name='Bibliotheca' - and dm.content_type='audio/mpeg' -); diff --git a/migration/20170926-migrate-log-configuration.py b/migration/20170926-migrate-log-configuration.py deleted file mode 100755 index dc3d496e..00000000 --- a/migration/20170926-migrate-log-configuration.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -"""Move log details from the Configuration file into the -database as ExternalIntegrations -""" - -import logging -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..") -sys.path.append(os.path.abspath(package_dir)) - -from config import Configuration -from model import ExternalIntegration as EI -from model import production_session - -_db = production_session() -log = logging.getLogger(name="Log configuration import") -loggly_conf = Configuration.integration("loggly") - -if loggly_conf: - integration = EI(goal=EI.LOGGING_GOAL, protocol=EI.LOGGLY) - _db.add(integration) - integration.url = loggly_conf.get( - "url", "https://logs-01.loggly.com/inputs/%(token)s/tag/python/" - ) - integration.password = loggly_conf.get("token") -_db.commit() diff --git a/migration/20170928-add-registered-coverage-status.sql b/migration/20170928-add-registered-coverage-status.sql deleted file mode 100644 index 80a217fc..00000000 --- a/migration/20170928-add-registered-coverage-status.sql +++ /dev/null @@ -1,9 +0,0 @@ -ALTER TYPE coverage_status ADD VALUE IF NOT EXISTS 'registered' AFTER 'persistent failure'; - -update coveragerecords - set - status = 'registered', - exception = null - where - status = 'transient failure' and exception like 'No work done yet%'; - diff --git a/migration/20171004-add-customlists-library-id.sql b/migration/20171004-add-customlists-library-id.sql deleted file mode 100644 index ae8de984..00000000 --- a/migration/20171004-add-customlists-library-id.sql +++ /dev/null @@ -1,21 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE customlists ADD COLUMN library_id integer; - alter table customlists - add constraint customlists_library_id_fkey - foreign key (library_id) - references libraries(id); - - create index "ix_customlists_library_id" ON customlists (library_id); - - alter table customlists drop constraint if exists "customlists_data_source_id_name_key"; - alter table customlists add constraint "customlists_data_source_id_name_library_id_key" unique (data_source_id, name, library_id); - - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'WARNING: column library_id already exists in customlists. Verify that 20171004-add-customlists-library-id.sql is redundant.'; - END; - END; -$$; - - diff --git a/migration/20171020-add-python-timestamp.sql b/migration/20171020-add-python-timestamp.sql deleted file mode 100644 index 4b2f724e..00000000 --- a/migration/20171020-add-python-timestamp.sql +++ /dev/null @@ -1,3 +0,0 @@ -insert into timestamps(service, timestamp) - values('Database Migration - Python', date '2017-10-20' + time '00:00'); - diff --git a/migration/20171022-add-loan-and-hold-external-identifier.sql b/migration/20171022-add-loan-and-hold-external-identifier.sql deleted file mode 100644 index 2764e4d9..00000000 --- a/migration/20171022-add-loan-and-hold-external-identifier.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE loans ADD COLUMN external_identifier varchar UNIQUE; diff --git a/migration/20171026-update-rbdigital-delivery-mechanism.sql b/migration/20171026-update-rbdigital-delivery-mechanism.sql deleted file mode 100644 index 5794093f..00000000 --- a/migration/20171026-update-rbdigital-delivery-mechanism.sql +++ /dev/null @@ -1 +0,0 @@ -update deliverymechanisms set content_type='application/audiobook+json', drm_scheme=null where content_type='vnd.librarysimplified/obfuscated-one-click' and drm_scheme='OneClick DRM'; diff --git a/migration/20171103-migrate-axis-and-bibliotheca-classifications-to-bisac.sql b/migration/20171103-migrate-axis-and-bibliotheca-classifications-to-bisac.sql deleted file mode 100644 index 252c321a..00000000 --- a/migration/20171103-migrate-axis-and-bibliotheca-classifications-to-bisac.sql +++ /dev/null @@ -1,16 +0,0 @@ --- "3M" subjects can now be processed as BISAC. -update subjects set type='BISAC' where type='3M'; - --- Up to this point we've been storing BISAC names in the 'identifier' --- slot. But BISAC subjects actually have identifiers. Bibliotheca --- and Axis don't mention them, but other vendors do. Move all BISAC --- identifiers into the 'name' slot. -update subjects set name=identifier where type='BISAC' and name is null; -update subjects set identifier=null where name=identifier and name not like '%0'; - --- All existing 'BISAC' subjects should be rechecked with the new rules. -update subjects set checked=false where type='BISAC'; - --- Old code incorrectly classified 'Fiction / Urban' as 'Urban Fiction'. --- All such subjects need to be reevaluated. -update subjects set checked=false where name ilike 'fiction%urban'; diff --git a/migration/20171107-rename-rbdigital-subjects.sql b/migration/20171107-rename-rbdigital-subjects.sql deleted file mode 100644 index 19160e21..00000000 --- a/migration/20171107-rename-rbdigital-subjects.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Rename OneClick subject types to RBdigital types. -update subjects set type='RBdigital' where type='OneClick'; -update subjects set type='RBdigital Audience' where type='OneClick Audience'; - --- Re-weight RBdigital classifications based on observed reliability. -update classifications set weight=500 where subject_id in (select id from subjects where type='RBdigital Audience'); -update classifications set weight=200 where subject_id in (select id from subjects where type='RBdigital'); - --- Mark subjects as unchecked where classification rules have been improved --- so that bin/work_classify_unchecked_subjects can reclassify --- the books that use these subjects. -update subjects set checked=false where type like 'RBdigital'; - -update subjects set checked=false where identifier ilike 'true%crime'; -update subjects set checked=false where identifier ilike 'sci%fi'; -update subjects set checked=false where identifier ilike 'language arts%'; -update subjects set checked=false where identifier ilike 'inspirational nonfiction'; -update subjects set checked=false where identifier ilike 'women%s fiction'; -update subjects set checked=false where identifier ilike 'beginning reader'; diff --git a/migration/20171108-overdrive-ils-name-per-library.sql b/migration/20171108-overdrive-ils-name-per-library.sql deleted file mode 100644 index 8d488e54..00000000 --- a/migration/20171108-overdrive-ils-name-per-library.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Find every Overdrive integration that defines an 'ils_name' key, --- and copy that key to every library that uses the corresponding --- Overdrive collection. -insert into configurationsettings (external_integration_id, library_id, key, value) - select distinct ei.id, cl.library_id, key, value - from configurationsettings cs - join externalintegrations ei on cs.external_integration_id=ei.id - join collections c on c.external_integration_id=ei.id - join collections_libraries cl on c.id=cl.collection_id - where ei.protocol='Overdrive' - and cs.key='ils_name' - and not exists (select * from configurationsettings where library_id = cl.library_id and external_integration_id = ei.id and key = 'ils_name'); - --- Delete all 'ils_name' configuration settings associated with an --- Overdrive integration but not affiliated with any library. -delete from configurationsettings where library_id is null and key='ils_name' and external_integration_id in (select id from externalintegrations where protocol='Overdrive'); diff --git a/migration/20171127-revisit-overdrive-subjects.sql b/migration/20171127-revisit-overdrive-subjects.sql deleted file mode 100644 index 54d5fec6..00000000 --- a/migration/20171127-revisit-overdrive-subjects.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Mark subjects as unchecked where Overdrive classification rules --- have been improved so that bin/work_classify_unchecked_subjects can --- reclassify the books that use these subjects. - -update subjects set checked=false where type='Overdrive' and identifier in ( -'Antiquarian', -'Biology', -'Child Development', -'Drama', -'Economics', -'Gay/Lesbian', -'Genealogy', -'Latin', -'Literary Anthologies', -'Media Studies', -'Mythology', -'Outdoor Recreation', -'Poetry', -'Recovery', -'Social Media', -'Songbook', -'Text Book', -'Western', -'Writing' -); diff --git a/migration/20171129-change-cachedfeeds-to-use-lane-id.sql b/migration/20171129-change-cachedfeeds-to-use-lane-id.sql deleted file mode 100644 index a7a3c37e..00000000 --- a/migration/20171129-change-cachedfeeds-to-use-lane-id.sql +++ /dev/null @@ -1 +0,0 @@ -drop table cachedfeeds; \ No newline at end of file diff --git a/migration/20171201-rebuild-search-index.py b/migration/20171201-rebuild-search-index.py deleted file mode 100755 index 4b984c04..00000000 --- a/migration/20171201-rebuild-search-index.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -"""Running the search index updater script will create the new -circulation-works-v3 index and change the circulation-works-current -alias to point to it. -""" -import os -import sys - -bin_dir = os.path.split(__file__)[0] -package_dir = os.path.join(bin_dir, "..") -sys.path.append(os.path.abspath(package_dir)) -from scripts import UpdateSearchIndexScript - -UpdateSearchIndexScript().run() diff --git a/migration/20171204-set-library-id-for-lists.sql b/migration/20171204-set-library-id-for-lists.sql deleted file mode 100644 index b15e48e9..00000000 --- a/migration/20171204-set-library-id-for-lists.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Associate all 'Staff Picks' customlists with the default library, --- unless they're already associated with some other library. -update customlists set library_id = ( - select id from libraries where is_default=true limit 1 -) where library_id is null and foreign_identifier='Staff Picks'; diff --git a/migration/20171207-odilo-add-image-video-medium.sql b/migration/20171207-odilo-add-image-video-medium.sql deleted file mode 100644 index fcb4e736..00000000 --- a/migration/20171207-odilo-add-image-video-medium.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TYPE medium ADD VALUE IF NOT EXISTS 'Image' AFTER 'Video'; -ALTER TYPE medium ADD VALUE IF NOT EXISTS 'Courseware' AFTER 'Image'; \ No newline at end of file diff --git a/migration/20171208-add-lane-size.sql b/migration/20171208-add-lane-size.sql deleted file mode 100644 index 987a48ff..00000000 --- a/migration/20171208-add-lane-size.sql +++ /dev/null @@ -1,9 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE lanes ADD COLUMN size integer not null default 0; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column lanes.size already exists, not creating it.'; - END; - END $$; - diff --git a/migration/20180105-1-lanes-inherit-parent-restrictions.sql b/migration/20180105-1-lanes-inherit-parent-restrictions.sql deleted file mode 100644 index 508cedb1..00000000 --- a/migration/20180105-1-lanes-inherit-parent-restrictions.sql +++ /dev/null @@ -1,6 +0,0 @@ --- The default for lanes.inherit_parent_restrictions has changed --- to true. Change the setting for all lanes not based on custom lists. --- For all lanes based on custom lists, a human has made a decision --- to set the value one way or the other. -update lanes set inherit_parent_restrictions=true where _list_datasource_id is null and id not in (select lane_id from lanes_customlists); -ALTER TABLE lanes ALTER COLUMN inherit_parent_restrictions SET DEFAULT true; diff --git a/migration/20180109-add-workgenres-index.sql b/migration/20180109-add-workgenres-index.sql deleted file mode 100644 index 265d2b06..00000000 --- a/migration/20180109-add-workgenres-index.sql +++ /dev/null @@ -1,3 +0,0 @@ -drop index if exists mv_works_genres_work_id_genre_id; -create index mv_works_genres_work_id_genre_id on mv_works_editions_workgenres_datasources_identifiers (works_id, genre_id); - diff --git a/migration/20180110-lanes-limited-to-ebooks.sql b/migration/20180110-lanes-limited-to-ebooks.sql deleted file mode 100644 index 106aad20..00000000 --- a/migration/20180110-lanes-limited-to-ebooks.sql +++ /dev/null @@ -1,2 +0,0 @@ --- All lanes created up to this point should only display ebooks. -update lanes set media='{"Book"}'; diff --git a/migration/20180112-redo-opds-for-non-author-contributors.sql b/migration/20180112-redo-opds-for-non-author-contributors.sql deleted file mode 100644 index da4267ae..00000000 --- a/migration/20180112-redo-opds-for-non-author-contributors.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Remove the 'generate-opds` coverage record for any works --- that have an associated series. -delete from workcoveragerecords where operation='generate-opds' and work_id in ( - select w.id from works w join editions e on w.presentation_edition_id=e.id where e.series is not null -); - --- Remove the 'generate-opds` coverage record for any works --- that have contributors in non-author roles. -delete from workcoveragerecords where operation='generate-opds' and work_id in ( - select distinct w.id from works w join editions e on w.presentation_edition_id=e.id join contributions c on e.id=c.edition_id where c.role not in ('Author', 'Primary Author') -); - --- These records will be added back when the OPDSEntryWorkCoverageProvider --- regenerates their OPDS entries. diff --git a/migration/20180117-regenerate-opds-for-dc-issued.sql b/migration/20180117-regenerate-opds-for-dc-issued.sql deleted file mode 100644 index 5830f9f5..00000000 --- a/migration/20180117-regenerate-opds-for-dc-issued.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Remove the 'generate-opds` coverage record for all works. -delete from workcoveragerecords where operation='generate-opds'; diff --git a/migration/20180119-add-unique-index-to-materialized-view.sql b/migration/20180119-add-unique-index-to-materialized-view.sql deleted file mode 100644 index 4eed0af6..00000000 --- a/migration/20180119-add-unique-index-to-materialized-view.sql +++ /dev/null @@ -1,3 +0,0 @@ -drop index if exists mv_works_for_lanes_work_id_genre_id; -drop index if exists mv_works_for_lanes_unique; -create unique index mv_works_for_lanes_unique on mv_works_for_lanes (works_id, genre_id, license_pool_id); diff --git a/migration/20180129-collection-mirror-integration.sql b/migration/20180129-collection-mirror-integration.sql deleted file mode 100644 index c82e3169..00000000 --- a/migration/20180129-collection-mirror-integration.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Add collections.mirror_integration_id, a foreign key --- against externalintegrations.id -alter table collections add column mirror_integration_id integer; -alter table collections add constraint collections_mirror_integration_id_fkey FOREIGN KEY (mirror_integration_id) REFERENCES externalintegrations(id); diff --git a/migration/20180202-genre-name-is-unique.sql b/migration/20180202-genre-name-is-unique.sql deleted file mode 100644 index 790fab84..00000000 --- a/migration/20180202-genre-name-is-unique.sql +++ /dev/null @@ -1,10 +0,0 @@ --- genres.name should be indexed and unique. -DO $$ - BEGIN - BEGIN -create unique index ix_genres_name on genres (name); - EXCEPTION - WHEN OTHERS THEN RAISE NOTICE 'WARNING: it looks like ix_genres_name already exists; it was probably created on initial database creation.'; - END; - END; -$$; diff --git a/migration/20180207-1-change-custom-list-name-constraint.sql b/migration/20180207-1-change-custom-list-name-constraint.sql deleted file mode 100644 index 25b3831b..00000000 --- a/migration/20180207-1-change-custom-list-name-constraint.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table customlists drop constraint if exists "customlists_data_source_id_name_library_id_key"; -alter table customlists add constraint "customlists_name_library_id_key" unique (name, library_id); \ No newline at end of file diff --git a/migration/20180212-recreate-materialized-view.sql b/migration/20180212-recreate-materialized-view.sql deleted file mode 100644 index 81ef1045..00000000 --- a/migration/20180212-recreate-materialized-view.sql +++ /dev/null @@ -1,140 +0,0 @@ -drop materialized view mv_works_for_lanes; - --- Create the materialized view with no data. -create materialized view mv_works_for_lanes -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - licensepools.data_source_id, - licensepools.identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - licensepools.id AS license_pool_id, - licensepools.open_access_download_url, - licensepools.availability_time, - licensepools.collection_id, - customlistentries.list_id, - customlistentries.edition_id as list_edition_id, - customlistentries.first_appearance - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on licensepools.identifier_id = identifiers.id - LEFT JOIN customlistentries on works.id = customlistentries.work_id - LEFT JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time) - WITH NO DATA; - --- First create an index that allows work/genre lookup. It's unique and incorporates license_pool_id so that the materialized view can be refreshed CONCURRENTLY. --- NOTE: All fields mentioned here also need to be part of the primary key --- for the model object defined in model.py. -create unique index mv_works_for_lanes_unique on mv_works_for_lanes (works_id, genre_id, list_id, list_edition_id, license_pool_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. - -create index mv_works_for_lanes_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id); - --- Create an index on everything, sorted by the maximum of entry first appearance, license pool availability time, and work update time, so that crawlable feeds are fast. - --- MIGRATION NOTE: We don't create mv_works_for_lanes_by_recently_updated here --- because it will be created in 20180307-replace-recently-updated-index. - --- Create indexes that are helpful in running the query to find featured works. - -create index mv_works_for_lanes_by_random_and_genre on mv_works_for_lanes (random, language, genre_id); - -create index mv_works_for_lanes_by_random_audience_target_age on mv_works_for_lanes (random, language, audience, target_age); - -create index mv_works_for_lanes_by_random_fiction_audience_target_age on mv_works_for_lanes (random, language, fiction, audience, target_age); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_for_lanes_by_modification on mv_works_for_lanes (last_update_time DESC, sort_author, sort_title, works_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_for_lanes_english_adult_fiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_fiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_fiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_for_lanes_english_adult_nonfiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_nonfiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_nonfiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_for_lanes_nonenglish_adult_fiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_fiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_fiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_for_lanes_nonenglish_adult_nonfiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_nonfiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_nonfiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_for_lanes_ya_fiction_by_author on mv_works_for_lanes (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_for_lanes_ya_fiction_by_title on mv_works_for_lanes (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_for_lanes_ya_fiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_for_lanes_ya_nonfiction_by_author on mv_works_for_lanes (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_for_lanes_ya_nonfiction_by_title on mv_works_for_lanes (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_for_lanes_ya_nonfiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - --- Refresh the new materialized view. -refresh materialized view mv_works_for_lanes; - diff --git a/migration/20180228-improve-materialized-view-performance.sql b/migration/20180228-improve-materialized-view-performance.sql deleted file mode 100644 index ddd05bc5..00000000 --- a/migration/20180228-improve-materialized-view-performance.sql +++ /dev/null @@ -1,53 +0,0 @@ -DO $$ - BEGIN - -- Create indices for foreign keys on materialized views. - BEGIN - create index ix_mv_works_for_lanes_works_id on - mv_works_for_lanes (works_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_mv_works_for_lanes_works_id already exists.'; - END; - - BEGIN - create index ix_mv_works_for_lanes_license_pool_id on - mv_works_for_lanes (license_pool_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_mv_works_for_lanes_license_pool_id already exists.'; - END; - - BEGIN - create index ix_mv_works_for_lanes_workgenres_id on - mv_works_for_lanes (workgenres_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_mv_works_for_lanes_workgenres_id already exists.'; - END; - - BEGIN - create index ix_mv_works_for_lanes_list_id on - mv_works_for_lanes (list_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_mv_works_for_lanes_list_id already exists.'; - END; - - BEGIN - create index ix_mv_works_for_lanes_list_edition_id on - mv_works_for_lanes (list_edition_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_mv_works_for_lanes_list_edition_id already exists.'; - END; - - BEGIN - create index ix_licensepools_collection_id on - licensepools (collection_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_licensepools_collection_id already exists.'; - END; - - BEGIN - create index ix_licensepools_licenses_owned on - licensepools (licenses_owned); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_licensepools_licenses_owned already exists.'; - END; - END; -$$; diff --git a/migration/20180307-replace-recently-updated-index.sql b/migration/20180307-replace-recently-updated-index.sql deleted file mode 100644 index b003e57c..00000000 --- a/migration/20180307-replace-recently-updated-index.sql +++ /dev/null @@ -1,11 +0,0 @@ -DO $$ - BEGIN - -- Delete the index if it already exists. - BEGIN - drop index mv_works_for_lanes_by_recently_updated; - EXCEPTION - WHEN OTHERS THEN RAISE NOTICE 'mv_works_for_lanes_by_recently_updated did not previously exist.'; - END; - create index mv_works_for_lanes_by_recently_updated on mv_works_for_lanes (GREATEST(availability_time, first_appearance, last_update_time) DESC, collection_id, works_id); - END -$$; diff --git a/migration/20180313-1-add-identifier-id-to-materialized-view.sql b/migration/20180313-1-add-identifier-id-to-materialized-view.sql deleted file mode 100644 index 9bccc2cb..00000000 --- a/migration/20180313-1-add-identifier-id-to-materialized-view.sql +++ /dev/null @@ -1,9 +0,0 @@ -DO $$ - BEGIN - BEGIN - create index mv_works_for_lanes_identifier_id on mv_works_for_lanes (identifier_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: mv_works_for_lanes_identifier_id already exists.'; - END; - END -$$; diff --git a/migration/20180313-2-rename-s3-integration.sql b/migration/20180313-2-rename-s3-integration.sql deleted file mode 100644 index 9097857e..00000000 --- a/migration/20180313-2-rename-s3-integration.sql +++ /dev/null @@ -1 +0,0 @@ -update externalintegrations set protocol='Amazon S3' where protocol='S3'; diff --git a/migration/20180315-1-add-loan-hold-integration-client.sql b/migration/20180315-1-add-loan-hold-integration-client.sql deleted file mode 100644 index cc906156..00000000 --- a/migration/20180315-1-add-loan-hold-integration-client.sql +++ /dev/null @@ -1,33 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE loans ADD COLUMN integration_client_id integer; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column loans.integration_client_id already exists, not creating it.'; - END; - - BEGIN - ALTER TABLE loans ADD CONSTRAINT loans_integration_client_id_fkey foreign key (integration_client_id) references integrationclients(id); - EXCEPTION - WHEN duplicate_object THEN RAISE NOTICE 'constraint loans_integration_client_id_fkey already exists, not creating it.'; - END; - - BEGIN - ALTER TABLE holds ADD COLUMN integration_client_id integer; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column holds.integration_client_id already exists, not creating it.'; - END; - - BEGIN - ALTER TABLE holds ADD CONSTRAINT holds_integration_client_id_fkey foreign key (integration_client_id) references integrationclients(id); - EXCEPTION - WHEN duplicate_object THEN RAISE NOTICE 'constraint holds_integration_client_id_fkey already exists, not creating it.'; - END; - - BEGIN - ALTER TABLE holds ADD COLUMN external_identifier varchar UNIQUE; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column holds.external_identifier already exists, not creating it.'; - END; - END -$$; diff --git a/migration/20180315-2-add-performance-indexes.sql b/migration/20180315-2-add-performance-indexes.sql deleted file mode 100644 index 985d4a83..00000000 --- a/migration/20180315-2-add-performance-indexes.sql +++ /dev/null @@ -1,22 +0,0 @@ -DO $$ - BEGIN - BEGIN - drop index mv_works_for_lanes_list_and_collection_id; - EXCEPTION - WHEN OTHERS THEN RAISE NOTICE 'mv_works_for_lanes_list_and_collection_id is already gone.'; - END; - - BEGIN - create index mv_works_for_lanes_list_id_collection_id_language_medium on mv_works_for_lanes (list_id, collection_id, language, medium); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: mv_works_for_lanes_list_id_collection_id_language_medium already exists.'; - END; - - BEGIN - create index customlistentries_work_id_list_id on customlistentries (work_id, list_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: customlistentries_work_id_list_id already exists.'; - END; - END -$$; - diff --git a/migration/20180321-add-timestamp-indexes.sql b/migration/20180321-add-timestamp-indexes.sql deleted file mode 100644 index 8d72e73e..00000000 --- a/migration/20180321-add-timestamp-indexes.sql +++ /dev/null @@ -1,21 +0,0 @@ - DO $$ - BEGIN - BEGIN - create index ix_loans_start on loans (start); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: is_loans_start already exists.'; - END; - - BEGIN - create index ix_loans_end on loans ("end"); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_loans_end already exists.'; - END; - - BEGIN - create index ix_cachedfeeds_timestamp on cachedfeeds (timestamp); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_cachedfeeds_timestamp already exists.'; - END; - END; -$$; diff --git a/migration/20180419-remove-all-search-coverage.sql b/migration/20180419-remove-all-search-coverage.sql deleted file mode 100644 index b2216917..00000000 --- a/migration/20180419-remove-all-search-coverage.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Remove all WorkCoverageRecords pertaining to the search index. This --- will force a complete reindex on the next run of bin/search_index_refresh. -delete from workcoveragerecords where operation='update-search-index'; diff --git a/migration/20180424-add-admin-roles.sql b/migration/20180424-add-admin-roles.sql deleted file mode 100644 index 0dd8bc9d..00000000 --- a/migration/20180424-add-admin-roles.sql +++ /dev/null @@ -1,24 +0,0 @@ -DO $$ - BEGIN - BEGIN - CREATE TABLE adminroles ( - id SERIAL PRIMARY KEY, - admin_id INTEGER NOT NULL REFERENCES admins(id), - library_id INTEGER REFERENCES libraries(id), - role VARCHAR NOT NULL - ); - ALTER TABLE adminroles ADD CONSTRAINT adminroles_admin_id_library_id_role UNIQUE (admin_id, library_id, role); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: adminroles already exists.'; - END; - - BEGIN - CREATE UNIQUE INDEX ix_adminroles_admin_id_library_id_role - ON adminroles (admin_id, library_id, role); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: ix_adminroles_admin_id_library_id_role already exists.'; - END; - - INSERT INTO adminroles (admin_id, role) (SELECT id, 'system' from admins); - END; -$$; \ No newline at end of file diff --git a/migration/20180427-bearer-token-supported-by-default.sql b/migration/20180427-bearer-token-supported-by-default.sql deleted file mode 100644 index fd25dd99..00000000 --- a/migration/20180427-bearer-token-supported-by-default.sql +++ /dev/null @@ -1 +0,0 @@ -update deliverymechanisms set default_client_can_fulfill=true where content_type='application/epub+zip' and drm_scheme='application/vnd.librarysimplified.bearer-token+json'; diff --git a/migration/20180508-drop-recursive-equivalents.sql b/migration/20180508-drop-recursive-equivalents.sql deleted file mode 100644 index 5086ad7e..00000000 --- a/migration/20180508-drop-recursive-equivalents.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This function needs to be recreated with different arguments. Dropping --- it will ensure it's recreated on the next database connection. -DROP FUNCTION IF EXISTS fn_recursive_equivalents(int, int, double precision); diff --git a/migration/20180517-add-lane-include-self-in-grouped-feeds.sql b/migration/20180517-add-lane-include-self-in-grouped-feeds.sql deleted file mode 100644 index a4ab474b..00000000 --- a/migration/20180517-add-lane-include-self-in-grouped-feeds.sql +++ /dev/null @@ -1,9 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE lanes ADD COLUMN include_self_in_grouped_feed boolean default true not null; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column lanes.include_self_in_grouped_feed already exists, not creating it.'; - END; - END; -$$; diff --git a/migration/20180521-not-really-mirrored.sql b/migration/20180521-not-really-mirrored.sql deleted file mode 100644 index 14830580..00000000 --- a/migration/20180521-not-really-mirrored.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Some representations have mirror_url set even though they were --- never mirrored. Their mirror_urls should be blanked out. - --- This shouldn't have happened, but just in case, so we don't lose information. -update representations set url=mirror_url where url is null and mirror_url is not null; - -update representations set mirror_url=null where url is not null and mirrored_at is null and mirror_url is not null; - --- A representation is never 'mirrored' to its original URL. --- More likely Representation.set_as_mirrored() was called, but we now --- prefer to leave mirror_url alone. -update representations set mirror_url=null, mirrored_at=null where url=mirror_url; diff --git a/migration/20180528-add-resource-rights-status-and-derivatives.sql b/migration/20180528-add-resource-rights-status-and-derivatives.sql deleted file mode 100644 index cdeed133..00000000 --- a/migration/20180528-add-resource-rights-status-and-derivatives.sql +++ /dev/null @@ -1,16 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE resources ADD COLUMN rights_status_id integer; - ALTER TABLE resources ADD CONSTRAINT resources_rightsstatus_id_fkey FOREIGN KEY (rights_status_id) REFERENCES rightsstatus(id); - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column resources.rights_status_id already exists, not creating it.'; - END; - - BEGIN - ALTER TABLE resources ADD COLUMN rights_explanation varchar; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column resources.rights_explanation already exists, not creating it.'; - END; - END; -$$; diff --git a/migration/20180712-na-is-not-an-isbn.sql b/migration/20180712-na-is-not-an-isbn.sql deleted file mode 100644 index b154998f..00000000 --- a/migration/20180712-na-is-not-an-isbn.sql +++ /dev/null @@ -1,7 +0,0 @@ -delete from equivalents where output_id in ( - select id from identifiers where type='ISBN' and identifier='n/a' -); -delete from equivalents where input_id in ( - select id from identifiers where type='ISBN' and identifier='n/a' -); -delete from identifiers where type='ISBN' and identifier='n/a'; diff --git a/migration/20180928-add-cascade-on-delete-patron.sql b/migration/20180928-add-cascade-on-delete-patron.sql deleted file mode 100644 index 874e5642..00000000 --- a/migration/20180928-add-cascade-on-delete-patron.sql +++ /dev/null @@ -1,11 +0,0 @@ -ALTER TABLE loans DROP CONSTRAINT loans_patron_id_fkey; -ALTER TABLE loans ADD CONSTRAINT loans_patron_id_fkey FOREIGN KEY (patron_id) REFERENCES patrons (id) ON DELETE CASCADE; - -ALTER TABLE holds DROP CONSTRAINT holds_patron_id_fkey; -ALTER TABLE holds ADD CONSTRAINT holds_patron_id_fkey FOREIGN KEY (patron_id) REFERENCES patrons (id) ON DELETE CASCADE; - -ALTER TABLE annotations DROP CONSTRAINT annotations_patron_id_fkey; -ALTER TABLE annotations ADD CONSTRAINT annotations_patron_id_fkey FOREIGN KEY (patron_id) REFERENCES patrons (id) ON DELETE CASCADE; - -ALTER TABLE credentials DROP CONSTRAINT credentials_patron_id_fkey; -ALTER TABLE credentials ADD CONSTRAINT credentials_patron_id_fkey FOREIGN KEY (patron_id) REFERENCES patrons (id) ON DELETE CASCADE; diff --git a/migration/20181001-lanes-have-all-media.sql b/migration/20181001-lanes-have-all-media.sql deleted file mode 100644 index ba4ed086..00000000 --- a/migration/20181001-lanes-have-all-media.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Remove the media requirement set for precreated lanes back when we --- thought audiobooks and ebooks would have separate lane structures. -update lanes set media=NULL where media='{Book}'; diff --git a/migration/20181002-deliverymechanism-media-type-can-be-null.sql b/migration/20181002-deliverymechanism-media-type-can-be-null.sql deleted file mode 100644 index 8ba429b1..00000000 --- a/migration/20181002-deliverymechanism-media-type-can-be-null.sql +++ /dev/null @@ -1,8 +0,0 @@ --- If a DRM document is also the manifest document, a delivery --- mechanism has no separate content type. -ALTER TABLE deliverymechanisms ALTER COLUMN content_type DROP NOT NULL; - --- MP3 files are not a content type for purposes of describing --- fulfillment, because they are always obtained by processing a --- manifest. In that case the manifest document is the content type. -update deliverymechanisms set content_type=NULL where content_type='audio/mpeg' and drm_scheme='application/vnd.librarysimplified.findaway.license+json'; diff --git a/migration/20181003-1-drop-unused-materialized-views.sql b/migration/20181003-1-drop-unused-materialized-views.sql deleted file mode 100644 index 59b56656..00000000 --- a/migration/20181003-1-drop-unused-materialized-views.sql +++ /dev/null @@ -1,5 +0,0 @@ --- These materialized views haven't been used since January 2018, and --- don't exist in servers created after that date. We can get rid of --- them in servers that were created before that date. -drop materialized view if exists mv_works_editions_datasources_identifiers; -drop materialized view if exists mv_works_editions_workgenres_datasources_identifiers; diff --git a/migration/20181015-size-by-entrypoint.sql b/migration/20181015-size-by-entrypoint.sql deleted file mode 100644 index f3cb8f3c..00000000 --- a/migration/20181015-size-by-entrypoint.sql +++ /dev/null @@ -1 +0,0 @@ -alter table lanes add column size_by_entrypoint json; diff --git a/migration/20181018-1-reindex-works-with-target-age.sql b/migration/20181018-1-reindex-works-with-target-age.sql deleted file mode 100644 index 581c378b..00000000 --- a/migration/20181018-1-reindex-works-with-target-age.sql +++ /dev/null @@ -1,3 +0,0 @@ --- The target ages of works were being incorrectly indexed. --- Reindex all works that have a nontrivial target age. -delete from workcoveragerecords where operation='update-search-index' and work_id in (select id from works where target_age not in ('[,]', '[18,]')); diff --git a/migration/20181018-2-remove-default-medium.sql b/migration/20181018-2-remove-default-medium.sql deleted file mode 100644 index 92fa64c8..00000000 --- a/migration/20181018-2-remove-default-medium.sql +++ /dev/null @@ -1,3 +0,0 @@ --- We no longer make any assumptions about the medium associated --- with an edition. -ALTER TABLE editions ALTER COLUMN medium DROP DEFAULT; diff --git a/migration/20181107-remove-coverage-of-non-books-incorrectly-classified-as-books.sql b/migration/20181107-remove-coverage-of-non-books-incorrectly-classified-as-books.sql deleted file mode 100644 index 75cd8bf3..00000000 --- a/migration/20181107-remove-coverage-of-non-books-incorrectly-classified-as-books.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Delete all choose-edition WorkCoverageRecords for works whose --- presentation edition says they are books, but whose data sources say --- they are not books. --- --- Later, WorkPresentationEditionCoverageProvider will run and recalculate --- the presentation editions properly. - --- We're going to delete some rows from workcoveragerecords. -delete from workcoveragerecords where operation='choose-edition' and - work_id in ( - -- We're looking for works whose presentation editions say they are books. - select w.id - from works w - join licensepools lp on lp.work_id=w.id - join editions e on lp.presentation_edition_id=e.id - where - e.medium='Book' - and lp.id in ( - -- We're looking for works associated with license pools - -- with an associated identifier that says they are _not_ books. - select lp.id - from licensepools lp - join editions e on ( - lp.data_source_id=e.data_source_id - and lp.identifier_id=e.primary_identifier_id - ) - where e.medium != 'Book' - ) - ) -; diff --git a/migration/20181108-remove-overdrive-standin-ids.sql b/migration/20181108-remove-overdrive-standin-ids.sql deleted file mode 100644 index 0ca8befb..00000000 --- a/migration/20181108-remove-overdrive-standin-ids.sql +++ /dev/null @@ -1,17 +0,0 @@ --- This script takes a belt-and-suspenders approach to removing --- placeholder images mistakenly downloaded from Overdrive. - --- Remove the Overdrive bibliographic coverage for affected editions. The --- Overdrive bibliographic coverage provider will try to find new images. -delete from coveragerecords where identifier_id in (select i.id from identifiers i join editions e on e.primary_identifier_id=i.id where (e.cover_full_url like '%00000000-0000-0000-0000%' or e.cover_thumbnail_url like '%00000000-0000-0000-0000%')) and data_source_id in (select ds.id from datasources ds where ds.name='Overdrive'); - --- Delete work coverage records for works whose presentation editions ended up with bad cover images. They'll be refreshed and end up with no cover image. Once a better image is located, their presentation will be refreshed again. -delete from workcoveragerecords where operation='choose-edition' and work_id in (select w.id from works w where presentation_edition_id in (select id from editions where cover_full_url like '%00000000-0000-0000-0000%' or cover_thumbnail_url like '%00000000-0000-0000-0000%')); - --- Remove all hyperlinks that caused an edition to end up with a bad cover or thumbnail. -delete from hyperlinks where id in (select h.id from hyperlinks h join resources res on h.resource_id=res.id join representations rep on res.representation_id=rep.id join editions e on h.identifier_id=e.primary_identifier_id where (e.cover_full_url like '%00000000-0000-0000-0000%' or e.cover_thumbnail_url like '%00000000-0000-0000-0000%') and h.rel='http://opds-spec.org/image' and res.url like '%00000000-0000-0000-0000%'); - --- Change any editions that use bad cover images so that they have no cover image. -update editions set cover_full_url=null where cover_full_url like '%00000000-0000-0000-0000%'; -update editions set cover_thumbnail_url=null where cover_thumbnail_url like '%00000000-0000-0000-0000%'; - diff --git a/migration/20181130-1-add-marc-columns.sql b/migration/20181130-1-add-marc-columns.sql deleted file mode 100644 index 1c7e5edf..00000000 --- a/migration/20181130-1-add-marc-columns.sql +++ /dev/null @@ -1,7 +0,0 @@ -DO $$ - BEGIN - ALTER TABLE works ADD COLUMN marc_record varchar; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'marc_record column already exists, not creating it.'; - END; -$$; diff --git a/migration/20181130-2-recreate-materialized-view.sql b/migration/20181130-2-recreate-materialized-view.sql deleted file mode 100644 index 98642900..00000000 --- a/migration/20181130-2-recreate-materialized-view.sql +++ /dev/null @@ -1,153 +0,0 @@ -drop materialized view mv_works_for_lanes; - --- Create the materialized view with no data. -create materialized view mv_works_for_lanes -as - SELECT - works.id AS works_id, - editions.id AS editions_id, - licensepools.data_source_id, - licensepools.identifier_id, - editions.sort_title, - editions.permanent_work_id, - editions.sort_author, - editions.medium, - editions.language, - editions.cover_full_url, - editions.cover_thumbnail_url, - editions.series, - editions.series_position, - datasources.name, - identifiers.type, - identifiers.identifier, - workgenres.id AS workgenres_id, - workgenres.genre_id, - workgenres.affinity, - works.audience, - works.target_age, - works.fiction, - works.quality, - works.rating, - works.popularity, - works.random, - works.last_update_time, - works.simple_opds_entry, - works.verbose_opds_entry, - works.marc_record, - licensepools.id AS license_pool_id, - licensepools.open_access_download_url, - licensepools.availability_time, - licensepools.collection_id, - customlistentries.list_id, - customlistentries.edition_id as list_edition_id, - customlistentries.first_appearance - - FROM works - JOIN editions ON editions.id = works.presentation_edition_id - JOIN licensepools ON editions.id = licensepools.presentation_edition_id - JOIN datasources ON licensepools.data_source_id = datasources.id - JOIN identifiers on licensepools.identifier_id = identifiers.id - LEFT JOIN customlistentries on works.id = customlistentries.work_id - LEFT JOIN workgenres ON works.id = workgenres.work_id - WHERE works.presentation_ready = true - AND works.simple_opds_entry IS NOT NULL - - ORDER BY (editions.sort_title, editions.sort_author, licensepools.availability_time) - WITH NO DATA; - --- Put an index on all foreign keys. -create index ix_mv_works_for_lanes_works_id on mv_works_for_lanes (works_id); -create index ix_mv_works_for_lanes_license_pool_id on mv_works_for_lanes (license_pool_id); -create index ix_mv_works_for_lanes_workgenres_id on mv_works_for_lanes (workgenres_id); -create index ix_mv_works_for_lanes_list_id on mv_works_for_lanes (list_id); -create index ix_mv_works_for_lanes_list_edition_id on mv_works_for_lanes (list_edition_id); - --- First create an index that allows work/genre lookup. It's unique and incorporates license_pool_id so that the materialized view can be refreshed CONCURRENTLY. --- NOTE: All fields mentioned here also need to be part of the primary key --- for the model object defined in model.py. -create unique index mv_works_for_lanes_unique on mv_works_for_lanes (works_id, genre_id, list_id, list_edition_id, license_pool_id); - --- Create an index on everything, sorted by descending availability time, so that sync feeds are fast. --- TODO: This index might not be necessary anymore. -create index mv_works_for_lanes_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id); - --- Create an index on everything, sorted by 'last update time' (the thing used by crawlable feeds). -create index mv_works_for_lanes_by_recently_updated on mv_works_for_lanes (GREATEST(availability_time, first_appearance, last_update_time) DESC, collection_id, works_id); - --- This index quickly cuts down the number of rows considered when generating feeds for a custom list or the intersection of multiple custom lists. -create index mv_works_for_lanes_list_id_collection_id_language_medium on mv_works_for_lanes (list_id, collection_id, language, medium); - --- Create indexes that are helpful in running the query to find featured works. - -create index mv_works_for_lanes_by_random_and_genre on mv_works_for_lanes (random, language, genre_id); - -create index mv_works_for_lanes_by_random_audience_target_age on mv_works_for_lanes (random, language, audience, target_age); - -create index mv_works_for_lanes_by_random_fiction_audience_target_age on mv_works_for_lanes (random, language, fiction, audience, target_age); - --- Similarly, an index on everything, sorted by descending update time. - -create index mv_works_for_lanes_by_modification on mv_works_for_lanes (last_update_time DESC, sort_author, sort_title, works_id); - --- This index is useful when building feeds of recommended titles. -create index mv_works_for_lanes_identifier_id on mv_works_for_lanes (identifier_id); - --- We need three versions of each index: ---- One that orders by sort_author, sort_title, and works_id ---- One that orders by sort_title, sort_author, and works_id ---- One that orders by availability_time (descending!), sort_title, sort_author, and works_id - --- English adult fiction - -create index mv_works_for_lanes_english_adult_fiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_fiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_fiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language = 'eng'; - --- English adult nonfiction - -create index mv_works_for_lanes_english_adult_nonfiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_nonfiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - -create index mv_works_for_lanes_english_adult_nonfiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language = 'eng'; - --- Nonenglish adult fiction ---- These are also ordered by language - -create index mv_works_for_lanes_nonenglish_adult_fiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_fiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_fiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = true AND language <> 'eng'; - --- Nonenglish adult nonfiction ---- These are also ordered by language - -create index mv_works_for_lanes_nonenglish_adult_nonfiction_by_author on mv_works_for_lanes (sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_nonfiction_by_title on mv_works_for_lanes (sort_title, sort_author, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - -create index mv_works_for_lanes_nonenglish_adult_nonfiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, works_id, language) WHERE audience in ('Adult', 'Adults Only') AND fiction = false AND language <> 'eng'; - --- YA/Children's fiction, regardless of language - -create index mv_works_for_lanes_ya_fiction_by_author on mv_works_for_lanes (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_for_lanes_ya_fiction_by_title on mv_works_for_lanes (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - -create index mv_works_for_lanes_ya_fiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = true; - --- YA/Children's nonfiction, regardless of language - -create index mv_works_for_lanes_ya_nonfiction_by_author on mv_works_for_lanes (sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_for_lanes_ya_nonfiction_by_title on mv_works_for_lanes (sort_title, sort_author, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - -create index mv_works_for_lanes_ya_nonfiction_by_availability on mv_works_for_lanes (availability_time DESC, sort_author, sort_title, language, works_id) WHERE audience in ('Children', 'Young Adult') AND fiction = false; - --- Refresh the new materialized view. -refresh materialized view mv_works_for_lanes; - - diff --git a/migration/20181210-add-cached-marc-file-times.sql b/migration/20181210-add-cached-marc-file-times.sql deleted file mode 100644 index b28f41cd..00000000 --- a/migration/20181210-add-cached-marc-file-times.sql +++ /dev/null @@ -1,14 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE cachedmarcfiles ADD COLUMN start_time timestamp; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'start_time column already exists, not creating it.'; - END; - BEGIN - ALTER TABLE cachedmarcfiles ADD COLUMN end_time timestamp; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'end_time column already exists, not creating it.'; - END; - END; -$$; \ No newline at end of file diff --git a/migration/20190111-unique-delivery-mechanisms.sql b/migration/20190111-unique-delivery-mechanisms.sql deleted file mode 100644 index 3450537f..00000000 --- a/migration/20190111-unique-delivery-mechanisms.sql +++ /dev/null @@ -1,20 +0,0 @@ --- The ix_licensepooldeliveries_datasource_identifier_mechanism index --- is not necessary - an index is automatically created on the same --- fields to enforce a uniqueness constraint. -drop index if exists ix_licensepooldeliveries_datasource_identifier_mechanism; - --- However, the uniqueness constraint doesn't enforce uniqueness when one --- of the fields is null, and one of these fields -- resource_id -- is --- _usually_ null. So we need a unique partial index to properly enforce --- the constraint. -CREATE UNIQUE INDEX if not exists ix_licensepooldeliveries_unique_when_no_resource ON public.licensepooldeliveries USING btree (data_source_id, identifier_id, delivery_mechanism_id) WHERE (resource_id IS NULL); - - --- deliverymechanisms doesn't have a uniqueness constraint, just a unique index. --- Let's change it to a uniqueness constraint (which comes with an implicit --- index) and then add a conditional unique index. -drop index if exists ix_deliverymechanisms_drm_scheme_content_type; - -ALTER TABLE deliverymechanisms ADD CONSTRAINT deliverymechanisms_content_type_drm_scheme UNIQUE (content_type, drm_scheme); - -CREATE UNIQUE INDEX if not exists ix_deliverymechanisms_unique_when_no_drm ON public.deliverymechanisms USING btree (content_type) WHERE (drm_scheme IS NULL); diff --git a/migration/20190125-add-timestamp-fields.sql b/migration/20190125-add-timestamp-fields.sql deleted file mode 100644 index c1b5348b..00000000 --- a/migration/20190125-add-timestamp-fields.sql +++ /dev/null @@ -1,120 +0,0 @@ --- Create a new enumerated type for types of timestamps. -DO $$ - BEGIN - CREATE TYPE service_type AS ENUM ( - 'monitor', - 'coverage_provider', - 'script' - ); - EXCEPTION - WHEN duplicate_object THEN RAISE NOTICE 'service_type already exists, not creating it.'; - END; -$$; - - --- Add columns to the timestamp table, including service_type for the --- enumerated type. - -DO $$ - BEGIN - ALTER TABLE timestamps ADD COLUMN service_type service_type; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'service_type column already exists, not creating it.'; - END; -$$; - -CREATE INDEX if not exists ix_timestamps_service_type ON timestamps USING btree (service_type); - -DO $$ - BEGIN - ALTER TABLE timestamps ADD COLUMN start TIMESTAMP WITHOUT TIME ZONE; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column timestamps already exists, not creating it.'; - END; -$$; - -DO $$ - BEGIN - ALTER TABLE timestamps ADD COLUMN achievements CHARACTER VARYING; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column achievements already exists, not creating it.'; - END; -$$; - -DO $$ - BEGIN - ALTER TABLE timestamps ADD COLUMN exception CHARACTER VARYING; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column exception already exists, not creating it.'; - END; -$$; - -DO $$ - BEGIN - ALTER TABLE timestamps RENAME COLUMN timestamp TO finish; - EXCEPTION - WHEN undefined_column THEN RAISE NOTICE 'column timestamp does not exist, assuming already renamed.'; - END; -$$; - --- Set service_type for all known monitors, coverage providers, and --- scripts. - --- First let's take care of the easy cases. - --- All 'monitor' and 'sweep' services are monitors. -update timestamps set service_type='monitor' where ( - service ilike '%monitor%' - or service ilike '%sweep%' -); - --- All 'coverage' services are coverage providers. -update timestamps set service_type='coverage_provider' where service ilike '%coverage%'; - --- The database migration timestamps are managed by scripts. -update timestamps set service_type='script' where ( - service like 'Database Migration%' -); - --- The metadata wrangler reaper is a coverage provider; all other reapers --- are monitors. -update timestamps set service_type='coverage_provider' where ( - service='Metadata Wrangler Reaper' -); -update timestamps set service_type='monitor' where ( - service ilike '%reaper%' and service_type is null -); - --- Now we get into specific cases where it's not clear from the service --- name what is what. - --- All RBdigital and search index services are monitors. -update timestamps set service_type='monitor' where ( - service ilike 'search index update%' or - service ilike 'rbdigital%' -); - -update timestamps set service_type='monitor' where service in ( - 'Metadata Wrangler Collection Updates', - 'Metadata Wrangler Auxiliary Metadata Delivery', - 'Work Randomness Updater', - 'Overdrive Collection Overview' -); - -update timestamps set service_type='coverage_provider' where ( - service ilike '%metadata wrangler collection registrar%' -); - -update timestamps set service_type='coverage_provider' where service in ( - 'OCLC Classify Identifier Lookup' -); - --- Finally, apart from the database migration scripts, which are --- covered above, every timestamp that uses the counter is a Monitor. -update timestamps set service_type='monitor' where ( - counter is not null and service_type is null -); - --- Fill in the 'start' values -- they'll be replaced with more accurate values --- as the scripts run. -update timestamps set start=finish where start is null; diff --git a/migration/20190126-drop-work-contributions.sql b/migration/20190126-drop-work-contributions.sql deleted file mode 100644 index f48cbaeb..00000000 --- a/migration/20190126-drop-work-contributions.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Remove the never-used workcontributions table. -drop table if exists workcontributions; diff --git a/migration/20190204-enabled-equivalents.sql b/migration/20190204-enabled-equivalents.sql deleted file mode 100644 index b319b36f..00000000 --- a/migration/20190204-enabled-equivalents.sql +++ /dev/null @@ -1,6 +0,0 @@ -alter table equivalents add column enabled boolean default true; -CREATE INDEX ix_equivalents_enabled ON equivalents USING btree (enabled); - --- Delete the recursive_equivalents function -- it will be recreated in --- its new form when the app server starts up. -DROP FUNCTION IF EXISTS fn_recursive_equivalents(int, int, double precision, int); diff --git a/migration/20190219-add-licensepooldeliveries-constraint-if-not-exists.sql b/migration/20190219-add-licensepooldeliveries-constraint-if-not-exists.sql deleted file mode 100644 index e0f57e66..00000000 --- a/migration/20190219-add-licensepooldeliveries-constraint-if-not-exists.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Make sure the unique constraint on licensepooldeliveries exists; it may not exist in older databases. -DO $$ - BEGIN - alter table licensepooldeliveries add constraint licensepooldeliveries_data_source_id_identifier_id_delivery_key unique (data_source_id, identifier_id, delivery_mechanism_id, resource_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'licensepooldeliveries constraint already exists, not creating it.'; - END; -$$; diff --git a/migration/20190328-1-add-license-to-loans.sql b/migration/20190328-1-add-license-to-loans.sql deleted file mode 100644 index 77ae5843..00000000 --- a/migration/20190328-1-add-license-to-loans.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table loans add column license_id integer; -alter table loans add constraint loans_license_id_key foreign key (license_id) references licenses(id); diff --git a/migration/20190404-add-deletion-flag-collections.sql b/migration/20190404-add-deletion-flag-collections.sql deleted file mode 100644 index cbdcf7b4..00000000 --- a/migration/20190404-add-deletion-flag-collections.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Add a marked_for_deletion column to keep track if a collection should be deleted. -alter table collections add column "marked_for_deletion" boolean default false; diff --git a/migration/20190522-1-add-collection-to-credentials.sql b/migration/20190522-1-add-collection-to-credentials.sql deleted file mode 100644 index 5fd17187..00000000 --- a/migration/20190522-1-add-collection-to-credentials.sql +++ /dev/null @@ -1,5 +0,0 @@ -alter table credentials add column collection_id integer; -alter table credentials add constraint "credentials_collection_id_fkey" FOREIGN KEY (collection_id) REFERENCES collections(id); -alter table credentials add constraint "credentials_data_source_id_patron_id_collection_id_type_key" UNIQUE (data_source_id, patron_id, collection_id, type); -alter table credentials drop constraint credentials_data_source_id_patron_id_type_key; -create index ix_credentials_collection_id on credentials(collection_id); \ No newline at end of file diff --git a/migration/20190617-add-size-to-customlists.sql b/migration/20190617-add-size-to-customlists.sql deleted file mode 100644 index c5db3f97..00000000 --- a/migration/20190617-add-size-to-customlists.sql +++ /dev/null @@ -1,13 +0,0 @@ -DO $$ - BEGIN - BEGIN - ALTER TABLE customlists ADD COLUMN size integer not null default 0; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column customlists.size already exists, not creating it.'; - END; - UPDATE customlists SET size = COALESCE( - (SELECT subq.c FROM - (SELECT list_id AS l, count(*) AS c FROM customlistentries GROUP BY list_id) - AS subq WHERE subq.l = customlists.id), - 0); - END $$; diff --git a/migration/20190629-remove-sitewide-cache-time-settings.sql b/migration/20190629-remove-sitewide-cache-time-settings.sql deleted file mode 100644 index 97977ba4..00000000 --- a/migration/20190629-remove-sitewide-cache-time-settings.sql +++ /dev/null @@ -1,2 +0,0 @@ --- These site-wide configuration settings are no longer used. -delete from configurationsettings where library_id is null and external_integration_id is null and key in ('default_nongrouped_feed_max_age', 'default_grouped_feed_max_age'); diff --git a/migration/20190710-drop-materialized-view.sql b/migration/20190710-drop-materialized-view.sql deleted file mode 100644 index e918cb0e..00000000 --- a/migration/20190710-drop-materialized-view.sql +++ /dev/null @@ -1 +0,0 @@ -drop materialized view if exists mv_works_for_lanes; diff --git a/migration/20190907-circulation-event-migration.sql b/migration/20190907-circulation-event-migration.sql deleted file mode 100644 index 077e1238..00000000 --- a/migration/20190907-circulation-event-migration.sql +++ /dev/null @@ -1,52 +0,0 @@ --- Delete all duplicate circulation events before creating unique indexes. -delete from circulationevents where id in (select ce1.id from circulationevents ce1 join circulationevents as ce2 on ce1.license_pool_id = ce2.license_pool_id and ce1.type = ce2.type and ce1.start = ce2.start and ce1.id < ce2.id); - -DO $$ - BEGIN - -- Add the 'location' column - BEGIN - ALTER TABLE circulationevents ADD COLUMN location varchar; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column circulationevents.location already exists, not creating it.'; - END; - - -- Add the 'library_id' column - BEGIN - ALTER TABLE circulationevents ADD COLUMN library_id int; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column circulationevents.library_id already exists, not creating it.'; - END; - - -- Make the 'library_id' column a foreign key reference to libraries.id. - BEGIN - ALTER TABLE circulationevents ADD CONSTRAINT circulationevents_library_id_fkey FOREIGN KEY (library_id) REFERENCES libraries(id); - EXCEPTION - when duplicate_object THEN RAISE NOTICE 'column circulationevents.library_id is already a foreign key; leaving it alone.'; - END; - - -- Remove the unique constraint that involves the 'foreign_patron_id' field. - alter table circulationevents drop constraint if exists circulationevents_license_pool_id_type_start_foreign_patron_key; - - -- Drop the 'foreign_patron_id' field itself. - alter table circulationevents drop column if exists foreign_patron_id; - - -- Create some indexes to enforce uniqueness constraints. - - -- If there is no library ID, then licence pool + type + start - -- must be unique. - BEGIN - CREATE UNIQUE index ix_circulationevents_license_pool_type_start on circulationevents (license_pool_id, type, start) where library_id is null; - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'unique index ix_circulationevents_license_pool_type_start already exists; leaving it alone.'; - END; - - -- If there is a library ID, then license pool + library + - -- type + start must be unique. - BEGIN - CREATE UNIQUE index ix_circulationevents_license_pool_library_type_start on circulationevents (license_pool_id, library_id, type, start); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'unique index ix_circulationevents_license_pool_library_type_start already exists; leaving it alone.'; - END; - - END; -$$; diff --git a/migration/20190912-patron-cached-neighborhood.sql b/migration/20190912-patron-cached-neighborhood.sql deleted file mode 100644 index 4a36129c..00000000 --- a/migration/20190912-patron-cached-neighborhood.sql +++ /dev/null @@ -1,18 +0,0 @@ -DO $$ - BEGIN - -- Add the 'cached_neighborhood' column - BEGIN - ALTER TABLE patrons ADD COLUMN cached_neighborhood varchar; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column patrons.cached_neighborhood already exists, not creating it.'; - END; - - -- Index this field so we can easily scrub values when the cache expires. - BEGIN - CREATE index ix_patrons_cached_neighborhood on patrons (cached_neighborhood); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'index ix_patrons_cached_neighborhood already exists; leaving it alone.'; - END; - - END; -$$; diff --git a/migration/20190916-remove-main-collection-facet.sql b/migration/20190916-remove-main-collection-facet.sql deleted file mode 100644 index 7ff82589..00000000 --- a/migration/20190916-remove-main-collection-facet.sql +++ /dev/null @@ -1,17 +0,0 @@ --- 'main' is no longer a recognized facet in the 'collection' facet --- group. - --- This migration script removes it from the list of enabled facets, --- if it's present, and changes the default if 'main' is the default. - --- Remove '"main",' for cases where "main" is not the last item in the JSON list. -update configurationsettings set value = replace(value, '"main",', '') where key='facets_enabled_collection'; - --- Remove ', "main"' for cases where "main" is the last item in the JSON list. -update configurationsettings set value = replace(value, ', "main"]', ']') where key='facets_enabled_collection'; - --- If 'main' is the *only* enabled collection facet, use 'full' instead. -update configurationsettings set value = '["full"]' where key='facets_enabled_collection' and value='["main"]'; - --- If 'main' was the default collection, change it. -update configurationsettings set value='full' where key='facets_default_collection' and value='main'; diff --git a/migration/20190926-reweight-classifications.sql b/migration/20190926-reweight-classifications.sql deleted file mode 100644 index 90814689..00000000 --- a/migration/20190926-reweight-classifications.sql +++ /dev/null @@ -1,33 +0,0 @@ --- This reweights some classifications to reflect changes in the way --- we weight data from different sources and the fact that we no longer --- infer audience from Dewey and LCC classifications. - --- Give any Axis 360 classifications weights that are more like what --- we see from other commercial distributors. -update classifications set weight=100 where id in (select c.id from classifications c join datasources ds on ds.id=c.data_source_id where ds.name='Axis 360' and weight=1); - --- Weight any Bibliotheca BISAC classifications similarly. -update classifications set weight=100 where id in (select c.id from classifications c join subjects s on s.id=c.subject_id join datasources ds on ds.id=c.data_source_id where ds.name='Bibliotheca' and s.type='BISAC' and c.weight=15); - --- Same for ENKI tags. -update classifications set weight=100 where id in (select c.id from classifications c join subjects s on s.id=c.subject_id join datasources ds on ds.id=c.data_source_id where ds.name='Enki' and s.type='tag' and c.weight=1); - --- Same for all classifications from RBdigital or Odilo, regardless of type. -update classifications set weight=100 where id in (select c.id from classifications c join subjects s on s.id=c.subject_id join datasources ds on ds.id=c.data_source_id where ds.name in ('RBdigital', 'Odilo')); - --- Give a weight of 1 to any schema:audience and schema:targetAge --- classifications that came from the metadata wrangler. -update classifications set weight=1 where id in (select c.id from classifications c join datasources ds on ds.id=c.data_source_id join subjects s on s.id=c.subject_id where ds.name='Library Simplified metadata wrangler' and s.type in ('schema:audience', 'schema:targetAge') and c.weight=100); - --- Clear out the audience of all DDC and LCC subjects where were were --- deriving audience=Adult from a lack of explicit information. -update subjects set audience=NULL where type in ('DDC', 'LCC') and audience='Adult'; - --- Delete the 'classify' coverage record for every work on the site, --- effectively forcing all works to be reclassified. --- --- We don't technically need to delete every single 'classify' record, --- but on a normal site we will effectively end up deleting all of --- them, and the query to pinpoint only the ones that need to be --- deleted takes a really long time to run. -delete from workcoveragerecords where operation='classify'; diff --git a/migration/20191010-update-external-integration-mirrors.sql b/migration/20191010-update-external-integration-mirrors.sql deleted file mode 100644 index f666d0cd..00000000 --- a/migration/20191010-update-external-integration-mirrors.sql +++ /dev/null @@ -1,38 +0,0 @@ -DO $$ - BEGIN - BEGIN - CREATE TABLE externalintegrationslinks ( - id SERIAL PRIMARY KEY, - external_integration_id INTEGER REFERENCES externalintegrations(id), - library_id INTEGER REFERENCES libraries(id), - other_integration_id INTEGER REFERENCES externalintegrations(id), - purpose VARCHAR - ); - - CREATE INDEX ix_externalintegrationslinks_external_integration_id ON externalintegrationslinks USING btree (external_integration_id); - CREATE INDEX ix_externalintegrationslinks_library_id ON externalintegrationslinks USING btree (library_id); - CREATE INDEX ix_externalintegrationslinks_other_integration_id ON externalintegrationslinks USING btree (other_integration_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Warning: externalintegrationslinks already exists.'; - END; - END; -$$; - --- Previously, collections could only have one mirror integration associated --- with it. Now, a collection can currently have two external integration storages --- for its "books_mirror" and "covers_mirror" mirrors. Any existing mirror integration --- associated with a collection is now linked through the ExternalIntegrationsLinks --- table. The mirror integration will be set to both "books_mirror" and "covers_mirror" mirrors. - - -insert into externalintegrationslinks (external_integration_id, other_integration_id, purpose) -select external_integration_id, mirror_integration_id, 'books_mirror' -from collections -where mirror_integration_id is not null; - -insert into externalintegrationslinks (external_integration_id, other_integration_id, purpose) -select external_integration_id, mirror_integration_id, 'covers_mirror' -from collections -where mirror_integration_id is not null; - -ALTER TABLE collections DROP COLUMN mirror_integration_id; \ No newline at end of file diff --git a/migration/20191022-unique-credentials.sql b/migration/20191022-unique-credentials.sql deleted file mode 100644 index 51dd370d..00000000 --- a/migration/20191022-unique-credentials.sql +++ /dev/null @@ -1,52 +0,0 @@ --- Delete all duplicate credentials before creating unique indexes. -delete from credentials where id in (select c1.id from credentials c1 join credentials c2 on c1.data_source_id = c2.data_source_id and c1.patron_id=c2.patron_id and c1.type = c2.type and (c1.collection_id = c2.collection_id or (c1.collection_id is null and c2.collection_id is null)) and c1.id < c2.id); - -DO $$ - BEGIN - -- Remove the unique index on (data_source_id, type, credential). - -- We'll recreate a better version of it immediately afterwards. - drop index if exists ix_credentials_data_source_id_type_token; - - -- If both patron_id and collection_id are null, then (data_source_id, - -- type, credential) must be unique. - BEGIN - CREATE UNIQUE index ix_credentials_data_source_id_type_credential on credentials (data_source_id, type, credential) where patron_id is null and collection_id is null; - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Index ix_credentials_data_source_id_type_credential already exists, leaving it alone.'; - END; - - -- If patron_id is null, then (data_source_id, type, collection_id) - -- must be unique. - BEGIN - CREATE UNIQUE index ix_credentials_data_source_id_type_collection_id on credentials (data_source_id, type, collection_id) where patron_id is null; - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Index ix_credentials_data_source_id_type_collection_id already exists, leaving it alone.'; - END; - - -- If collection_id is null but patron_id is not, then - -- (data_source_id, type, patron_id) must be unique. - BEGIN - CREATE UNIQUE index ix_credentials_data_source_id_type_patron_id on credentials (data_source_id, type, patron_id) where collection_id is null; - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Index ix_credentials_data_source_id_type_patron_id already exists, leaving it alone.'; - END; - - -- If patron_id is null but collection_id is not, then - -- (data_source_id, type, collection_id) must be unique. - -- (At the moment this never happens.) - BEGIN - CREATE UNIQUE index ix_credentials_data_source_id_type_collection_id on credentials (data_source_id, type, collection_id) where patron_id is null; - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Index ix_credentials_data_source_id_type_collection_id already exists, leaving it alone.'; - END; - - -- If both patron_id and collection_id have values, then - -- (data_source_id, type, patron_id, collection_id) must be unique. - BEGIN - CREATE UNIQUE index ix_credentials_data_source_id_type_patron_id_collection_id on credentials (data_source_id, type, patron_id, collection_id); - EXCEPTION - WHEN duplicate_table THEN RAISE NOTICE 'Index ix_credentials_data_source_id_type_patron_id_collection_id already exists, leaving it alone.'; - END; - - END; -$$; diff --git a/migration/20191119-integrationclient-enabled.sql b/migration/20191119-integrationclient-enabled.sql deleted file mode 100644 index f72bf25b..00000000 --- a/migration/20191119-integrationclient-enabled.sql +++ /dev/null @@ -1,10 +0,0 @@ -DO $$ - BEGIN - -- Add the 'enabled' column - BEGIN - ALTER TABLE integrationclients ADD COLUMN enabled boolean default true; - EXCEPTION - WHEN duplicate_column THEN RAISE NOTICE 'column integrationclients.enabled already exists, not creating it.'; - END; - END; -$$;