From 5d87858b979996d533df267e9aa2469668accc5b Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Sat, 20 Aug 2022 01:20:57 +0200 Subject: [PATCH 01/10] feat: 17 created package and class structure, added scrapers.run.cron.expression property to application.properties, added test for scrapers core --- pom.xml | 6 + .../hub/event/scrapers/ScrapersService.java | 7 - .../java/hub/event/scrapers/Scrappable.java | 4 - .../scrapers/core/AnalyzedEventCandidate.java | 32 ++ .../scrapers/core/EventCandidateAnalyzer.java | 16 + .../scrapers/core/EventFacadeAdapter.java | 9 + .../hub/event/scrapers/core/ExistsEvent.java | 9 + .../scrapers/core/LastScrapedEventMarker.java | 6 + .../LastScrapedEventMarkerRepository.java | 9 + .../event/scrapers/core/PageScraperPort.java | 8 + .../ProbablyDuplicatedEventCandidate.java | 27 ++ ...blyDuplicatedEventCandidateRepository.java | 7 + .../hub/event/scrapers/core/ScrapedEvent.java | 59 +++ .../scrapers/core/ScrapedEventBuilder.java | 50 +++ .../event/scrapers/core/ScraperConfig.java | 4 + .../core/ScraperConfigRepository.java | 16 + .../event/scrapers/core/ScraperFacade.java | 38 ++ .../scrapers/core/ScraperRunService.java | 27 ++ .../core/datewithlocation/EventDateType.java | 6 + .../EventDateWithLocation.java | 41 +++ .../core/datewithlocation/EventLocation.java | 22 ++ .../MultipleEventDateWithLocations.java | 22 ++ .../SingleEventDateWithLocation.java | 67 ++++ .../LastScrapedEventMarkerEntity.java | 52 +++ ...astScrapedEventMarkerEntityRepository.java | 45 +++ .../LastScrapedEventMarkerJpaRepository.java | 10 + ...ntDateEndDateBeforeStartDateException.java | 4 + ...ntDateEndTimeBeforeStartTimeException.java | 4 + .../exceptions/EventDateInPastException.java | 4 + .../ScraperConfigurationByNameNotExists.java | 4 + .../core/runlog/ScraperRunErrorLog.java | 6 + .../core/runlog/ScraperRunLogFacade.java | 21 ++ .../core/runlog/ScraperRunLogRepository.java | 8 + .../core/runlog/ScraperRunStatusLog.java | 7 + .../event/scrapers/ebilet/EbiletScraper.java | 17 +- .../empikbilet/EmpikBiletScraper.java | 18 +- .../scrapers/goingapp/GoingAppScraper.java | 18 +- .../kupbilecik/KupBilecikScraper.java | 17 +- .../scrapers/proanima/ProanimaScraper.java | 17 +- .../hub/event/statistics/StatsService.java | 4 - src/main/resources/application.properties | 4 +- .../core/ScrapedEventBuilderTest.java | 88 +++++ .../scrapers/core/ScraperFacadeTest.java | 78 ++++ .../scrapers/core/ScraperRunServiceTest.java | 335 ++++++++++++++++++ .../MultipleEventDateWithLocationsTest.java | 63 ++++ .../SingleEventDateWithLocationTest.java | 148 ++++++++ .../LastScrapedEventMarkerRepositoryTest.java | 103 ++++++ .../core/runlog/ScraperRunLogFacadeTest.java | 80 +++++ 48 files changed, 1621 insertions(+), 26 deletions(-) delete mode 100644 src/main/java/hub/event/scrapers/ScrapersService.java delete mode 100644 src/main/java/hub/event/scrapers/Scrappable.java create mode 100644 src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java create mode 100644 src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java create mode 100644 src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java create mode 100644 src/main/java/hub/event/scrapers/core/ExistsEvent.java create mode 100644 src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java create mode 100644 src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/PageScraperPort.java create mode 100644 src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java create mode 100644 src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScrapedEvent.java create mode 100644 src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfig.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperFacade.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunService.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/EventDateType.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java create mode 100644 src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java create mode 100644 src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java create mode 100644 src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java create mode 100644 src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java create mode 100644 src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java create mode 100644 src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java create mode 100644 src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java create mode 100644 src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java create mode 100644 src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java create mode 100644 src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java create mode 100644 src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java create mode 100644 src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java diff --git a/pom.xml b/pom.xml index e1eb2fc..6d5487f 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,12 @@ 1.0.0-rc1 test + + org.assertj + assertj-core + 3.23.1 + test + diff --git a/src/main/java/hub/event/scrapers/ScrapersService.java b/src/main/java/hub/event/scrapers/ScrapersService.java deleted file mode 100644 index 50eb8c4..0000000 --- a/src/main/java/hub/event/scrapers/ScrapersService.java +++ /dev/null @@ -1,7 +0,0 @@ -package hub.event.scrapers; - -import hub.event.events.EventService; - -public class ScrapersService { - -} diff --git a/src/main/java/hub/event/scrapers/Scrappable.java b/src/main/java/hub/event/scrapers/Scrappable.java deleted file mode 100644 index 2a1a9ee..0000000 --- a/src/main/java/hub/event/scrapers/Scrappable.java +++ /dev/null @@ -1,4 +0,0 @@ -package hub.event.scrapers; - -public interface Scrappable { -} diff --git a/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java b/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java new file mode 100644 index 0000000..df38d56 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java @@ -0,0 +1,32 @@ +package hub.event.scrapers.core; + +import java.util.ArrayList; +import java.util.List; + +class AnalyzedEventCandidate { + private final List probablyDuplicateCandidates; + private final List probablyDuplicateEvents; + private final ScrapedEvent scrapedEvent; + + AnalyzedEventCandidate(ScrapedEvent scrapedEvent) { + this.scrapedEvent = scrapedEvent; + this.probablyDuplicateCandidates = new ArrayList<>(); + this.probablyDuplicateEvents = new ArrayList<>(); + } + + boolean isDuplicateWithCandidate() { + return !this.probablyDuplicateCandidates.isEmpty(); + } + + void addDuplicateCandidate(ScrapedEvent scrapedEvent) { + this.probablyDuplicateCandidates.add(scrapedEvent); + } + + boolean isDuplicateWithEvent() { + return !this.probablyDuplicateEvents.isEmpty(); + } + + void addDuplicateEvent(ExistsEvent existsEvent) { + this.probablyDuplicateEvents.add(existsEvent); + } +} diff --git a/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java b/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java new file mode 100644 index 0000000..56fb81c --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java @@ -0,0 +1,16 @@ +package hub.event.scrapers.core; + +import java.util.List; + +class EventCandidateAnalyzer { + List analyze(List scrapedEventList) { + //TODO replace this just map implementation by real one + return scrapedEventList.stream() + .map(this::mapAnalyzedEventCandidate) + .toList(); + } + + private AnalyzedEventCandidate mapAnalyzedEventCandidate(ScrapedEvent scrapedEvent) { + return new AnalyzedEventCandidate(scrapedEvent); + } +} diff --git a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java new file mode 100644 index 0000000..341e5eb --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java @@ -0,0 +1,9 @@ +package hub.event.scrapers.core; + +import java.util.List; + +class EventFacadeAdapter { + public void saveAll(List scrapedEventList) { + //TODO integration with EventFacade from event module + } +} diff --git a/src/main/java/hub/event/scrapers/core/ExistsEvent.java b/src/main/java/hub/event/scrapers/core/ExistsEvent.java new file mode 100644 index 0000000..09242e6 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ExistsEvent.java @@ -0,0 +1,9 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; + +record ExistsEvent(int eventId, String title, String description, + SingleEventDateWithLocation singleEventDateWithLocation, + MultipleEventDateWithLocations multipleEventDateWithLocations) { +} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java new file mode 100644 index 0000000..ba8a202 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java @@ -0,0 +1,6 @@ +package hub.event.scrapers.core; + +import java.time.LocalDateTime; + +public record LastScrapedEventMarker(String scraperConfigurationName, LocalDateTime eventDate , String eventTitle, String marker) { +} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java new file mode 100644 index 0000000..9ee441f --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java @@ -0,0 +1,9 @@ +package hub.event.scrapers.core; + +import java.util.Optional; + +public interface LastScrapedEventMarkerRepository { + void store(LastScrapedEventMarker lastScrapedEventMarker); + + Optional findByScraperConfigurationName(String configurationName); +} diff --git a/src/main/java/hub/event/scrapers/core/PageScraperPort.java b/src/main/java/hub/event/scrapers/core/PageScraperPort.java new file mode 100644 index 0000000..13799d0 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/PageScraperPort.java @@ -0,0 +1,8 @@ +package hub.event.scrapers.core; + +import java.util.Collection; + +public interface PageScraperPort { + String configurationName(); + Collection scrap(); +} \ No newline at end of file diff --git a/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java b/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java new file mode 100644 index 0000000..efdd2ca --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java @@ -0,0 +1,27 @@ +package hub.event.scrapers.core; + +import java.util.List; +import java.util.UUID; + +class ProbablyDuplicatedEventCandidate { + private UUID uuid; + private ScrapedEvent scrapedEvent; + private List duplicatedCandidates; + private List duplicatedEvents; + + UUID getUuid() { + return uuid; + } + + ScrapedEvent getScrapedEvent() { + return scrapedEvent; + } + + List getDuplicatedCandidates() { + return duplicatedCandidates; + } + + List getDuplicatedEvents() { + return duplicatedEvents; + } +} diff --git a/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java b/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java new file mode 100644 index 0000000..65bddbb --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core; + +import java.util.List; + +interface ProbablyDuplicatedEventCandidateRepository { + void saveAll(List candidates); +} diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java new file mode 100644 index 0000000..5315056 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java @@ -0,0 +1,59 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; + +import java.time.LocalDateTime; +import java.util.Map; +import java.util.Objects; + +public class ScrapedEvent { + + private Map metadata; + private String title; + private String description; + private String sourceLink; + private SingleEventDateWithLocation singleEventDateWithLocation; + private MultipleEventDateWithLocations multipleEventDateWithLocations; + private LocalDateTime scrapedTime; + + private ScrapedEvent() { + } + + public static ScrapedEventBuilder builder() { + return new ScrapedEventBuilder(); + } + + String title() { + return title; + } + + String description() { + return description; + } + + String sourceLink() { + return sourceLink; + } + + Map metadata() { + return metadata; + } + + SingleEventDateWithLocation singleEventDateWithLocation() { + return singleEventDateWithLocation; + } + + MultipleEventDateWithLocations multipleEventDateWithLocations() { + return multipleEventDateWithLocations; + } + + public LocalDateTime scrapedTime() { + return scrapedTime; + } + + public boolean hasMultipleDateAndLocations() { + return Objects.nonNull(multipleEventDateWithLocations); + } + +} diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java b/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java new file mode 100644 index 0000000..3b0277b --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java @@ -0,0 +1,50 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; + +class ScrapedEventBuilder { + + private final Map metadata; + + ScrapedEventBuilder() { + this.metadata = new HashMap<>(); + } + + public ScrapedEvent build() { + return null; + } + + public ScrapedEventBuilder title(String title) { + return this; + } + + public ScrapedEventBuilder description(String description) { + return this; + } + + public ScrapedEventBuilder sourceLink(String sourceLink) { + return this; + } + + public ScrapedEventBuilder metadata(String key, String value) { + this.metadata.put(key, value); + return this; + } + + public ScrapedEventBuilder date(SingleEventDateWithLocation singleEventDateWithLocation) { + return this; + } + + public ScrapedEventBuilder date(MultipleEventDateWithLocations multipleEventDateWithLocations) { + return this; + } + + public ScrapedEventBuilder scrapedTime(LocalDateTime scrapedTime) { + return this; + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfig.java b/src/main/java/hub/event/scrapers/core/ScraperConfig.java new file mode 100644 index 0000000..5f9fda1 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperConfig.java @@ -0,0 +1,4 @@ +package hub.event.scrapers.core; + +public record ScraperConfig(String configurationName, boolean isActive) { +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java new file mode 100644 index 0000000..5ecc26a --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java @@ -0,0 +1,16 @@ +package hub.event.scrapers.core; + +import java.util.Collection; + +interface ScraperConfigRepository { + + boolean exists(String scraperName); + + void create(String scraperName); + + void activate(String scraperConfigurationName); + + void deactivate(String scraperConfigurationName); + + Collection allScraperConfigs(); +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperFacade.java b/src/main/java/hub/event/scrapers/core/ScraperFacade.java new file mode 100644 index 0000000..ed08ca4 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperFacade.java @@ -0,0 +1,38 @@ +package hub.event.scrapers.core; + +import org.apache.commons.lang3.NotImplementedException; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.UUID; + +@Service +public class ScraperFacade { +private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; +private final ScraperConfigRepository scraperConfigRepository; + +private final ProbablyDuplicatedEventCandidateRepository probablyDuplicatedEventCandidateRepository; + + public ScraperFacade(LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, ScraperConfigRepository scraperConfigRepository, ProbablyDuplicatedEventCandidateRepository probablyDuplicatedEventCandidateRepository) { + this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; + this.scraperConfigRepository = scraperConfigRepository; + this.probablyDuplicatedEventCandidateRepository = probablyDuplicatedEventCandidateRepository; + } + + public void activateScraperByConfigurationName(String scraperName) { + } + + public void deactivateScraperByConfigurationName(String scraperName) { + } + + //Maybe should be in separated module + public List getAllProbablyDuplicatedEventCandidates(){ + throw new NotImplementedException("getAllProbablyDuplicatedEventCandidates is not implemented. "); + } + + //Maybe should be in separated module + public boolean deleteByProbablyDuplicatedEventCandidateUuid(UUID eventCandidateUuid){ + throw new NotImplementedException("deleteByProbablyDuplicatedEventCandidateUuid is not implemented. "); + } + +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java new file mode 100644 index 0000000..5732bd3 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -0,0 +1,27 @@ +package hub.event.scrapers.core; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.util.List; + + +@Service +@EnableScheduling +class ScraperRunService { + private final ScraperConfigRepository scraperConfigRepository; + private final List pageScrapers; + + @Autowired + ScraperRunService(ScraperConfigRepository scraperConfigRepository, List pageScrapers) { + this.scraperConfigRepository = scraperConfigRepository; + this.pageScrapers = pageScrapers; + } + + @Scheduled(cron = "${scrapers.run.cron.expression}") +// @Scheduled(cron = "0 0 * * *") + void start() { + } +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateType.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateType.java new file mode 100644 index 0000000..eb4a752 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateType.java @@ -0,0 +1,6 @@ +package hub.event.scrapers.core.datewithlocation; + +enum EventDateType { + SINGLE, + PERIOD, +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java new file mode 100644 index 0000000..b6e70c8 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java @@ -0,0 +1,41 @@ +package hub.event.scrapers.core.datewithlocation; + +import java.time.LocalDate; +import java.time.LocalTime; + +class EventDateWithLocation { + private LocalDate startDate; + private LocalDate endDate; + private LocalTime startTime; + private LocalTime endTime; + + private EventLocation eventLocation; + + String city() { + return eventLocation.city(); + } + + LocalDate startDate() { + return startDate; + } + + LocalTime startTime() { + return startTime; + } + + LocalDate endDate() { + return endDate; + } + + LocalTime endTime() { + return endTime; + } + + String address() { + return eventLocation.address(); + } + + String locationName() { + return eventLocation.name(); + } +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java new file mode 100644 index 0000000..d8fc946 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java @@ -0,0 +1,22 @@ +package hub.event.scrapers.core.datewithlocation; + +class EventLocation { + private String city; + + private String address; + + private String name; + + public String city() { + return city; + } + + public String address() { + return address; + } + + public String name() { + return name; + } + +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java new file mode 100644 index 0000000..039dfb0 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java @@ -0,0 +1,22 @@ +package hub.event.scrapers.core.datewithlocation; + +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.Collection; + +public class MultipleEventDateWithLocations { + private Collection eventDateWithLocations; + + public static MultipleEventDateWithLocations create(LocalDate date, LocalTime time, String city, String address, String locationName) { + return null; + } + + public MultipleEventDateWithLocations add(LocalDate date, LocalTime time, String city, String address, String locationName) { + return this; + } + + Collection eventDateWithLocations(){ + return this.eventDateWithLocations; + } + +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java new file mode 100644 index 0000000..d578e42 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java @@ -0,0 +1,67 @@ +package hub.event.scrapers.core.datewithlocation; + +import java.time.LocalDate; +import java.time.LocalTime; + +public class SingleEventDateWithLocation { + private EventDateType eventDateType; + private EventDateWithLocation eventDateWithLocation; + + private SingleEventDateWithLocation() { + + } + + public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, String city, String address, String locationName) { + return null; + } + + public static SingleEventDateWithLocation single(LocalDate startDate, LocalDate endDate, LocalTime startTime, LocalTime endTime, String city, String address, String locationName) { + return null; + } + + public static SingleEventDateWithLocation period(LocalDate startDate, LocalDate endDate, LocalTime startTime, String city, String address, String locationName) { + return null; + } + + public static SingleEventDateWithLocation period(LocalDate startDate, LocalDate endDate, LocalTime startTime, LocalTime endTime, String city, String address, String locationName) { + return null; + } + + LocalDate startDate() { + return eventDateWithLocation.startDate(); + } + + LocalTime startTime() { + return eventDateWithLocation.startTime(); + } + + String city() { + return eventDateWithLocation.city(); + } + + LocalDate endDate() { + return eventDateWithLocation.endDate(); + } + + LocalTime endTime() { + return eventDateWithLocation.endTime(); + } + + + boolean isSingleDate() { + return EventDateType.SINGLE.equals(this.eventDateType); + } + + boolean isPeriodDate() { + return EventDateType.PERIOD.equals(this.eventDateType); + } + + String address() { + return eventDateWithLocation.address(); + } + + String locationName() { + return eventDateWithLocation.locationName(); + } + +} diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java new file mode 100644 index 0000000..00f316f --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java @@ -0,0 +1,52 @@ +package hub.event.scrapers.core.entityrepository; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import java.time.LocalDateTime; + +@Entity +class LastScrapedEventMarkerEntity { + @Id + private String scraperConfigurationName; + @Column(nullable = false) + private LocalDateTime eventDate; + private String eventTitle; + @Column(nullable = false) + private String marker; + + LastScrapedEventMarkerEntity() { + } + + public String getScraperConfigurationName() { + return scraperConfigurationName; + } + + public void setScraperConfigurationName(String scraperConfigurationName) { + this.scraperConfigurationName = scraperConfigurationName; + } + + public LocalDateTime getEventDate() { + return eventDate; + } + + public void setEventDate(LocalDateTime eventDate) { + this.eventDate = eventDate; + } + + public String getEventTitle() { + return eventTitle; + } + + public void setEventTitle(String eventTitle) { + this.eventTitle = eventTitle; + } + + public String getMarker() { + return marker; + } + + public void setMarker(String marker) { + this.marker = marker; + } +} diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java new file mode 100644 index 0000000..f2daf4a --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java @@ -0,0 +1,45 @@ +package hub.event.scrapers.core.entityrepository; + +import hub.event.scrapers.core.LastScrapedEventMarker; +import hub.event.scrapers.core.LastScrapedEventMarkerRepository; +import org.springframework.stereotype.Repository; + +import java.time.LocalDateTime; +import java.util.Optional; + +@Repository +class LastScrapedEventMarkerEntityRepository implements LastScrapedEventMarkerRepository { + private final LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository; + + LastScrapedEventMarkerEntityRepository(LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository) { + this.lastScrapedEventMarkerEntityRepository = lastScrapedEventMarkerEntityRepository; + } + + public void store(LastScrapedEventMarker lastScrapedEventMarker) { + final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = mapToEntity(lastScrapedEventMarker); + lastScrapedEventMarkerEntityRepository.save(lastScrapedEventMarkerEntity); + } + + public Optional findByScraperConfigurationName(String configurationName) { + final Optional scraperConfigurationName = lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(configurationName); + return scraperConfigurationName.map(this::mapToMaker); + } + + private LastScrapedEventMarkerEntity mapToEntity(LastScrapedEventMarker lastScrapedEventMarker) { + final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); + lastScrapedEventMarkerEntity.setMarker(lastScrapedEventMarker.marker()); + lastScrapedEventMarkerEntity.setEventTitle(lastScrapedEventMarker.eventTitle()); + lastScrapedEventMarkerEntity.setEventDate(lastScrapedEventMarker.eventDate()); + lastScrapedEventMarkerEntity.setScraperConfigurationName(lastScrapedEventMarker.scraperConfigurationName()); + return lastScrapedEventMarkerEntity; + } + + private LastScrapedEventMarker mapToMaker(LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity) { + final String marker = lastScrapedEventMarkerEntity.getMarker(); + final LocalDateTime date = lastScrapedEventMarkerEntity.getEventDate(); + final String title = lastScrapedEventMarkerEntity.getEventTitle(); + final String configurationName = lastScrapedEventMarkerEntity.getScraperConfigurationName(); + + return new LastScrapedEventMarker(configurationName, date, title, marker); + } +} diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java new file mode 100644 index 0000000..118da53 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java @@ -0,0 +1,10 @@ +package hub.event.scrapers.core.entityrepository; + +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +interface LastScrapedEventMarkerJpaRepository extends JpaRepository { + + Optional findByScraperConfigurationName(String configurationName); +} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java new file mode 100644 index 0000000..e526dc6 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java @@ -0,0 +1,4 @@ +package hub.event.scrapers.core.exceptions; + +public class EventDateEndDateBeforeStartDateException extends Exception { +} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java new file mode 100644 index 0000000..006af8a --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java @@ -0,0 +1,4 @@ +package hub.event.scrapers.core.exceptions; + +public class EventDateEndTimeBeforeStartTimeException extends Exception { +} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java new file mode 100644 index 0000000..643e00a --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java @@ -0,0 +1,4 @@ +package hub.event.scrapers.core.exceptions; + +public class EventDateInPastException extends Exception { +} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java b/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java new file mode 100644 index 0000000..4cd407b --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java @@ -0,0 +1,4 @@ +package hub.event.scrapers.core.exceptions; + +public class ScraperConfigurationByNameNotExists extends Exception{ +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java new file mode 100644 index 0000000..ed21e60 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java @@ -0,0 +1,6 @@ +package hub.event.scrapers.core.runlog; + +import java.time.LocalDateTime; + +public record ScraperRunErrorLog(String configurationName, LocalDateTime time, String errorCode, String description) { +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java new file mode 100644 index 0000000..48ca6de --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java @@ -0,0 +1,21 @@ +package hub.event.scrapers.core.runlog; + +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; + +@Service +public class ScraperRunLogFacade { + private final ScraperRunLogRepository scraperRunLogRepository; + + public ScraperRunLogFacade(ScraperRunLogRepository scraperRunLogRepository) { + this.scraperRunLogRepository = scraperRunLogRepository; + } + + + public void logError(String configurationName, LocalDateTime time, String errorCode, String description) { + } + + public void logStatus(String configurationName, LocalDateTime startTime, LocalDateTime finishTime, Integer scannedEventCount, Integer errorCount) { + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java new file mode 100644 index 0000000..aefdcbe --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java @@ -0,0 +1,8 @@ +package hub.event.scrapers.core.runlog; + +interface ScraperRunLogRepository { + + void save(ScraperRunStatusLog scraperRunStatusLog); + + void save (ScraperRunErrorLog scraperRunError); +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java new file mode 100644 index 0000000..0636fac --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core.runlog; + +import java.time.LocalDateTime; + +public record ScraperRunStatusLog(String configurationName, LocalDateTime startTime, LocalDateTime finishTime, + Integer scannedEventCount, Integer errorCount) { +} diff --git a/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java b/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java index fa175cd..0edbf35 100644 --- a/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java +++ b/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java @@ -1,6 +1,19 @@ package hub.event.scrapers.ebilet; -import hub.event.scrapers.Scrappable; +import hub.event.scrapers.core.PageScraperPort; +import hub.event.scrapers.core.ScrapedEvent; -class EbiletScraper implements Scrappable { +import java.util.Collection; +import java.util.Collections; + +class EbiletScraper implements PageScraperPort { + @Override + public String configurationName() { + return null; + } + + @Override + public Collection scrap() { + return Collections.emptyList(); + } } diff --git a/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java b/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java index 3942aab..87bb513 100644 --- a/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java +++ b/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java @@ -1,6 +1,20 @@ package hub.event.scrapers.empikbilet; -import hub.event.scrapers.Scrappable; +import hub.event.scrapers.core.PageScraperPort; +import hub.event.scrapers.core.ScrapedEvent; -class EmpikBiletScraper implements Scrappable { +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +class EmpikBiletScraper implements PageScraperPort { + @Override + public String configurationName() { + return null; + } + + @Override + public Collection scrap() { + return Collections.emptyList(); + } } diff --git a/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java b/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java index 724336a..869fe1d 100644 --- a/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java +++ b/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java @@ -1,7 +1,21 @@ package hub.event.scrapers.goingapp; -import hub.event.scrapers.Scrappable; +import hub.event.scrapers.core.PageScraperPort; +import hub.event.scrapers.core.ScrapedEvent; +import java.util.Collection; +import java.util.Collections; -class GoingAppScraper implements Scrappable { + +class GoingAppScraper implements PageScraperPort { + + @Override + public String configurationName() { + return null; + } + + @Override + public Collection scrap() { + return Collections.emptyList(); + } } diff --git a/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java b/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java index cda11f1..696501e 100644 --- a/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java +++ b/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java @@ -1,9 +1,22 @@ package hub.event.scrapers.kupbilecik; -import hub.event.scrapers.Scrappable; +import hub.event.scrapers.core.PageScraperPort; +import hub.event.scrapers.core.ScrapedEvent; +import java.util.Collection; +import java.util.Collections; -class KupBilecikScraper implements Scrappable { +class KupBilecikScraper implements PageScraperPort { + + @Override + public String configurationName() { + return null; + } + + @Override + public Collection scrap() { + return Collections.emptyList(); + } } diff --git a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java index ca8f821..461c8ed 100644 --- a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java +++ b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java @@ -1,6 +1,19 @@ package hub.event.scrapers.proanima; -import hub.event.scrapers.Scrappable; +import hub.event.scrapers.core.PageScraperPort; +import hub.event.scrapers.core.ScrapedEvent; -class ProanimaScraper implements Scrappable { +import java.util.Collection; +import java.util.Collections; + +class ProanimaScraper implements PageScraperPort { + @Override + public String configurationName() { + return null; + } + + @Override + public Collection scrap() { + return Collections.emptyList(); + } } diff --git a/src/main/java/hub/event/statistics/StatsService.java b/src/main/java/hub/event/statistics/StatsService.java index 5e4365d..9302f0e 100644 --- a/src/main/java/hub/event/statistics/StatsService.java +++ b/src/main/java/hub/event/statistics/StatsService.java @@ -1,8 +1,4 @@ package hub.event.statistics; -import hub.event.events.EventService; -import hub.event.scrapers.ScrapersService; -import hub.event.users.UserService; - public class StatsService { } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 037a898..1fcf242 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -19,4 +19,6 @@ spring.datasource.driver-class-name=org.h2.Driver spring.datasource.url=jdbc:h2:mem:devdb spring.datasource.username=sa spring.datasource.password=sa -spring.h2.console.enabled=true \ No newline at end of file +spring.h2.console.enabled=true + +scrapers.run.cron.expression=0 0 * * * \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java b/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java new file mode 100644 index 0000000..d7a8669 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java @@ -0,0 +1,88 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class ScrapedEventBuilderTest { + + @Test + void whenUseBuilderThenEventIsBuiltCorrectly() { + //given + final String title = "Long party on Normandy"; + final String sourceLink = "citadel://eden.news@human.colonies.gh/evens/123edkfke344"; + final String city = "Eden Prime"; + final String address = "Dark Rose 12"; + final String location = "Blue Bar"; + final String description = "example description"; + final Map metadata = Map.of("MetaKey1", "MetaValue1", "MetaKey2", "MetaValue2", "MetaKey3", "MetaValue3"); + final LocalDateTime scrapedTime = LocalDateTime.of(2022, 8, 21, 10, 0); + + final LocalDate startDate = LocalDate.of(2022, 10, 12); + final LocalTime startTime = LocalTime.of(14, 0); + final LocalDate endDate = LocalDate.of(2022, 10, 16); + final LocalTime endTime = LocalTime.of(22, 30); + + final SingleEventDateWithLocation singleEventDateWithLocation = SingleEventDateWithLocation.period(startDate, endDate, startTime, endTime, city, address, location); + + final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(startDate, startTime, city, address, location); + + //then + ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + .title(title) + .description(description) + .sourceLink(sourceLink) + .metadata("MetaKey1", "MetaValue1") + .metadata("MetaKey2", "MetaValue2") + .metadata("MetaKey3", "MetaValue3") + .date(singleEventDateWithLocation) + .scrapedTime(scrapedTime) + .build(); + + ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + .title(title) + .description(description) + .sourceLink(sourceLink) + .date(multipleEventDateWithLocations) + .scrapedTime(scrapedTime) + .build(); + + ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + .title(title) + .description(description) + .sourceLink(sourceLink) + .date(multipleEventDateWithLocations) + .date(singleEventDateWithLocation) + .scrapedTime(scrapedTime) + .build(); + + assertNotNull(scrapedEvent1); + assertNotNull(scrapedEvent2); + assertNotNull(scrapedEvent3); + + assertEquals(title, scrapedEvent1.title()); + assertEquals(description, scrapedEvent1.description()); + assertEquals(sourceLink, scrapedEvent1.sourceLink()); + assertEquals(metadata, scrapedEvent1.metadata()); + assertEquals(singleEventDateWithLocation, scrapedEvent1.singleEventDateWithLocation()); + assertNull(scrapedEvent1.multipleEventDateWithLocations()); + assertEquals(scrapedTime, scrapedEvent1.scrapedTime()); + assertFalse(scrapedEvent1.hasMultipleDateAndLocations()); + + assertEquals(multipleEventDateWithLocations, scrapedEvent2.multipleEventDateWithLocations()); + assertNull(scrapedEvent2.singleEventDateWithLocation()); + assertFalse(scrapedEvent2.hasMultipleDateAndLocations()); + + assertEquals(multipleEventDateWithLocations, scrapedEvent3.multipleEventDateWithLocations()); + assertNull(scrapedEvent3.singleEventDateWithLocation()); + assertFalse(scrapedEvent3.hasMultipleDateAndLocations()); + } + +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java new file mode 100644 index 0000000..d0a8e52 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java @@ -0,0 +1,78 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class ScraperFacadeTest { + @Mock + private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; + @Mock + private ScraperConfigRepository scraperConfigRepository; + @InjectMocks + private ScraperFacade scraperFacade; + + @Test + void whenActivateScraperThanNotExistThenCreateNewRecord() { + //given + final String scraperName = "sc1"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(false); + //then + scraperFacade.activateScraperByConfigurationName(scraperName); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository).create(scraperName); + verify(scraperConfigRepository, never()).activate(scraperName); + } + + @Test + void whenActivateScraperThanExistThenFinishSuccessfully() { + //given + final String scraperName = "sc4"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(true); + //then + scraperFacade.activateScraperByConfigurationName(scraperName); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository, never()).create(scraperName); + verify(scraperConfigRepository).activate(scraperName); + } + + @Test + void whenDeactivateNotExistsScraperThanThrowConfigNotExistsException() { + //given + final String scraperName = "sc2"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(false); + //then + assertThrows(ScraperConfigurationByNameNotExists.class, () -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository, never()).create(scraperName); + verify(scraperConfigRepository, never()).deactivate(scraperName); + } + + @Test + void whenDeactivateExistsScraperThanFinishSuccessfully() { + //given + final String scraperName = "sc3"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(true); + //then + assertDoesNotThrow(() -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository, never()).create(scraperName); + verify(scraperConfigRepository).deactivate(scraperName); + } +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java new file mode 100644 index 0000000..6e7661d --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java @@ -0,0 +1,335 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Collection; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class ScraperRunServiceTest { + + @Mock + private ScraperConfigRepository scraperConfigRepository; + @Mock + private List pageScrapers; + @Mock + private EventCandidateAnalyzer eventCandidateAnalyzer; + @Mock + private EventFacadeAdapter eventFacadeAdapter; + @Mock + private ProbablyDuplicatedEventCandidateRepository probablyDuplicatedEventCandidateRepository; + @InjectMocks + private ScraperRunService scraperRunService; + @Captor + private ArgumentCaptor> scrapedEventListCaptor; + private ArgumentCaptor> duplicatedEventCandidateListCaptor; + + @Test + void whenListContainsScraperWithoutConfigThenCreateNewConfig() { + //given + final PageScraperPort activeScraper = mock(PageScraperPort.class); + final PageScraperPort noConfigScraper1 = mock(PageScraperPort.class); + final PageScraperPort noConfigScraper2 = mock(PageScraperPort.class); + final PageScraperPort inactiveScraper = mock(PageScraperPort.class); + + final String activeScraperName = "active1"; + final String inactiveScraperName = "inactive2"; + final String noConfigScraper1Name = "no-config-1"; + final String noConfigScraper2Name = "no-config-2-Atr-45"; + + final ScraperConfig activeScraperConfig = new ScraperConfig(activeScraperName, true); + final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, false); + final Collection scraperConfigs = List.of(activeScraperConfig, inactiveScraperConfig); + + //when + when(activeScraper.configurationName()).thenReturn(activeScraperName); + when(noConfigScraper1.configurationName()).thenReturn(noConfigScraper1Name); + when(noConfigScraper2.configurationName()).thenReturn(noConfigScraper2Name); + when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); + when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper, noConfigScraper1, noConfigScraper2, inactiveScraper)); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + //then + scraperRunService.start(); + + verify(scraperConfigRepository).allScraperConfigs(); + verify(pageScrapers).stream(); + verify(scraperConfigRepository).create(noConfigScraper1Name); + verify(scraperConfigRepository).create(noConfigScraper2Name); + verify(scraperConfigRepository, never()).create(activeScraperName); + verify(scraperConfigRepository, never()).create(inactiveScraperName); + } + + @Test + void whenListContainsInactiveScraperThenSkipRunIt() { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + final PageScraperPort inactiveScraper = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + final String inactiveScraperName = "inactive2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); + final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, false); + final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config, inactiveScraperConfig); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); + + when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2, inactiveScraper)); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + //then + scraperRunService.start(); + + verify(scraperConfigRepository).allScraperConfigs(); + verify(pageScrapers).stream(); + verify(scraperConfigRepository, never()).create(anyString()); + verify(activeScraper1).scrap(); + verify(activeScraper2).scrap(); + verify(inactiveScraper, never()).scrap(); + } + + @Test + void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); + final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .build(); + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + .title("Title2") + .description("Description2") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Thessia", "Addres 2", "location 22")) + .build(); + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Eden Prime", "Addres 134", "location 166")) + .build(); + final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3); + + final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); + final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); + final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + + when(activeScraper1.scrap()).thenReturn(List.of(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(List.of(scrapedEvent2, scrapedEvent3)); + + when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2)); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + when(eventCandidateAnalyzer.analyze(scrapedEventList)) + .thenReturn(List.of(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + + //then + scraperRunService.start(); + + verify(activeScraper1).scrap(); + verify(activeScraper2).scrap(); + verify(eventCandidateAnalyzer).analyze(scrapedEventList); + verify(eventFacadeAdapter).saveAll(scrapedEventList); + verifyNoInteractions(probablyDuplicatedEventCandidateRepository); + } + + @Test + void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEventCandidate() { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); + final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .build(); + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + .title("Title2") + .description("Description2") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Thessia", "Addres 2", "location 22")) + .build(); + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .build(); + + final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3); + + final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); + final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); + final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); + + analyzedEventCandidate1.addDuplicateCandidate(scrapedEvent3); + analyzedEventCandidate3.addDuplicateCandidate(scrapedEvent1); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + + when(activeScraper1.scrap()).thenReturn(List.of(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(List.of(scrapedEvent2, scrapedEvent3)); + + when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2)); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + when(eventCandidateAnalyzer.analyze(scrapedEventList)) + .thenReturn(List.of(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + + //then + scraperRunService.start(); + + verify(activeScraper1).scrap(); + verify(activeScraper2).scrap(); + verify(eventCandidateAnalyzer).analyze(scrapedEventList); + verify(eventFacadeAdapter).saveAll(scrapedEventListCaptor.capture()); + verify(probablyDuplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); + + assertThat(scrapedEventListCaptor.getValue()) + .hasSize(1) + .contains(scrapedEvent2) + .doesNotContain(scrapedEvent1, scrapedEvent3); + + assertThat(duplicatedEventCandidateListCaptor.getValue()) + .hasSize(2) + .extracting( + ProbablyDuplicatedEventCandidate::getScrapedEvent, + eventCandidate -> eventCandidate.getDuplicatedCandidates().size(), + eventCandidate -> eventCandidate.getDuplicatedEvents().size() + ) + .contains( + tuple(scrapedEvent1, 1, 0), + tuple(scrapedEvent3, 1, 0) + ); + + } + + @Test + void whenScrapedEventContainsDuplicatedWithEventThenSaveAsProbablyDuplicatedEventCandidate() { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); + final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .build(); + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + .title("Title2") + .description("Description2") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Thessia", "Addres 2", "location 22")) + .build(); + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + .title("Title3") + .description("Description3") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Eden Prime", "Addres 133", "location 2")) + .build(); + + final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3); + + final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); + final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); + final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); + + final ExistsEvent existsEvent = new ExistsEvent(1012,"title","description", null, null); + + analyzedEventCandidate2.addDuplicateEvent(existsEvent); + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + + when(activeScraper1.scrap()).thenReturn(List.of(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(List.of(scrapedEvent2, scrapedEvent3)); + + when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2)); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + when(eventCandidateAnalyzer.analyze(scrapedEventList)) + .thenReturn(List.of(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + + //then + scraperRunService.start(); + + verify(activeScraper1).scrap(); + verify(activeScraper2).scrap(); + verify(eventCandidateAnalyzer).analyze(scrapedEventList); + verify(eventFacadeAdapter).saveAll(scrapedEventListCaptor.capture()); + verify(probablyDuplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); + + assertThat(scrapedEventListCaptor.getValue()) + .hasSize(2) + .contains(scrapedEvent1,scrapedEvent3) + .doesNotContain(scrapedEvent2); + + assertThat(duplicatedEventCandidateListCaptor.getValue()) + .hasSize(1) + .extracting( + ProbablyDuplicatedEventCandidate::getScrapedEvent, + eventCandidate -> eventCandidate.getDuplicatedCandidates().size(), + eventCandidate -> eventCandidate.getDuplicatedEvents().size(), + ProbablyDuplicatedEventCandidate::getScrapedEvent + ) + .contains( + tuple(scrapedEvent2, 0, 1, existsEvent) + ); + } +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java new file mode 100644 index 0000000..29bb183 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java @@ -0,0 +1,63 @@ +package hub.event.scrapers.core.datewithlocation; + +import hub.event.scrapers.core.exceptions.EventDateInPastException; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; +import java.time.LocalTime; + +import static org.assertj.core.api.Assertions.*; + +class MultipleEventDateWithLocationsTest { + + @Test + void whenBuildWithIncorrectInputThenThrows() { + final LocalDate date = LocalDate.of(2022, 7, 12); + final LocalDate dateInPast = LocalDate.of(2022, 7, 12); + final LocalTime time = LocalTime.of(14, 20); + final String city = "Thessia"; + final String address= "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertThatExceptionOfType(EventDateInPastException.class) + .isThrownBy(() -> MultipleEventDateWithLocations.create(dateInPast, time, city, address, locationName)); + + assertThatExceptionOfType(EventDateInPastException.class) + .isThrownBy(() -> MultipleEventDateWithLocations.create(dateInPast, time, city, address, locationName) + .add(date, time, city, address, locationName)); + } + + @Test + void whenCorrectInputThenBuildCorrectly() { + final LocalDate date1 = LocalDate.of(2022, 7, 12); + final LocalTime time1 = LocalTime.of(10, 20); + final String city1 = "Thessia"; + final LocalDate date2 = LocalDate.of(2022, 7, 14); + final LocalTime time2 = LocalTime.of(13, 0); + final String city2 = "Eden Prime"; + final LocalDate date3 = LocalDate.of(2022, 7, 18); + final LocalTime time3 = LocalTime.of(18, 30); + final String city3 = "Rannoch"; + final String address= "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + Assertions.assertThatNoException().isThrownBy(() -> { + MultipleEventDateWithLocations multipleDate = MultipleEventDateWithLocations.create(date1, time1, city1, address, locationName) + .add(date2, time2, city2, address, locationName) + .add(date3, time3, city3, address, locationName); + + assertThat(multipleDate).isNotNull(); + + assertThat(multipleDate.eventDateWithLocations()).isNotNull() + .hasSize(3) + .extracting(EventDateWithLocation::startDate, EventDateWithLocation::startTime, EventDateWithLocation::city, EventDateWithLocation::address, EventDateWithLocation::locationName) + .contains(tuple(date1, time1, city1, address, locationName), + tuple(date2, time2, city2, address, locationName), + tuple(date3, time3, city3, address, locationName) + ); + }); + + } + +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java new file mode 100644 index 0000000..77fbd65 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java @@ -0,0 +1,148 @@ +package hub.event.scrapers.core.datewithlocation; + +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import hub.event.scrapers.core.exceptions.EventDateEndDateBeforeStartDateException; +import hub.event.scrapers.core.exceptions.EventDateEndTimeBeforeStartTimeException; +import hub.event.scrapers.core.exceptions.EventDateInPastException; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; +import java.time.LocalTime; + +import static org.junit.jupiter.api.Assertions.*; + +class SingleEventDateWithLocationTest { + + @Test + void whenBuildSingleTypedDateWithIncorrectInputThenThrows() { + final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); + final LocalTime startTime = LocalTime.of(14, 20); + final LocalDate incorrectEndDate = LocalDate.of(2022, 7, 11); + final LocalDate correctEndDate = LocalDate.of(2022, 7, 12); + final LocalDate endDateInPast = LocalDate.of(2022, 7, 12); + final LocalTime incorrectEndTime = LocalTime.of(10, 10); + final LocalTime correctEndTime = LocalTime.of(20, 10); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertThrows(EventDateEndDateBeforeStartDateException.class, () -> SingleEventDateWithLocation.single(startDate, incorrectEndDate, startTime, correctEndTime, city, address, locationName)); + assertThrows(EventDateEndTimeBeforeStartTimeException.class, () -> SingleEventDateWithLocation.single(startDate, correctEndDate, startTime, incorrectEndTime, city, address, locationName)); + + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, correctEndDate, startTime, correctEndTime, city, address, locationName)); + + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, endDateInPast, startTime, correctEndTime, city, address, locationName)); + } + + @Test + void whenBuildPeriodTypedDateWithIncorrectInputThenThrows() { + final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); + final LocalTime startTime = LocalTime.of(14, 20); + final LocalDate incorrectEndDate = LocalDate.of(2022, 7, 11); + final LocalDate correctEndDate = LocalDate.of(2022, 7, 12); + final LocalDate endDateInPast = LocalDate.of(2022, 2, 12); + final LocalTime incorrectEndTime = LocalTime.of(10, 10); + final LocalTime correctEndTime = LocalTime.of(20, 10); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertThrows(EventDateEndDateBeforeStartDateException.class, () -> SingleEventDateWithLocation.period(startDate, incorrectEndDate, startTime, correctEndTime, city, address, locationName)); + assertThrows(EventDateEndTimeBeforeStartTimeException.class, () -> SingleEventDateWithLocation.period(startDate, correctEndDate, startTime, incorrectEndTime, city, address, locationName)); + + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, correctEndDate, startTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, correctEndDate, startTime, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, endDateInPast, startTime, correctEndTime, city, address, locationName)); + } + + @Test + void whenCorrectInputThenSingleTypedEventDateBuildCorrectly() { + final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalTime startTime = LocalTime.of(10, 20); + final LocalDate endDate = LocalDate.of(2022, 7, 12); + final LocalTime endTime = LocalTime.of(20, 10); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation singleDateContainsStartDateAndTime = SingleEventDateWithLocation.single(startDate, startTime, city, address, locationName); + assertNotNull(singleDateContainsStartDateAndTime); + + assertEquals(startDate, singleDateContainsStartDateAndTime.startDate()); + assertEquals(startTime, singleDateContainsStartDateAndTime.startTime()); + assertEquals(city, singleDateContainsStartDateAndTime.city()); + assertEquals(address, singleDateContainsStartDateAndTime.address()); + assertEquals(locationName, singleDateContainsStartDateAndTime.locationName()); + assertNull(singleDateContainsStartDateAndTime.endDate()); + assertNull(singleDateContainsStartDateAndTime.endTime()); + assertTrue(singleDateContainsStartDateAndTime.isSingleDate()); + assertFalse(singleDateContainsStartDateAndTime.isPeriodDate()); + + }); + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, endDate, startTime, endTime, city, address, locationName); + assertNotNull(fullEventDate); + + assertEquals(startDate, fullEventDate.startDate()); + assertEquals(startTime, fullEventDate.startTime()); + assertEquals(city, fullEventDate.city()); + assertEquals(endDate, fullEventDate.endDate()); + assertEquals(endTime, fullEventDate.endTime()); + assertEquals(address, fullEventDate.address()); + assertEquals(locationName, fullEventDate.locationName()); + assertTrue(fullEventDate.isSingleDate()); + assertFalse(fullEventDate.isPeriodDate()); + }); + + + } + + @Test + void whenCorrectInputThenPeriodTypedEventDateBuildCorrectly() { + final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalTime startTime = LocalTime.of(14, 0); + final LocalDate endDate = LocalDate.of(2022, 7, 17); + final LocalTime endTime = LocalTime.of(16, 30); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, endDate, startTime, city, address, locationName); + + assertNotNull(periodDateContainsStartDateAndTime); + + assertEquals(startDate, periodDateContainsStartDateAndTime.startDate()); + assertEquals(startTime, periodDateContainsStartDateAndTime.startTime()); + assertEquals(city, periodDateContainsStartDateAndTime.city()); + assertEquals(endDate, periodDateContainsStartDateAndTime.endDate()); + assertEquals(address, periodDateContainsStartDateAndTime.address()); + assertEquals(locationName, periodDateContainsStartDateAndTime.locationName()); + assertNull(periodDateContainsStartDateAndTime.endTime()); + assertTrue(periodDateContainsStartDateAndTime.isPeriodDate()); + assertFalse(periodDateContainsStartDateAndTime.isSingleDate()); + }); + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, endDate, startTime, endTime, city, address, locationName); + + assertNotNull(fullEventDate); + + assertEquals(startDate, fullEventDate.startDate()); + assertEquals(startTime, fullEventDate.startTime()); + assertEquals(city, fullEventDate.city()); + assertEquals(endDate, fullEventDate.endDate()); + assertEquals(endTime, fullEventDate.endTime()); + assertEquals(address, fullEventDate.address()); + assertEquals(locationName, fullEventDate.locationName()); + assertTrue(fullEventDate.isPeriodDate()); + assertFalse(fullEventDate.isSingleDate()); + }); + } + +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java b/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java new file mode 100644 index 0000000..04767c7 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java @@ -0,0 +1,103 @@ +package hub.event.scrapers.core.entityrepository; + +import hub.event.scrapers.core.LastScrapedEventMarker; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class LastScrapedEventMarkerRepositoryTest { + + @Mock + private LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository; + @InjectMocks + private LastScrapedEventMarkerEntityRepository lastScrapedEventMarkerRepository; + + @Captor + ArgumentCaptor lastScrapedEventMarkerEntityCaptor; + + @Test + void saveTest() { + //given + LastScrapedEventMarker eventMarker1 = new LastScrapedEventMarker("config1", LocalDateTime.now(), "title1", "marker1"); + LastScrapedEventMarker eventMarker2 = new LastScrapedEventMarker("config2", LocalDateTime.now(), "title2", "marker2"); + + + //then + lastScrapedEventMarkerRepository.store(eventMarker1); + lastScrapedEventMarkerRepository.store(eventMarker2); + + verify(lastScrapedEventMarkerEntityRepository, times(2)).save(lastScrapedEventMarkerEntityCaptor.capture()); + + assertThat(lastScrapedEventMarkerEntityCaptor.getAllValues()) + .extracting( + LastScrapedEventMarkerEntity::getScraperConfigurationName, + LastScrapedEventMarkerEntity::getEventDate, + LastScrapedEventMarkerEntity::getEventTitle, + LastScrapedEventMarkerEntity::getMarker + ).contains( + tuple(eventMarker1.scraperConfigurationName(), eventMarker1.eventDate(), eventMarker1.eventTitle(), eventMarker1.marker()), + tuple(eventMarker2.scraperConfigurationName(), eventMarker2.eventDate(), eventMarker2.eventTitle(), eventMarker2.marker()) + ); + + } + + @Test + void whenFindByNotExistsScraperConfigurationNameThenReturnEmptyMarker() { + //given + final String scraperConfigurationName = "not_exists_scraper"; + + //when + when(lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(scraperConfigurationName)) + .thenReturn(Optional.empty()); + + //then + final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName); + + assertThat(lastScrapedEventMarker).isEmpty(); + verify(lastScrapedEventMarkerEntityRepository).findByScraperConfigurationName(scraperConfigurationName); + } + + @Test + void whenFindByExistsScraperConfigurationNameThenReturnMarker() { + //given + final String scraperConfigurationName = "exists_scraper"; + final LocalDateTime localDateTime = LocalDateTime.now(); + + final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); + lastScrapedEventMarkerEntity.setMarker("maker1"); + lastScrapedEventMarkerEntity.setEventDate(localDateTime); + lastScrapedEventMarkerEntity.setEventTitle("title1"); + lastScrapedEventMarkerEntity.setScraperConfigurationName("exists_scraper"); + + //when + when(lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(scraperConfigurationName)) + .thenReturn(Optional.of(lastScrapedEventMarkerEntity)); + + //then + final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName); + + assertThat(lastScrapedEventMarker).isNotEmpty() + .get() + .extracting( + LastScrapedEventMarker::scraperConfigurationName, + LastScrapedEventMarker::eventDate, + LastScrapedEventMarker::eventTitle, + LastScrapedEventMarker::marker + ) + .contains(scraperConfigurationName, localDateTime, "title1", "maker1"); + + verify(lastScrapedEventMarkerEntityRepository).findByScraperConfigurationName(scraperConfigurationName); + } +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java b/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java new file mode 100644 index 0000000..d452c00 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java @@ -0,0 +1,80 @@ +package hub.event.scrapers.core.runlog; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class ScraperRunLogFacadeTest { + + @Mock + private ScraperRunLogRepository scraperRunLogRepository; + + @InjectMocks + private ScraperRunLogFacade scraperRunLogFacade; + + @Captor + private ArgumentCaptor scraperRunStatusLogArgumentCaptor; + @Captor + private ArgumentCaptor scraperRunErrorLogArgumentCaptor; + + @Test + void logError() { + //given + final String configurationName = "scraper1"; + final LocalDateTime time = LocalDateTime.now(); + final String errorCode = "ER1"; + final String description = "Error 1 server is down"; + + //then + scraperRunLogFacade.logError(configurationName, time, errorCode, description); + + verify(scraperRunLogRepository).save(scraperRunErrorLogArgumentCaptor.capture()); + + final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.capture(); + + assertNotNull(scraperRunErrorLog); + assertEquals(configurationName, scraperRunErrorLog.configurationName()); + assertEquals(time, scraperRunErrorLog.time()); + assertEquals(errorCode, scraperRunErrorLog.errorCode()); + assertEquals(description, scraperRunErrorLog.description()); + + } + + @Test + void logStatus() { + //given + final String configurationName = "scraper1"; + final LocalDateTime startTime = LocalDateTime.now(); + final LocalDateTime finishTime = LocalDateTime.now().plusMinutes(10); + final Integer scannedEventCount = 23; + final Integer errorCount = 1; + + + //then + scraperRunLogFacade.logStatus(configurationName, startTime, finishTime, scannedEventCount, errorCount); + + verify(scraperRunLogRepository).save(scraperRunStatusLogArgumentCaptor.capture()); + + final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.capture(); + + assertNotNull(scraperRunStatusLog); + assertEquals(configurationName, scraperRunStatusLog.configurationName()); + assertEquals(startTime, scraperRunStatusLog.startTime()); + assertEquals(finishTime, scraperRunStatusLog.finishTime()); + assertEquals(scannedEventCount, scraperRunStatusLog.scannedEventCount()); + assertEquals(errorCount, scraperRunStatusLog.errorCount()); + + } + +} \ No newline at end of file From f93c0b2bb88ff1a85839b6c6c368002d298db3d4 Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Wed, 24 Aug 2022 23:57:47 +0200 Subject: [PATCH 02/10] feat: 17 added implementation for ScraperRunService and all required class , remove stupid idea with use pager scrapers list mock from test --- .../scrapers/core/AnalyzedEventCandidate.java | 39 ++++-- .../core/DuplicatedEventCandidate.java | 29 +++++ .../DuplicatedEventCandidateRepository.java | 7 ++ .../ProbablyDuplicatedEventCandidate.java | 27 ---- ...blyDuplicatedEventCandidateRepository.java | 7 -- .../hub/event/scrapers/core/ScrapedEvent.java | 30 ++++- .../scrapers/core/ScrapedEventBuilder.java | 35 +++++- .../scrapers/core/ScrapedEventRepository.java | 4 + .../event/scrapers/core/ScraperFacade.java | 8 +- .../scrapers/core/ScraperRunService.java | 79 +++++++++++- .../EventDateWithLocation.java | 36 +++++- .../core/datewithlocation/EventLocation.java | 19 +-- .../MultipleEventDateWithLocations.java | 23 +++- .../SingleEventDateWithLocation.java | 55 +++++++-- ...ntDateEndDateBeforeStartDateException.java | 4 - ...dDateTimeBeforeStartDateTimeException.java | 9 ++ ...ntDateEndTimeBeforeStartTimeException.java | 4 - .../exceptions/EventDateInPastException.java | 10 ++ .../core/ScrapedEventBuilderTest.java | 15 ++- .../scrapers/core/ScraperRunServiceTest.java | 116 ++++++++++-------- .../MultipleEventDateWithLocationsTest.java | 14 +-- .../SingleEventDateWithLocationTest.java | 48 ++++---- 22 files changed, 426 insertions(+), 192 deletions(-) create mode 100644 src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java create mode 100644 src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java delete mode 100644 src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java create mode 100644 src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateTimeBeforeStartDateTimeException.java delete mode 100644 src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java diff --git a/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java b/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java index df38d56..534f17c 100644 --- a/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java +++ b/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java @@ -2,31 +2,48 @@ import java.util.ArrayList; import java.util.List; +import java.util.UUID; class AnalyzedEventCandidate { - private final List probablyDuplicateCandidates; - private final List probablyDuplicateEvents; + private final List duplicateWithCandidates; + private final List duplicateWithEvents; private final ScrapedEvent scrapedEvent; AnalyzedEventCandidate(ScrapedEvent scrapedEvent) { this.scrapedEvent = scrapedEvent; - this.probablyDuplicateCandidates = new ArrayList<>(); - this.probablyDuplicateEvents = new ArrayList<>(); + this.duplicateWithCandidates = new ArrayList<>(); + this.duplicateWithEvents = new ArrayList<>(); } - boolean isDuplicateWithCandidate() { - return !this.probablyDuplicateCandidates.isEmpty(); + ScrapedEvent scrapedEvent() { + return scrapedEvent; } void addDuplicateCandidate(ScrapedEvent scrapedEvent) { - this.probablyDuplicateCandidates.add(scrapedEvent); + duplicateWithCandidates.add(scrapedEvent); } - boolean isDuplicateWithEvent() { - return !this.probablyDuplicateEvents.isEmpty(); + void addDuplicateEvent(ExistsEvent existsEvent) { + duplicateWithEvents.add(existsEvent); } - void addDuplicateEvent(ExistsEvent existsEvent) { - this.probablyDuplicateEvents.add(existsEvent); + boolean isDuplicate() { + return !duplicateWithCandidates.isEmpty() || !duplicateWithEvents.isEmpty(); + } + + boolean isNotDuplicate() { + return duplicateWithCandidates.isEmpty() && duplicateWithEvents.isEmpty(); + } + + public List duplicateCandidateUUIDsList() { + return duplicateWithCandidates.stream() + .map(ScrapedEvent::uuid) + .toList(); + } + + public List duplicateEventIdList() { + return duplicateWithEvents.stream() + .map(ExistsEvent::eventId) + .toList(); } } diff --git a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java new file mode 100644 index 0000000..d2bb474 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java @@ -0,0 +1,29 @@ +package hub.event.scrapers.core; + +import java.util.List; +import java.util.UUID; + +class DuplicatedEventCandidate { + private ScrapedEvent scrapedEvent; + private List duplicatedCandidates; + private List duplicatedEvents; + + public DuplicatedEventCandidate(ScrapedEvent scrapedEvent, List duplicatedCandidates, List duplicatedEvents) { + this.scrapedEvent = scrapedEvent; + this.duplicatedCandidates = duplicatedCandidates; + this.duplicatedEvents = duplicatedEvents; + } + + + ScrapedEvent getScrapedEvent() { + return scrapedEvent; + } + + List getDuplicatedCandidates() { + return duplicatedCandidates; + } + + List getDuplicatedEvents() { + return duplicatedEvents; + } +} diff --git a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java new file mode 100644 index 0000000..030b246 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core; + +import java.util.List; + +interface DuplicatedEventCandidateRepository { + void saveAll(List candidates); +} diff --git a/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java b/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java deleted file mode 100644 index efdd2ca..0000000 --- a/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidate.java +++ /dev/null @@ -1,27 +0,0 @@ -package hub.event.scrapers.core; - -import java.util.List; -import java.util.UUID; - -class ProbablyDuplicatedEventCandidate { - private UUID uuid; - private ScrapedEvent scrapedEvent; - private List duplicatedCandidates; - private List duplicatedEvents; - - UUID getUuid() { - return uuid; - } - - ScrapedEvent getScrapedEvent() { - return scrapedEvent; - } - - List getDuplicatedCandidates() { - return duplicatedCandidates; - } - - List getDuplicatedEvents() { - return duplicatedEvents; - } -} diff --git a/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java b/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java deleted file mode 100644 index 65bddbb..0000000 --- a/src/main/java/hub/event/scrapers/core/ProbablyDuplicatedEventCandidateRepository.java +++ /dev/null @@ -1,7 +0,0 @@ -package hub.event.scrapers.core; - -import java.util.List; - -interface ProbablyDuplicatedEventCandidateRepository { - void saveAll(List candidates); -} diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java index 5315056..8c4126e 100644 --- a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java +++ b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java @@ -4,19 +4,36 @@ import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; import java.time.LocalDateTime; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; public class ScrapedEvent { + private UUID uuid; private Map metadata; private String title; private String description; private String sourceLink; + + private List types; private SingleEventDateWithLocation singleEventDateWithLocation; private MultipleEventDateWithLocations multipleEventDateWithLocations; private LocalDateTime scrapedTime; + ScrapedEvent(UUID uuid, String title, String description, String sourceLink, SingleEventDateWithLocation singleEventDateWithLocation, MultipleEventDateWithLocations multipleEventDateWithLocations, LocalDateTime scrapedTime, Map metadata, List types) { + this.uuid = uuid; + this.metadata = metadata; + this.title = title; + this.description = description; + this.sourceLink = sourceLink; + this.types = types; + this.singleEventDateWithLocation = singleEventDateWithLocation; + this.multipleEventDateWithLocations = multipleEventDateWithLocations; + this.scrapedTime = scrapedTime; + } + private ScrapedEvent() { } @@ -48,12 +65,19 @@ MultipleEventDateWithLocations multipleEventDateWithLocations() { return multipleEventDateWithLocations; } - public LocalDateTime scrapedTime() { + LocalDateTime scrapedTime() { return scrapedTime; } - public boolean hasMultipleDateAndLocations() { + boolean hasMultipleDateAndLocations() { return Objects.nonNull(multipleEventDateWithLocations); } -} + UUID uuid() { + return uuid; + } + + List types() { + return types; + } +} \ No newline at end of file diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java b/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java index 3b0277b..4796584 100644 --- a/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java +++ b/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java @@ -4,30 +4,36 @@ import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.Map; +import java.util.*; class ScrapedEventBuilder { private final Map metadata; + private List types; + private String title; + private String description; + private String sourceLink; + private SingleEventDateWithLocation singleEventDateWithLocation; + private MultipleEventDateWithLocations multipleEventDateWithLocations; + private LocalDateTime scrapedTime; ScrapedEventBuilder() { this.metadata = new HashMap<>(); - } - - public ScrapedEvent build() { - return null; + this.types = new ArrayList<>(); } public ScrapedEventBuilder title(String title) { + this.title = title; return this; } public ScrapedEventBuilder description(String description) { + this.description = description; return this; } public ScrapedEventBuilder sourceLink(String sourceLink) { + this.sourceLink = sourceLink; return this; } @@ -37,14 +43,31 @@ public ScrapedEventBuilder metadata(String key, String value) { } public ScrapedEventBuilder date(SingleEventDateWithLocation singleEventDateWithLocation) { + if (Objects.isNull(multipleEventDateWithLocations)) { + this.singleEventDateWithLocation = singleEventDateWithLocation; + } + return this; } public ScrapedEventBuilder date(MultipleEventDateWithLocations multipleEventDateWithLocations) { + this.singleEventDateWithLocation = null; + this.multipleEventDateWithLocations = multipleEventDateWithLocations; return this; } public ScrapedEventBuilder scrapedTime(LocalDateTime scrapedTime) { + this.scrapedTime = scrapedTime; + return this; + } + + public ScrapedEventBuilder type(String type) { + this.types.add(type); return this; } + + public ScrapedEvent build() { + final UUID eventUuid = UUID.randomUUID(); + return new ScrapedEvent(eventUuid, title, description, sourceLink, singleEventDateWithLocation, multipleEventDateWithLocations, scrapedTime, metadata, types); + } } diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java b/src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java new file mode 100644 index 0000000..a6c9981 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java @@ -0,0 +1,4 @@ +package hub.event.scrapers.core; + +public interface ScrapedEventRepository { +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperFacade.java b/src/main/java/hub/event/scrapers/core/ScraperFacade.java index ed08ca4..e482949 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperFacade.java +++ b/src/main/java/hub/event/scrapers/core/ScraperFacade.java @@ -11,12 +11,12 @@ public class ScraperFacade { private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; private final ScraperConfigRepository scraperConfigRepository; -private final ProbablyDuplicatedEventCandidateRepository probablyDuplicatedEventCandidateRepository; +private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; - public ScraperFacade(LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, ScraperConfigRepository scraperConfigRepository, ProbablyDuplicatedEventCandidateRepository probablyDuplicatedEventCandidateRepository) { + public ScraperFacade(LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, ScraperConfigRepository scraperConfigRepository, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository) { this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; this.scraperConfigRepository = scraperConfigRepository; - this.probablyDuplicatedEventCandidateRepository = probablyDuplicatedEventCandidateRepository; + this.duplicatedEventCandidateRepository = duplicatedEventCandidateRepository; } public void activateScraperByConfigurationName(String scraperName) { @@ -26,7 +26,7 @@ public void deactivateScraperByConfigurationName(String scraperName) { } //Maybe should be in separated module - public List getAllProbablyDuplicatedEventCandidates(){ + public List getAllProbablyDuplicatedEventCandidates(){ throw new NotImplementedException("getAllProbablyDuplicatedEventCandidates is not implemented. "); } diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java index 5732bd3..65aec57 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunService.java +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -5,23 +5,100 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; @Service @EnableScheduling class ScraperRunService { private final ScraperConfigRepository scraperConfigRepository; + private final EventCandidateAnalyzer eventCandidateAnalyzer; + private final EventFacadeAdapter eventFacadeAdapter; + + private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; private final List pageScrapers; @Autowired - ScraperRunService(ScraperConfigRepository scraperConfigRepository, List pageScrapers) { + ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventCandidateAnalyzer eventCandidateAnalyzer, EventFacadeAdapter eventFacadeAdapter, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository, List pageScrapers) { this.scraperConfigRepository = scraperConfigRepository; + this.eventCandidateAnalyzer = eventCandidateAnalyzer; + this.eventFacadeAdapter = eventFacadeAdapter; + this.duplicatedEventCandidateRepository = duplicatedEventCandidateRepository; this.pageScrapers = pageScrapers; } @Scheduled(cron = "${scrapers.run.cron.expression}") // @Scheduled(cron = "0 0 * * *") void start() { + final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); + + final List scrapersWithoutConfig = getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); + for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { + saveActiveConfigAndAppendToConfigList(pageScraperPort, scraperConfigs); + } + + final List pageScrapersToRun = getActiveScrapersThatShouldBeRun(scraperConfigs); + final List scrapedEventList = runScrapersForEvents(pageScrapersToRun); + final List analyzedEventCandidates = eventCandidateAnalyzer.analyze(scrapedEventList); + + final List notDuplicateScrapedEvents = extractNotDuplicateScrapedEvents(analyzedEventCandidates); + final List analyzedEventCandidateMarkedAsDuplicate = extractDuplicateScrapedEvents(analyzedEventCandidates); + + final List duplicatedEventCandidates = mapAnalyzedEventCandidateToDuplicatedEventCandidate(analyzedEventCandidateMarkedAsDuplicate); + + if (!notDuplicateScrapedEvents.isEmpty()) { + eventFacadeAdapter.saveAll(notDuplicateScrapedEvents); + } + + if (!duplicatedEventCandidates.isEmpty()) { + duplicatedEventCandidateRepository.saveAll(duplicatedEventCandidates); + } + } + + private List getScraperThatConfigNotFoundByConfigurationName(Collection scraperConfigs) { + final List availableScraperConfigByName = scraperConfigs.stream().map(ScraperConfig::configurationName).toList(); + + return pageScrapers.stream().filter(pageScraper -> !availableScraperConfigByName.contains(pageScraper.configurationName())).toList(); + } + + private void saveActiveConfigAndAppendToConfigList(PageScraperPort pageScraperPort, Collection scraperConfigs) { + scraperConfigRepository.create(pageScraperPort.configurationName()); + scraperConfigs.add(new ScraperConfig(pageScraperPort.configurationName(), true)); + } + + private List getActiveScrapersThatShouldBeRun(Collection scraperConfigs) { + final Map configStatusByScraperConfigurationNameMap = scraperConfigs.stream().collect(Collectors.toMap(ScraperConfig::configurationName, ScraperConfig::isActive)); + + return pageScrapers.stream().filter(scraper -> configStatusByScraperConfigurationNameMap.get(scraper.configurationName())).toList(); + } + + private List runScrapersForEvents(List pageScrapersToRun) { + return pageScrapersToRun.parallelStream().map(PageScraperPort::scrap).flatMap(Collection::stream).toList(); + } + + private List extractNotDuplicateScrapedEvents(List analyzedEventCandidates) { + return analyzedEventCandidates.stream().filter(AnalyzedEventCandidate::isNotDuplicate).map(AnalyzedEventCandidate::scrapedEvent).toList(); + } + + private List extractDuplicateScrapedEvents(List analyzedEventCandidates) { + return analyzedEventCandidates.stream().filter(AnalyzedEventCandidate::isDuplicate).toList(); + } + + private List mapAnalyzedEventCandidateToDuplicatedEventCandidate(List analyzedEventCandidateMarkedAsDuplicate) { + return analyzedEventCandidateMarkedAsDuplicate.stream().map(this::mapToDuplicatedEventCandidate).toList(); + } + + private DuplicatedEventCandidate mapToDuplicatedEventCandidate(AnalyzedEventCandidate analyzedEventCandidate) { + + final ScrapedEvent scrapedEvent = analyzedEventCandidate.scrapedEvent(); + final List duplicateCandidateUUIDsList = analyzedEventCandidate.duplicateCandidateUUIDsList(); + final List duplicateEventIdList = analyzedEventCandidate.duplicateEventIdList(); + + return new DuplicatedEventCandidate(scrapedEvent, duplicateCandidateUUIDsList, duplicateEventIdList); + } } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java index b6e70c8..25e2b42 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java @@ -1,15 +1,35 @@ package hub.event.scrapers.core.datewithlocation; +import hub.event.scrapers.core.exceptions.EventDateInPastException; + import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; class EventDateWithLocation { - private LocalDate startDate; - private LocalDate endDate; - private LocalTime startTime; - private LocalTime endTime; + private final LocalDate startDate; + private final LocalDate endDate; + private final LocalTime startTime; + private final LocalTime endTime; + + private final EventLocation eventLocation; + + public EventDateWithLocation(LocalDate startDate, LocalTime startTime,LocalDate endDate, LocalTime endTime, String city, String address, String locationName) { + this.startDate = startDate; + this.endDate = endDate; + this.startTime = startTime; + this.endTime = endTime; + this.eventLocation = new EventLocation(city, address, locationName); + } - private EventLocation eventLocation; + public EventDateWithLocation(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { + inputDateValidation(date); + this.startDate = date; + this.endDate = null; + this.startTime = time; + this.endTime = null; + this.eventLocation = new EventLocation(city, address, locationName); + } String city() { return eventLocation.city(); @@ -38,4 +58,10 @@ String address() { String locationName() { return eventLocation.name(); } + + private void inputDateValidation(LocalDate date) throws EventDateInPastException { + if(date.isBefore(LocalDate.now())){ + throw new EventDateInPastException(date); + } + } } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java index d8fc946..fbd13fa 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventLocation.java @@ -1,22 +1,5 @@ package hub.event.scrapers.core.datewithlocation; -class EventLocation { - private String city; - - private String address; - - private String name; - - public String city() { - return city; - } - - public String address() { - return address; - } - - public String name() { - return name; - } +record EventLocation(String city, String address, String name) { } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java index 039dfb0..32ce52a 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java @@ -1,21 +1,36 @@ package hub.event.scrapers.core.datewithlocation; +import hub.event.scrapers.core.exceptions.EventDateInPastException; + import java.time.LocalDate; import java.time.LocalTime; +import java.util.ArrayList; import java.util.Collection; public class MultipleEventDateWithLocations { private Collection eventDateWithLocations; - public static MultipleEventDateWithLocations create(LocalDate date, LocalTime time, String city, String address, String locationName) { - return null; + private MultipleEventDateWithLocations() { + eventDateWithLocations = new ArrayList<>(); + } + + private MultipleEventDateWithLocations(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { + this(); + final EventDateWithLocation eventDateWithLocation = new EventDateWithLocation(date, time, city, address, locationName); + this.eventDateWithLocations.add(eventDateWithLocation); + } + + public static MultipleEventDateWithLocations create(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { + return new MultipleEventDateWithLocations(date, time, city, address, locationName); } - public MultipleEventDateWithLocations add(LocalDate date, LocalTime time, String city, String address, String locationName) { + public MultipleEventDateWithLocations add(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { + final EventDateWithLocation eventDateWithLocation = new EventDateWithLocation(date, time, city, address, locationName); + this.eventDateWithLocations.add(eventDateWithLocation); return this; } - Collection eventDateWithLocations(){ + Collection eventDateWithLocations() { return this.eventDateWithLocations; } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java index d578e42..081707e 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java @@ -1,6 +1,10 @@ package hub.event.scrapers.core.datewithlocation; +import hub.event.scrapers.core.exceptions.EventDateEndDateTimeBeforeStartDateTimeException; +import hub.event.scrapers.core.exceptions.EventDateInPastException; + import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; public class SingleEventDateWithLocation { @@ -11,20 +15,29 @@ private SingleEventDateWithLocation() { } - public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, String city, String address, String locationName) { - return null; + private SingleEventDateWithLocation(EventDateType eventDateType, LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, String city, String address, String locationName) { + this.eventDateType = eventDateType; + this.eventDateWithLocation = new EventDateWithLocation(startDate, startTime, endDate, endTime, city, address, locationName); + } + + public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, String city, String address, String locationName) throws EventDateInPastException { + validateStartDateTime(startDate,startTime); + return new SingleEventDateWithLocation(EventDateType.SINGLE, startDate, startTime, null, null, city, address, locationName); } - public static SingleEventDateWithLocation single(LocalDate startDate, LocalDate endDate, LocalTime startTime, LocalTime endTime, String city, String address, String locationName) { - return null; + public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + validateStartDateTimeAndEndDateTime(startDate, startTime, endDate, endTime); + return new SingleEventDateWithLocation(EventDateType.SINGLE, startDate, startTime, endDate, endTime, city, address, locationName); } - public static SingleEventDateWithLocation period(LocalDate startDate, LocalDate endDate, LocalTime startTime, String city, String address, String locationName) { - return null; + public static SingleEventDateWithLocation period(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + validateStartDateTimeAndEndDateTime(startDate,startTime,endDate,endTime); + return new SingleEventDateWithLocation(EventDateType.PERIOD, startDate, startTime, endDate, endTime, city, address, locationName); } - public static SingleEventDateWithLocation period(LocalDate startDate, LocalDate endDate, LocalTime startTime, LocalTime endTime, String city, String address, String locationName) { - return null; + public static SingleEventDateWithLocation period(LocalDate startDate, LocalTime startTime, LocalDate endDate, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + validateStartDateTimeAndEndDateTime(startDate,startTime,endDate,startTime); + return new SingleEventDateWithLocation(EventDateType.PERIOD, startDate, startTime, endDate, null, city, address, locationName); } LocalDate startDate() { @@ -64,4 +77,30 @@ String locationName() { return eventDateWithLocation.locationName(); } + private static void validateStartDateTimeAndEndDateTime(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + final LocalDateTime startLocalDateTime = LocalDateTime.of(startDate, startTime); + final LocalDateTime endLocalDateTime = LocalDateTime.of(endDate, endTime); + final LocalDateTime now = LocalDateTime.now(); + + if(startLocalDateTime.isBefore(now)){ + throw new EventDateInPastException(startLocalDateTime); + } + if(endLocalDateTime.isBefore(now)){ + throw new EventDateInPastException(endLocalDateTime); + } + + if(endLocalDateTime.isBefore(startLocalDateTime)){ + throw new EventDateEndDateTimeBeforeStartDateTimeException(startLocalDateTime, endLocalDateTime); + } + } + + private static void validateStartDateTime(LocalDate startDate, LocalTime startTime) throws EventDateInPastException { + final LocalDateTime startLocalDateTime = LocalDateTime.of(startDate, startTime); + final LocalDateTime now = LocalDateTime.now(); + + if(startLocalDateTime.isBefore(now)){ + throw new EventDateInPastException(startLocalDateTime); + } + + } } diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java deleted file mode 100644 index e526dc6..0000000 --- a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateBeforeStartDateException.java +++ /dev/null @@ -1,4 +0,0 @@ -package hub.event.scrapers.core.exceptions; - -public class EventDateEndDateBeforeStartDateException extends Exception { -} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateTimeBeforeStartDateTimeException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateTimeBeforeStartDateTimeException.java new file mode 100644 index 0000000..990ded2 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndDateTimeBeforeStartDateTimeException.java @@ -0,0 +1,9 @@ +package hub.event.scrapers.core.exceptions; + +import java.time.LocalDateTime; + +public class EventDateEndDateTimeBeforeStartDateTimeException extends Exception { + public EventDateEndDateTimeBeforeStartDateTimeException(LocalDateTime startDate, LocalDateTime endDate) { + super(String.format("End date = %s is before start date = %s", endDate, startDate)); + } +} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java deleted file mode 100644 index 006af8a..0000000 --- a/src/main/java/hub/event/scrapers/core/exceptions/EventDateEndTimeBeforeStartTimeException.java +++ /dev/null @@ -1,4 +0,0 @@ -package hub.event.scrapers.core.exceptions; - -public class EventDateEndTimeBeforeStartTimeException extends Exception { -} diff --git a/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java b/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java index 643e00a..cacf1ce 100644 --- a/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java +++ b/src/main/java/hub/event/scrapers/core/exceptions/EventDateInPastException.java @@ -1,4 +1,14 @@ package hub.event.scrapers.core.exceptions; +import java.time.LocalDate; +import java.time.LocalDateTime; + public class EventDateInPastException extends Exception { + public EventDateInPastException(LocalDate date) { + super(String.format("Past event date = %s is not allowed, only time travelers can come to it ", date)); + } + + public EventDateInPastException(LocalDateTime incorrectLocalDateTime) { + super(String.format("Past event date = %s is not allowed, only time travelers can come to it ", incorrectLocalDateTime)); + } } diff --git a/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java b/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java index d7a8669..a2fb9cd 100644 --- a/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java +++ b/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java @@ -2,6 +2,8 @@ import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import hub.event.scrapers.core.exceptions.EventDateEndDateTimeBeforeStartDateTimeException; +import hub.event.scrapers.core.exceptions.EventDateInPastException; import org.junit.jupiter.api.Test; import java.time.LocalDate; @@ -14,7 +16,7 @@ class ScrapedEventBuilderTest { @Test - void whenUseBuilderThenEventIsBuiltCorrectly() { + void whenUseBuilderThenEventIsBuiltCorrectly() throws EventDateInPastException, EventDateEndDateTimeBeforeStartDateTimeException { //given final String title = "Long party on Normandy"; final String sourceLink = "citadel://eden.news@human.colonies.gh/evens/123edkfke344"; @@ -30,7 +32,7 @@ void whenUseBuilderThenEventIsBuiltCorrectly() { final LocalDate endDate = LocalDate.of(2022, 10, 16); final LocalTime endTime = LocalTime.of(22, 30); - final SingleEventDateWithLocation singleEventDateWithLocation = SingleEventDateWithLocation.period(startDate, endDate, startTime, endTime, city, address, location); + final SingleEventDateWithLocation singleEventDateWithLocation = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, city, address, location); final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(startDate, startTime, city, address, location); @@ -42,6 +44,8 @@ void whenUseBuilderThenEventIsBuiltCorrectly() { .metadata("MetaKey1", "MetaValue1") .metadata("MetaKey2", "MetaValue2") .metadata("MetaKey3", "MetaValue3") + .type("Inline Skating") + .type("Skating Workshops") .date(singleEventDateWithLocation) .scrapedTime(scrapedTime) .build(); @@ -71,6 +75,9 @@ void whenUseBuilderThenEventIsBuiltCorrectly() { assertEquals(description, scrapedEvent1.description()); assertEquals(sourceLink, scrapedEvent1.sourceLink()); assertEquals(metadata, scrapedEvent1.metadata()); + assertEquals(2, scrapedEvent1.types().size()); + assertTrue(scrapedEvent1.types().contains("Inline Skating")); + assertTrue(scrapedEvent1.types().contains("Skating Workshops")); assertEquals(singleEventDateWithLocation, scrapedEvent1.singleEventDateWithLocation()); assertNull(scrapedEvent1.multipleEventDateWithLocations()); assertEquals(scrapedTime, scrapedEvent1.scrapedTime()); @@ -78,11 +85,11 @@ void whenUseBuilderThenEventIsBuiltCorrectly() { assertEquals(multipleEventDateWithLocations, scrapedEvent2.multipleEventDateWithLocations()); assertNull(scrapedEvent2.singleEventDateWithLocation()); - assertFalse(scrapedEvent2.hasMultipleDateAndLocations()); + assertTrue(scrapedEvent2.hasMultipleDateAndLocations()); assertEquals(multipleEventDateWithLocations, scrapedEvent3.multipleEventDateWithLocations()); assertNull(scrapedEvent3.singleEventDateWithLocation()); - assertFalse(scrapedEvent3.hasMultipleDateAndLocations()); + assertTrue(scrapedEvent3.hasMultipleDateAndLocations()); } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java index 6e7661d..9b321ed 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java @@ -1,20 +1,22 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import hub.event.scrapers.core.exceptions.EventDateInPastException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.stream.Stream; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -26,18 +28,15 @@ class ScraperRunServiceTest { @Mock private ScraperConfigRepository scraperConfigRepository; @Mock - private List pageScrapers; - @Mock private EventCandidateAnalyzer eventCandidateAnalyzer; @Mock private EventFacadeAdapter eventFacadeAdapter; @Mock - private ProbablyDuplicatedEventCandidateRepository probablyDuplicatedEventCandidateRepository; - @InjectMocks - private ScraperRunService scraperRunService; + private DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; @Captor private ArgumentCaptor> scrapedEventListCaptor; - private ArgumentCaptor> duplicatedEventCandidateListCaptor; + @Captor + private ArgumentCaptor> duplicatedEventCandidateListCaptor; @Test void whenListContainsScraperWithoutConfigThenCreateNewConfig() { @@ -47,6 +46,7 @@ void whenListContainsScraperWithoutConfigThenCreateNewConfig() { final PageScraperPort noConfigScraper2 = mock(PageScraperPort.class); final PageScraperPort inactiveScraper = mock(PageScraperPort.class); + final String activeScraperName = "active1"; final String inactiveScraperName = "inactive2"; final String noConfigScraper1Name = "no-config-1"; @@ -54,21 +54,22 @@ void whenListContainsScraperWithoutConfigThenCreateNewConfig() { final ScraperConfig activeScraperConfig = new ScraperConfig(activeScraperName, true); final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, false); - final Collection scraperConfigs = List.of(activeScraperConfig, inactiveScraperConfig); + + final List pageScrapers = Arrays.asList(activeScraper, noConfigScraper1, noConfigScraper2, inactiveScraper); + final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraperConfig, inactiveScraperConfig)); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); //when when(activeScraper.configurationName()).thenReturn(activeScraperName); when(noConfigScraper1.configurationName()).thenReturn(noConfigScraper1Name); when(noConfigScraper2.configurationName()).thenReturn(noConfigScraper2Name); when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); - when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper, noConfigScraper1, noConfigScraper2, inactiveScraper)); when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); //then scraperRunService.start(); verify(scraperConfigRepository).allScraperConfigs(); - verify(pageScrapers).stream(); verify(scraperConfigRepository).create(noConfigScraper1Name); verify(scraperConfigRepository).create(noConfigScraper2Name); verify(scraperConfigRepository, never()).create(activeScraperName); @@ -89,21 +90,22 @@ void whenListContainsInactiveScraperThenSkipRunIt() { final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, false); - final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config, inactiveScraperConfig); + + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2, inactiveScraper); + final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config, inactiveScraperConfig)); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); when(activeScraper2.configurationName()).thenReturn(activeScraperName2); when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); - when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2, inactiveScraper)); when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); //then scraperRunService.start(); verify(scraperConfigRepository).allScraperConfigs(); - verify(pageScrapers).stream(); verify(scraperConfigRepository, never()).create(anyString()); verify(activeScraper1).scrap(); verify(activeScraper2).scrap(); @@ -111,7 +113,7 @@ void whenListContainsInactiveScraperThenSkipRunIt() { } @Test - void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() { + void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() throws EventDateInPastException { //given final PageScraperPort activeScraper1 = mock(PageScraperPort.class); final PageScraperPort activeScraper2 = mock(PageScraperPort.class); @@ -121,44 +123,46 @@ void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() { final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config); + final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() .title("Title1") .description("Description1") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() .title("Title2") .description("Description2") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Thessia", "Addres 2", "location 22")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) .build(); final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() .title("Title1") .description("Description1") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Eden Prime", "Addres 134", "location 166")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Eden Prime", "Addres 134", "location 166")) .build(); - final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3); + final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - when(activeScraper1.scrap()).thenReturn(List.of(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(List.of(scrapedEvent2, scrapedEvent3)); + when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); - when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2)); when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(List.of(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); //then scraperRunService.start(); @@ -167,11 +171,11 @@ void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() { verify(activeScraper2).scrap(); verify(eventCandidateAnalyzer).analyze(scrapedEventList); verify(eventFacadeAdapter).saveAll(scrapedEventList); - verifyNoInteractions(probablyDuplicatedEventCandidateRepository); + verifyNoInteractions(duplicatedEventCandidateRepository); } @Test - void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEventCandidate() { + void whenScrapedEventContainsDuplicatedCandidateThenSaveAsDuplicatedEventCandidate() throws EventDateInPastException { //given final PageScraperPort activeScraper1 = mock(PageScraperPort.class); final PageScraperPort activeScraper2 = mock(PageScraperPort.class); @@ -181,28 +185,28 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEven final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config); + final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() .title("Title1") .description("Description1") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() .title("Title2") .description("Description2") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Thessia", "Addres 2", "location 22")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) .build(); final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() .title("Title1") .description("Description1") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); - final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3); + final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); @@ -211,18 +215,20 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEven analyzedEventCandidate1.addDuplicateCandidate(scrapedEvent3); analyzedEventCandidate3.addDuplicateCandidate(scrapedEvent1); + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - when(activeScraper1.scrap()).thenReturn(List.of(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(List.of(scrapedEvent2, scrapedEvent3)); + when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); - when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2)); when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(List.of(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); //then scraperRunService.start(); @@ -231,7 +237,7 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEven verify(activeScraper2).scrap(); verify(eventCandidateAnalyzer).analyze(scrapedEventList); verify(eventFacadeAdapter).saveAll(scrapedEventListCaptor.capture()); - verify(probablyDuplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); + verify(duplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); assertThat(scrapedEventListCaptor.getValue()) .hasSize(1) @@ -241,7 +247,7 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEven assertThat(duplicatedEventCandidateListCaptor.getValue()) .hasSize(2) .extracting( - ProbablyDuplicatedEventCandidate::getScrapedEvent, + DuplicatedEventCandidate::getScrapedEvent, eventCandidate -> eventCandidate.getDuplicatedCandidates().size(), eventCandidate -> eventCandidate.getDuplicatedEvents().size() ) @@ -253,7 +259,7 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsProbablyDuplicatedEven } @Test - void whenScrapedEventContainsDuplicatedWithEventThenSaveAsProbablyDuplicatedEventCandidate() { + void whenScrapedEventContainsDuplicatedWithEventThenSaveAsDuplicatedEventCandidate() throws EventDateInPastException { //given final PageScraperPort activeScraper1 = mock(PageScraperPort.class); final PageScraperPort activeScraper2 = mock(PageScraperPort.class); @@ -263,48 +269,51 @@ void whenScrapedEventContainsDuplicatedWithEventThenSaveAsProbablyDuplicatedEven final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = List.of(activeScraper1Config, activeScraper2Config); + final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() .title("Title1") .description("Description1") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Palaven", "Addres 1", "location 2")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() .title("Title2") .description("Description2") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Thessia", "Addres 2", "location 22")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) .build(); final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() .title("Title3") .description("Description3") .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now(), LocalTime.now(), "Eden Prime", "Addres 133", "location 2")) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Eden Prime", "Addres 133", "location 2")) .build(); - final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3); + final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); - final ExistsEvent existsEvent = new ExistsEvent(1012,"title","description", null, null); + final ExistsEvent existsEvent = new ExistsEvent(1012, "title", "description", null, null); analyzedEventCandidate2.addDuplicateEvent(existsEvent); + + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - when(activeScraper1.scrap()).thenReturn(List.of(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(List.of(scrapedEvent2, scrapedEvent3)); + when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); - when(pageScrapers.stream()).thenReturn(Stream.of(activeScraper1, activeScraper2)); when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(List.of(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); //then scraperRunService.start(); @@ -313,23 +322,22 @@ void whenScrapedEventContainsDuplicatedWithEventThenSaveAsProbablyDuplicatedEven verify(activeScraper2).scrap(); verify(eventCandidateAnalyzer).analyze(scrapedEventList); verify(eventFacadeAdapter).saveAll(scrapedEventListCaptor.capture()); - verify(probablyDuplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); + verify(duplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); assertThat(scrapedEventListCaptor.getValue()) .hasSize(2) - .contains(scrapedEvent1,scrapedEvent3) + .contains(scrapedEvent1, scrapedEvent3) .doesNotContain(scrapedEvent2); assertThat(duplicatedEventCandidateListCaptor.getValue()) .hasSize(1) .extracting( - ProbablyDuplicatedEventCandidate::getScrapedEvent, + DuplicatedEventCandidate::getScrapedEvent, eventCandidate -> eventCandidate.getDuplicatedCandidates().size(), eventCandidate -> eventCandidate.getDuplicatedEvents().size(), - ProbablyDuplicatedEventCandidate::getScrapedEvent - ) + eventCandidate -> eventCandidate.getDuplicatedEvents().stream().map(String::valueOf).collect(Collectors.joining(","))) .contains( - tuple(scrapedEvent2, 0, 1, existsEvent) + tuple(scrapedEvent2, 0, 1, "1012") ); } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java index 29bb183..53bb1df 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java @@ -12,31 +12,31 @@ class MultipleEventDateWithLocationsTest { @Test - void whenBuildWithIncorrectInputThenThrows() { - final LocalDate date = LocalDate.of(2022, 7, 12); + void whenBuildWithIncorrectInputThenThrows() throws EventDateInPastException { + final LocalDate date = LocalDate.now().plusDays(10); final LocalDate dateInPast = LocalDate.of(2022, 7, 12); final LocalTime time = LocalTime.of(14, 20); final String city = "Thessia"; final String address= "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; + final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(date, time, city, address, locationName); assertThatExceptionOfType(EventDateInPastException.class) .isThrownBy(() -> MultipleEventDateWithLocations.create(dateInPast, time, city, address, locationName)); assertThatExceptionOfType(EventDateInPastException.class) - .isThrownBy(() -> MultipleEventDateWithLocations.create(dateInPast, time, city, address, locationName) - .add(date, time, city, address, locationName)); + .isThrownBy(() -> multipleEventDateWithLocations.add(dateInPast, time, city, address, locationName)); } @Test void whenCorrectInputThenBuildCorrectly() { - final LocalDate date1 = LocalDate.of(2022, 7, 12); + final LocalDate date1 = LocalDate.now().plusDays(2); final LocalTime time1 = LocalTime.of(10, 20); final String city1 = "Thessia"; - final LocalDate date2 = LocalDate.of(2022, 7, 14); + final LocalDate date2 = LocalDate.now().plusDays(4); final LocalTime time2 = LocalTime.of(13, 0); final String city2 = "Eden Prime"; - final LocalDate date3 = LocalDate.of(2022, 7, 18); + final LocalDate date3 = LocalDate.now().plusDays(2).plusDays(8); final LocalTime time3 = LocalTime.of(18, 30); final String city3 = "Rannoch"; final String address= "Nightmare Street 102/34"; diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java index 77fbd65..582fe5a 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java @@ -1,8 +1,6 @@ package hub.event.scrapers.core.datewithlocation; -import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; -import hub.event.scrapers.core.exceptions.EventDateEndDateBeforeStartDateException; -import hub.event.scrapers.core.exceptions.EventDateEndTimeBeforeStartTimeException; +import hub.event.scrapers.core.exceptions.EventDateEndDateTimeBeforeStartDateTimeException; import hub.event.scrapers.core.exceptions.EventDateInPastException; import org.junit.jupiter.api.Test; @@ -15,11 +13,11 @@ class SingleEventDateWithLocationTest { @Test void whenBuildSingleTypedDateWithIncorrectInputThenThrows() { - final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalDate startDate = LocalDate.now().plusDays(2); final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); final LocalTime startTime = LocalTime.of(14, 20); - final LocalDate incorrectEndDate = LocalDate.of(2022, 7, 11); - final LocalDate correctEndDate = LocalDate.of(2022, 7, 12); + final LocalDate incorrectEndDate = LocalDate.now().plusDays(1); + final LocalDate correctEndDate = LocalDate.now().plusDays(2); final LocalDate endDateInPast = LocalDate.of(2022, 7, 12); final LocalTime incorrectEndTime = LocalTime.of(10, 10); final LocalTime correctEndTime = LocalTime.of(20, 10); @@ -27,22 +25,22 @@ void whenBuildSingleTypedDateWithIncorrectInputThenThrows() { final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; - assertThrows(EventDateEndDateBeforeStartDateException.class, () -> SingleEventDateWithLocation.single(startDate, incorrectEndDate, startTime, correctEndTime, city, address, locationName)); - assertThrows(EventDateEndTimeBeforeStartTimeException.class, () -> SingleEventDateWithLocation.single(startDate, correctEndDate, startTime, incorrectEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, correctEndDate, startTime, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, endDateInPast, startTime, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); } @Test void whenBuildPeriodTypedDateWithIncorrectInputThenThrows() { - final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalDate startDate = LocalDate.now().plusDays(2); final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); final LocalTime startTime = LocalTime.of(14, 20); - final LocalDate incorrectEndDate = LocalDate.of(2022, 7, 11); - final LocalDate correctEndDate = LocalDate.of(2022, 7, 12); + final LocalDate incorrectEndDate = LocalDate.now().plusDays(1); + final LocalDate correctEndDate = LocalDate.now().plusDays(2); final LocalDate endDateInPast = LocalDate.of(2022, 2, 12); final LocalTime incorrectEndTime = LocalTime.of(10, 10); final LocalTime correctEndTime = LocalTime.of(20, 10); @@ -50,19 +48,19 @@ void whenBuildPeriodTypedDateWithIncorrectInputThenThrows() { final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; - assertThrows(EventDateEndDateBeforeStartDateException.class, () -> SingleEventDateWithLocation.period(startDate, incorrectEndDate, startTime, correctEndTime, city, address, locationName)); - assertThrows(EventDateEndTimeBeforeStartTimeException.class, () -> SingleEventDateWithLocation.period(startDate, correctEndDate, startTime, incorrectEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, correctEndDate, startTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, correctEndDate, startTime, correctEndTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, endDateInPast, startTime, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); } @Test void whenCorrectInputThenSingleTypedEventDateBuildCorrectly() { - final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalDate startDate = LocalDate.now().plusDays(2); final LocalTime startTime = LocalTime.of(10, 20); - final LocalDate endDate = LocalDate.of(2022, 7, 12); + final LocalDate endDate = LocalDate.now().plusDays(2); final LocalTime endTime = LocalTime.of(20, 10); final String city = "Thessia"; final String address = "Nightmare Street 102/34"; @@ -85,7 +83,7 @@ void whenCorrectInputThenSingleTypedEventDateBuildCorrectly() { }); assertDoesNotThrow(() -> { - SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, endDate, startTime, endTime, city, address, locationName); + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, startTime, endDate, endTime, city, address, locationName); assertNotNull(fullEventDate); assertEquals(startDate, fullEventDate.startDate()); @@ -104,16 +102,16 @@ void whenCorrectInputThenSingleTypedEventDateBuildCorrectly() { @Test void whenCorrectInputThenPeriodTypedEventDateBuildCorrectly() { - final LocalDate startDate = LocalDate.of(2022, 7, 12); + final LocalDate startDate = LocalDate.now().plusDays(2); final LocalTime startTime = LocalTime.of(14, 0); - final LocalDate endDate = LocalDate.of(2022, 7, 17); + final LocalDate endDate = LocalDate.now().plusDays(10); final LocalTime endTime = LocalTime.of(16, 30); final String city = "Thessia"; final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; assertDoesNotThrow(() -> { - SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, endDate, startTime, city, address, locationName); + SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, startTime, endDate, city, address, locationName); assertNotNull(periodDateContainsStartDateAndTime); @@ -129,7 +127,7 @@ void whenCorrectInputThenPeriodTypedEventDateBuildCorrectly() { }); assertDoesNotThrow(() -> { - SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, endDate, startTime, endTime, city, address, locationName); + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, city, address, locationName); assertNotNull(fullEventDate); From f60578cca974260a6662f2507573c623d5843e7b Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Wed, 31 Aug 2022 23:24:04 +0200 Subject: [PATCH 03/10] feat: 17 refactor few classes, added implementation for facades --- .../DuplicatedEventCandidateRepository.java | 5 + .../scrapers/core/LastScrapedEventMarker.java | 68 ++++- .../LastScrapedEventMarkerRepository.java | 8 +- .../hub/event/scrapers/core/ScrapedEvent.java | 73 ++++- .../scrapers/core/ScrapedEventBuilder.java | 73 ----- .../scrapers/core/ScrapedEventRepository.java | 4 - .../core/ScraperConfigRepository.java | 2 +- .../event/scrapers/core/ScraperFacade.java | 43 ++- .../scrapers/core/ScraperRunService.java | 53 +++- .../MultipleEventDateWithLocations.java | 2 +- .../LastScrapedEventMarkerEntity.java | 20 +- ...astScrapedEventMarkerEntityRepository.java | 28 +- .../ScraperConfigurationByNameNotExists.java | 5 +- .../core/runlog/ErrorLogSearchCommand.java | 40 +++ .../core/runlog/ScraperRunLogFacade.java | 14 + .../core/runlog/ScraperRunLogRepository.java | 6 + .../core/runlog/StatusLogSearchCommand.java | 123 ++++++++ .../scrapers/core/ScraperFacadeTest.java | 181 ++++++++---- .../scrapers/core/ScraperRunServiceTest.java | 81 +++++- .../SingleEventDateWithLocationTest.java | 264 +++++++++--------- .../LastScrapedEventMarkerRepositoryTest.java | 15 +- .../core/runlog/ScraperRunLogFacadeTest.java | 4 +- 22 files changed, 791 insertions(+), 321 deletions(-) delete mode 100644 src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java diff --git a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java index 030b246..20557fb 100644 --- a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java +++ b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java @@ -1,7 +1,12 @@ package hub.event.scrapers.core; import java.util.List; +import java.util.UUID; interface DuplicatedEventCandidateRepository { void saveAll(List candidates); + + List getAllDuplicatedEventCandidates(); + + boolean deleteDuplicatedEventCandidateByUuid(UUID eventCandidateUuid); } diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java index ba8a202..f926583 100644 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java @@ -1,6 +1,72 @@ package hub.event.scrapers.core; import java.time.LocalDateTime; +import java.util.Objects; + +public class LastScrapedEventMarker { + private final String scraperConfigurationName; + private final LocalDateTime runDateTime; + private final String eventTitle; + private final String marker; + private final boolean complete; + + public LastScrapedEventMarker(String scraperConfigurationName, LocalDateTime runDateTime, String eventTitle, String marker) { + this(scraperConfigurationName, runDateTime, eventTitle,marker,false); + } + + public LastScrapedEventMarker(String scraperConfigurationName, LocalDateTime runDateTime, String eventTitle, String marker, boolean complete) { + this.scraperConfigurationName = scraperConfigurationName; + this.runDateTime = runDateTime; + this.eventTitle = eventTitle; + this.marker = marker; + this.complete = complete; + } + + public String scraperConfigurationName() { + return scraperConfigurationName; + } + + public LocalDateTime runDateTime() { + return runDateTime; + } + + public String eventTitle() { + return eventTitle; + } + + public String marker() { + return marker; + } + + public boolean complete() { + return complete; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (LastScrapedEventMarker) obj; + return Objects.equals(this.scraperConfigurationName, that.scraperConfigurationName) && + Objects.equals(this.runDateTime, that.runDateTime) && + Objects.equals(this.eventTitle, that.eventTitle) && + Objects.equals(this.marker, that.marker) && + this.complete == that.complete; + } + + @Override + public int hashCode() { + return Objects.hash(scraperConfigurationName, runDateTime, eventTitle, marker, complete); + } + + @Override + public String toString() { + return "LastScrapedEventMarker[" + + "scraperConfigurationName=" + scraperConfigurationName + ", " + + "runDateTime=" + runDateTime + ", " + + "eventTitle=" + eventTitle + ", " + + "marker=" + marker + ", " + + "complete=" + complete + ']'; + } -public record LastScrapedEventMarker(String scraperConfigurationName, LocalDateTime eventDate , String eventTitle, String marker) { } diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java index 9ee441f..34bb2c2 100644 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java @@ -1,9 +1,15 @@ package hub.event.scrapers.core; +import java.util.List; import java.util.Optional; public interface LastScrapedEventMarkerRepository { void store(LastScrapedEventMarker lastScrapedEventMarker); - Optional findByScraperConfigurationName(String configurationName); + void drop(LastScrapedEventMarker lastScrapedEventMarker); + + void makeDraftActive(List configurationNameList); + + Optional findByScraperConfigurationName(String configurationName, boolean complete); + } diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java index 8c4126e..67e0ce2 100644 --- a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java +++ b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java @@ -4,10 +4,7 @@ import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; +import java.util.*; public class ScrapedEvent { @@ -22,7 +19,7 @@ public class ScrapedEvent { private MultipleEventDateWithLocations multipleEventDateWithLocations; private LocalDateTime scrapedTime; - ScrapedEvent(UUID uuid, String title, String description, String sourceLink, SingleEventDateWithLocation singleEventDateWithLocation, MultipleEventDateWithLocations multipleEventDateWithLocations, LocalDateTime scrapedTime, Map metadata, List types) { + private ScrapedEvent(UUID uuid, String title, String description, String sourceLink, SingleEventDateWithLocation singleEventDateWithLocation, MultipleEventDateWithLocations multipleEventDateWithLocations, LocalDateTime scrapedTime, Map metadata, List types) { this.uuid = uuid; this.metadata = metadata; this.title = title; @@ -80,4 +77,70 @@ UUID uuid() { List types() { return types; } + + static class ScrapedEventBuilder { + + private final Map metadata; + private final List types; + private String title; + private String description; + private String sourceLink; + private SingleEventDateWithLocation singleEventDateWithLocation; + private MultipleEventDateWithLocations multipleEventDateWithLocations; + private LocalDateTime scrapedTime; + + ScrapedEventBuilder() { + this.metadata = new HashMap<>(); + this.types = new ArrayList<>(); + } + + public ScrapedEventBuilder title(String title) { + this.title = title; + return this; + } + + public ScrapedEventBuilder description(String description) { + this.description = description; + return this; + } + + public ScrapedEventBuilder sourceLink(String sourceLink) { + this.sourceLink = sourceLink; + return this; + } + + public ScrapedEventBuilder metadata(String key, String value) { + this.metadata.put(key, value); + return this; + } + + public ScrapedEventBuilder date(SingleEventDateWithLocation singleEventDateWithLocation) { + if (Objects.isNull(multipleEventDateWithLocations)) { + this.singleEventDateWithLocation = singleEventDateWithLocation; + } + + return this; + } + + public ScrapedEventBuilder date(MultipleEventDateWithLocations multipleEventDateWithLocations) { + this.singleEventDateWithLocation = null; + this.multipleEventDateWithLocations = multipleEventDateWithLocations; + return this; + } + + public ScrapedEventBuilder scrapedTime(LocalDateTime scrapedTime) { + this.scrapedTime = scrapedTime; + return this; + } + + public ScrapedEventBuilder type(String type) { + this.types.add(type); + return this; + } + + public ScrapedEvent build() { + final UUID eventUuid = UUID.randomUUID(); + return new ScrapedEvent(eventUuid, title, description, sourceLink, singleEventDateWithLocation, multipleEventDateWithLocations, scrapedTime, metadata, types); + } + } } \ No newline at end of file diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java b/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java deleted file mode 100644 index 4796584..0000000 --- a/src/main/java/hub/event/scrapers/core/ScrapedEventBuilder.java +++ /dev/null @@ -1,73 +0,0 @@ -package hub.event.scrapers.core; - -import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; -import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; - -import java.time.LocalDateTime; -import java.util.*; - -class ScrapedEventBuilder { - - private final Map metadata; - private List types; - private String title; - private String description; - private String sourceLink; - private SingleEventDateWithLocation singleEventDateWithLocation; - private MultipleEventDateWithLocations multipleEventDateWithLocations; - private LocalDateTime scrapedTime; - - ScrapedEventBuilder() { - this.metadata = new HashMap<>(); - this.types = new ArrayList<>(); - } - - public ScrapedEventBuilder title(String title) { - this.title = title; - return this; - } - - public ScrapedEventBuilder description(String description) { - this.description = description; - return this; - } - - public ScrapedEventBuilder sourceLink(String sourceLink) { - this.sourceLink = sourceLink; - return this; - } - - public ScrapedEventBuilder metadata(String key, String value) { - this.metadata.put(key, value); - return this; - } - - public ScrapedEventBuilder date(SingleEventDateWithLocation singleEventDateWithLocation) { - if (Objects.isNull(multipleEventDateWithLocations)) { - this.singleEventDateWithLocation = singleEventDateWithLocation; - } - - return this; - } - - public ScrapedEventBuilder date(MultipleEventDateWithLocations multipleEventDateWithLocations) { - this.singleEventDateWithLocation = null; - this.multipleEventDateWithLocations = multipleEventDateWithLocations; - return this; - } - - public ScrapedEventBuilder scrapedTime(LocalDateTime scrapedTime) { - this.scrapedTime = scrapedTime; - return this; - } - - public ScrapedEventBuilder type(String type) { - this.types.add(type); - return this; - } - - public ScrapedEvent build() { - final UUID eventUuid = UUID.randomUUID(); - return new ScrapedEvent(eventUuid, title, description, sourceLink, singleEventDateWithLocation, multipleEventDateWithLocations, scrapedTime, metadata, types); - } -} diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java b/src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java deleted file mode 100644 index a6c9981..0000000 --- a/src/main/java/hub/event/scrapers/core/ScrapedEventRepository.java +++ /dev/null @@ -1,4 +0,0 @@ -package hub.event.scrapers.core; - -public interface ScrapedEventRepository { -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java index 5ecc26a..01080d2 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java @@ -6,7 +6,7 @@ interface ScraperConfigRepository { boolean exists(String scraperName); - void create(String scraperName); + void create(String scraperName, boolean activeState); void activate(String scraperConfigurationName); diff --git a/src/main/java/hub/event/scrapers/core/ScraperFacade.java b/src/main/java/hub/event/scrapers/core/ScraperFacade.java index e482949..017352c 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperFacade.java +++ b/src/main/java/hub/event/scrapers/core/ScraperFacade.java @@ -1,17 +1,20 @@ package hub.event.scrapers.core; +import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; import org.apache.commons.lang3.NotImplementedException; import org.springframework.stereotype.Service; +import java.time.LocalDateTime; import java.util.List; +import java.util.Optional; import java.util.UUID; @Service public class ScraperFacade { -private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; -private final ScraperConfigRepository scraperConfigRepository; + private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; + private final ScraperConfigRepository scraperConfigRepository; -private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; + private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; public ScraperFacade(LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, ScraperConfigRepository scraperConfigRepository, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository) { this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; @@ -20,19 +23,39 @@ public ScraperFacade(LastScrapedEventMarkerRepository lastScrapedEventMarkerRepo } public void activateScraperByConfigurationName(String scraperName) { + if (scraperConfigRepository.exists(scraperName)) { + scraperConfigRepository.activate(scraperName); + } else { + scraperConfigRepository.create(scraperName, true); + } } - public void deactivateScraperByConfigurationName(String scraperName) { + public void deactivateScraperByConfigurationName(String scraperName) throws ScraperConfigurationByNameNotExists { + if (!scraperConfigRepository.exists(scraperName)) { + throw new ScraperConfigurationByNameNotExists(scraperName); + } + + scraperConfigRepository.deactivate(scraperName); + } + + + public Optional lastScrapedEventMarkerByConfigurationName(String scraperConfigurationName) { + return lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, true); } - //Maybe should be in separated module - public List getAllProbablyDuplicatedEventCandidates(){ - throw new NotImplementedException("getAllProbablyDuplicatedEventCandidates is not implemented. "); + public void saveLastScrapedEventMarker(String scraperConfigurationName, LocalDateTime runDateTime, String eventTitle, String marker) { + final Optional savedScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, false); + savedScrapedEventMarker.ifPresent(lastScrapedEventMarkerRepository::drop); + + final LastScrapedEventMarker newScrapedEventMarkerToSave = new LastScrapedEventMarker(scraperConfigurationName, runDateTime, eventTitle, marker); + lastScrapedEventMarkerRepository.store(newScrapedEventMarkerToSave); } - //Maybe should be in separated module - public boolean deleteByProbablyDuplicatedEventCandidateUuid(UUID eventCandidateUuid){ - throw new NotImplementedException("deleteByProbablyDuplicatedEventCandidateUuid is not implemented. "); + public List getAllDuplicatedEventCandidates() { + return duplicatedEventCandidateRepository.getAllDuplicatedEventCandidates(); } + public boolean deleteDuplicatedEventCandidateByUuid(UUID eventCandidateUuid) { + return duplicatedEventCandidateRepository.deleteDuplicatedEventCandidateByUuid(eventCandidateUuid); + } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java index 65aec57..b56ca11 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunService.java +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -18,16 +18,17 @@ class ScraperRunService { private final ScraperConfigRepository scraperConfigRepository; private final EventCandidateAnalyzer eventCandidateAnalyzer; private final EventFacadeAdapter eventFacadeAdapter; - private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; + private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; private final List pageScrapers; @Autowired - ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventCandidateAnalyzer eventCandidateAnalyzer, EventFacadeAdapter eventFacadeAdapter, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository, List pageScrapers) { + ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventCandidateAnalyzer eventCandidateAnalyzer, EventFacadeAdapter eventFacadeAdapter, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository, LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, List pageScrapers) { this.scraperConfigRepository = scraperConfigRepository; this.eventCandidateAnalyzer = eventCandidateAnalyzer; this.eventFacadeAdapter = eventFacadeAdapter; this.duplicatedEventCandidateRepository = duplicatedEventCandidateRepository; + this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; this.pageScrapers = pageScrapers; } @@ -57,39 +58,62 @@ void start() { if (!duplicatedEventCandidates.isEmpty()) { duplicatedEventCandidateRepository.saveAll(duplicatedEventCandidates); } + + final List runScraperConfigurationsNames = getRunScraperConfigurationsNames(pageScrapersToRun); + lastScrapedEventMarkerRepository.makeDraftActive(runScraperConfigurationsNames); + } private List getScraperThatConfigNotFoundByConfigurationName(Collection scraperConfigs) { - final List availableScraperConfigByName = scraperConfigs.stream().map(ScraperConfig::configurationName).toList(); + final List availableScraperConfigByName = scraperConfigs.stream() + .map(ScraperConfig::configurationName) + .toList(); - return pageScrapers.stream().filter(pageScraper -> !availableScraperConfigByName.contains(pageScraper.configurationName())).toList(); + return pageScrapers.stream() + .filter(pageScraper -> !availableScraperConfigByName.contains(pageScraper.configurationName())) + .toList(); } private void saveActiveConfigAndAppendToConfigList(PageScraperPort pageScraperPort, Collection scraperConfigs) { - scraperConfigRepository.create(pageScraperPort.configurationName()); - scraperConfigs.add(new ScraperConfig(pageScraperPort.configurationName(), true)); + scraperConfigRepository.create(pageScraperPort.configurationName(), true); + final ScraperConfig scraperConfig = new ScraperConfig(pageScraperPort.configurationName(), true); + + scraperConfigs.add(scraperConfig); } private List getActiveScrapersThatShouldBeRun(Collection scraperConfigs) { - final Map configStatusByScraperConfigurationNameMap = scraperConfigs.stream().collect(Collectors.toMap(ScraperConfig::configurationName, ScraperConfig::isActive)); + final Map configStatusByScraperConfigurationNameMap = scraperConfigs.stream() + .collect(Collectors.toMap(ScraperConfig::configurationName, ScraperConfig::isActive)); - return pageScrapers.stream().filter(scraper -> configStatusByScraperConfigurationNameMap.get(scraper.configurationName())).toList(); + return pageScrapers.stream() + .filter(scraper -> configStatusByScraperConfigurationNameMap.get(scraper.configurationName())) + .toList(); } private List runScrapersForEvents(List pageScrapersToRun) { - return pageScrapersToRun.parallelStream().map(PageScraperPort::scrap).flatMap(Collection::stream).toList(); + return pageScrapersToRun.parallelStream() + .map(PageScraperPort::scrap) + .flatMap(Collection::stream) + .toList(); } private List extractNotDuplicateScrapedEvents(List analyzedEventCandidates) { - return analyzedEventCandidates.stream().filter(AnalyzedEventCandidate::isNotDuplicate).map(AnalyzedEventCandidate::scrapedEvent).toList(); + return analyzedEventCandidates.stream() + .filter(AnalyzedEventCandidate::isNotDuplicate) + .map(AnalyzedEventCandidate::scrapedEvent) + .toList(); } private List extractDuplicateScrapedEvents(List analyzedEventCandidates) { - return analyzedEventCandidates.stream().filter(AnalyzedEventCandidate::isDuplicate).toList(); + return analyzedEventCandidates.stream() + .filter(AnalyzedEventCandidate::isDuplicate) + .toList(); } private List mapAnalyzedEventCandidateToDuplicatedEventCandidate(List analyzedEventCandidateMarkedAsDuplicate) { - return analyzedEventCandidateMarkedAsDuplicate.stream().map(this::mapToDuplicatedEventCandidate).toList(); + return analyzedEventCandidateMarkedAsDuplicate.stream() + .map(this::mapToDuplicatedEventCandidate) + .toList(); } private DuplicatedEventCandidate mapToDuplicatedEventCandidate(AnalyzedEventCandidate analyzedEventCandidate) { @@ -99,6 +123,11 @@ private DuplicatedEventCandidate mapToDuplicatedEventCandidate(AnalyzedEventCand final List duplicateEventIdList = analyzedEventCandidate.duplicateEventIdList(); return new DuplicatedEventCandidate(scrapedEvent, duplicateCandidateUUIDsList, duplicateEventIdList); + } + private List getRunScraperConfigurationsNames(List pageScrapersToRun) { + return pageScrapersToRun.stream() + .map(PageScraperPort::configurationName) + .toList(); } } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java index 32ce52a..f12e4bd 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java @@ -8,7 +8,7 @@ import java.util.Collection; public class MultipleEventDateWithLocations { - private Collection eventDateWithLocations; + private final Collection eventDateWithLocations; private MultipleEventDateWithLocations() { eventDateWithLocations = new ArrayList<>(); diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java index 00f316f..6339efd 100644 --- a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java +++ b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java @@ -10,10 +10,12 @@ class LastScrapedEventMarkerEntity { @Id private String scraperConfigurationName; @Column(nullable = false) - private LocalDateTime eventDate; + private LocalDateTime runDateTime; private String eventTitle; @Column(nullable = false) private String marker; + @Column(nullable = false) + private Boolean complete; LastScrapedEventMarkerEntity() { } @@ -26,12 +28,12 @@ public void setScraperConfigurationName(String scraperConfigurationName) { this.scraperConfigurationName = scraperConfigurationName; } - public LocalDateTime getEventDate() { - return eventDate; + public LocalDateTime getRunDateTime() { + return runDateTime; } - public void setEventDate(LocalDateTime eventDate) { - this.eventDate = eventDate; + public void setRunDateTime(LocalDateTime runDateTime) { + this.runDateTime = runDateTime; } public String getEventTitle() { @@ -49,4 +51,12 @@ public String getMarker() { public void setMarker(String marker) { this.marker = marker; } + + public Boolean getComplete() { + return complete; + } + + public void setComplete(Boolean complete) { + this.complete = complete; + } } diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java index f2daf4a..8bbab1a 100644 --- a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java +++ b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java @@ -5,23 +5,34 @@ import org.springframework.stereotype.Repository; import java.time.LocalDateTime; +import java.util.List; import java.util.Optional; @Repository class LastScrapedEventMarkerEntityRepository implements LastScrapedEventMarkerRepository { - private final LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository; + private final LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerJpaRepository; LastScrapedEventMarkerEntityRepository(LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository) { - this.lastScrapedEventMarkerEntityRepository = lastScrapedEventMarkerEntityRepository; + this.lastScrapedEventMarkerJpaRepository = lastScrapedEventMarkerEntityRepository; } public void store(LastScrapedEventMarker lastScrapedEventMarker) { final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = mapToEntity(lastScrapedEventMarker); - lastScrapedEventMarkerEntityRepository.save(lastScrapedEventMarkerEntity); + lastScrapedEventMarkerJpaRepository.save(lastScrapedEventMarkerEntity); } - public Optional findByScraperConfigurationName(String configurationName) { - final Optional scraperConfigurationName = lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(configurationName); + @Override + public void drop(LastScrapedEventMarker lastScrapedEventMarker) { + + } + + @Override + public void makeDraftActive(List configurationNameList) { + + } + + public Optional findByScraperConfigurationName(String configurationName, boolean complete) { + final Optional scraperConfigurationName = lastScrapedEventMarkerJpaRepository.findByScraperConfigurationName(configurationName); return scraperConfigurationName.map(this::mapToMaker); } @@ -29,17 +40,18 @@ private LastScrapedEventMarkerEntity mapToEntity(LastScrapedEventMarker lastScra final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); lastScrapedEventMarkerEntity.setMarker(lastScrapedEventMarker.marker()); lastScrapedEventMarkerEntity.setEventTitle(lastScrapedEventMarker.eventTitle()); - lastScrapedEventMarkerEntity.setEventDate(lastScrapedEventMarker.eventDate()); + lastScrapedEventMarkerEntity.setRunDateTime(lastScrapedEventMarker.runDateTime()); lastScrapedEventMarkerEntity.setScraperConfigurationName(lastScrapedEventMarker.scraperConfigurationName()); return lastScrapedEventMarkerEntity; } private LastScrapedEventMarker mapToMaker(LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity) { final String marker = lastScrapedEventMarkerEntity.getMarker(); - final LocalDateTime date = lastScrapedEventMarkerEntity.getEventDate(); + final LocalDateTime date = lastScrapedEventMarkerEntity.getRunDateTime(); final String title = lastScrapedEventMarkerEntity.getEventTitle(); final String configurationName = lastScrapedEventMarkerEntity.getScraperConfigurationName(); + final Boolean isComplete = lastScrapedEventMarkerEntity.getComplete(); - return new LastScrapedEventMarker(configurationName, date, title, marker); + return new LastScrapedEventMarker(configurationName, date, title, marker, isComplete); } } diff --git a/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java b/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java index 4cd407b..60073f3 100644 --- a/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java +++ b/src/main/java/hub/event/scrapers/core/exceptions/ScraperConfigurationByNameNotExists.java @@ -1,4 +1,7 @@ package hub.event.scrapers.core.exceptions; -public class ScraperConfigurationByNameNotExists extends Exception{ +public class ScraperConfigurationByNameNotExists extends Exception { + public ScraperConfigurationByNameNotExists(String scraperConfigurationName) { + super(String.format("Scraper configuration by name %s not found", scraperConfigurationName)); + } } diff --git a/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java b/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java new file mode 100644 index 0000000..97653d9 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java @@ -0,0 +1,40 @@ +package hub.event.scrapers.core.runlog; + +import java.time.LocalDateTime; +import java.util.List; + +public class ErrorLogSearchCommand { + private final List configurationNames; + private final LocalDateTime fromDate; + private final LocalDateTime toDate; + private final List errorCodes; + private final String description; + + public ErrorLogSearchCommand(List configurationNames, LocalDateTime fromDate, LocalDateTime toDate, List errorCodes, String description) { + this.configurationNames = configurationNames; + this.fromDate = fromDate; + this.toDate = toDate; + this.errorCodes = errorCodes; + this.description = description; + } + + List configurationNames() { + return configurationNames; + } + + LocalDateTime fromDate() { + return fromDate; + } + + LocalDateTime toDate() { + return toDate; + } + + List errorCodes() { + return errorCodes; + } + + String description() { + return description; + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java index 48ca6de..0570c16 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java @@ -3,6 +3,7 @@ import org.springframework.stereotype.Service; import java.time.LocalDateTime; +import java.util.List; @Service public class ScraperRunLogFacade { @@ -14,8 +15,21 @@ public ScraperRunLogFacade(ScraperRunLogRepository scraperRunLogRepository) { public void logError(String configurationName, LocalDateTime time, String errorCode, String description) { + final ScraperRunErrorLog scraperRunErrorLog = new ScraperRunErrorLog(configurationName, time, errorCode, description); + scraperRunLogRepository.save(scraperRunErrorLog); } public void logStatus(String configurationName, LocalDateTime startTime, LocalDateTime finishTime, Integer scannedEventCount, Integer errorCount) { + final ScraperRunStatusLog scraperRunStatusLog = new ScraperRunStatusLog(configurationName, startTime, finishTime, scannedEventCount, errorCount); + scraperRunLogRepository.save(scraperRunStatusLog); } + + public List findAllErrorLog(ErrorLogSearchCommand errorLogSearchCommand) { + return scraperRunLogRepository.findAllErrorLog(errorLogSearchCommand); + } + public List findAllStatusLog(StatusLogSearchCommand statusLogSearchCommand) { + return scraperRunLogRepository.findAllStatusLog(statusLogSearchCommand); + } + + } diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java index aefdcbe..8c6265d 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java @@ -1,8 +1,14 @@ package hub.event.scrapers.core.runlog; +import java.util.List; + interface ScraperRunLogRepository { void save(ScraperRunStatusLog scraperRunStatusLog); void save (ScraperRunErrorLog scraperRunError); + + List findAllErrorLog(ErrorLogSearchCommand errorLogSearchCommand); + + List findAllStatusLog(StatusLogSearchCommand statusLogSearchCommand); } diff --git a/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java new file mode 100644 index 0000000..def9a67 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java @@ -0,0 +1,123 @@ +package hub.event.scrapers.core.runlog; + +import java.time.LocalDateTime; +import java.util.List; + +public class StatusLogSearchCommand { + private final List configurationNames; + private final LocalDateTime startTime; + private final LocalDateTime finishTime; + private final Integer scannedEventCount; + private final Integer errorCount; + private final Boolean hasErrors; + private final Integer limit; + private final Integer offset; + + private StatusLogSearchCommand(List configurationNames, LocalDateTime startTime, LocalDateTime finishTime, Integer scannedEventCount, Integer errorCount, Boolean hasErrors, Integer offset, Integer limit) { + this.configurationNames = configurationNames; + this.startTime = startTime; + this.finishTime = finishTime; + this.scannedEventCount = scannedEventCount; + this.errorCount = errorCount; + this.hasErrors = hasErrors; + this.limit = limit; + this.offset = offset; + } + + public StatusLogSearchCommandBuilder builder() { + return new StatusLogSearchCommandBuilder(); + } + + List configurationNames() { + return configurationNames; + } + + LocalDateTime startTime() { + return startTime; + } + + LocalDateTime finishTime() { + return finishTime; + } + + Integer scannedEventCount() { + return scannedEventCount; + } + + Integer errorCount() { + return errorCount; + } + + Boolean hasErrors() { + return hasErrors; + } + + Integer limit() { + return limit; + } + + Integer offset() { + return offset; + } + + class StatusLogSearchCommandBuilder { + List configurationNames; + LocalDateTime startTime; + LocalDateTime finishTime; + Integer scannedEventCount; + Integer errorCount; + Boolean hasErrors; + Integer limit; + Integer offset; + + StatusLogSearchCommandBuilder() { + this.limit = 10; + this.offset = 0; + } + + public StatusLogSearchCommandBuilder setConfigurationNames(List configurationNames) { + this.configurationNames = configurationNames; + return this; + } + + public StatusLogSearchCommandBuilder setStartTime(LocalDateTime startTime) { + this.startTime = startTime; + return this; + } + + public StatusLogSearchCommandBuilder setFinishTime(LocalDateTime finishTime) { + this.finishTime = finishTime; + return this; + } + + public StatusLogSearchCommandBuilder setScannedEventCount(Integer scannedEventCount) { + this.scannedEventCount = scannedEventCount; + return this; + } + + public StatusLogSearchCommandBuilder setErrorCount(Integer errorCount) { + this.errorCount = errorCount; + return this; + } + + public StatusLogSearchCommandBuilder setHasErrors(Boolean hasErrors) { + this.hasErrors = hasErrors; + return this; + } + + public StatusLogSearchCommandBuilder setLimit(Integer limit) { + this.limit = limit; + return this; + } + + public StatusLogSearchCommandBuilder setOffset(Integer offset) { + this.offset = offset; + return this; + } + + public StatusLogSearchCommand build() { + return new StatusLogSearchCommand(configurationNames, startTime, finishTime, scannedEventCount, errorCount, hasErrors, offset, limit); + } + } + +} diff --git a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java index d0a8e52..cecfc79 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java @@ -1,12 +1,19 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import java.time.LocalDateTime; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.*; @@ -17,62 +24,136 @@ class ScraperFacadeTest { private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; @Mock private ScraperConfigRepository scraperConfigRepository; + + @Captor + ArgumentCaptor lastScrapedEventMarkerArgumentCaptor; @InjectMocks private ScraperFacade scraperFacade; - @Test - void whenActivateScraperThanNotExistThenCreateNewRecord() { - //given - final String scraperName = "sc1"; - //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(false); - //then - scraperFacade.activateScraperByConfigurationName(scraperName); - - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository).create(scraperName); - verify(scraperConfigRepository, never()).activate(scraperName); - } + @Nested + class ScraperConfigTest { + @Test + void whenActivateScraperThatNotExistThenCreateNewRecord() { + //given + final String scraperName = "sc1"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(false); + //then + scraperFacade.activateScraperByConfigurationName(scraperName); - @Test - void whenActivateScraperThanExistThenFinishSuccessfully() { - //given - final String scraperName = "sc4"; - //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(true); - //then - scraperFacade.activateScraperByConfigurationName(scraperName); - - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName); - verify(scraperConfigRepository).activate(scraperName); - } + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository).create(scraperName, true); + verify(scraperConfigRepository, never()).activate(scraperName); + } + + @Test + void whenActivateScraperThatExistThenFinishSuccessfully() { + //given + final String scraperName = "sc4"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(true); + //then + scraperFacade.activateScraperByConfigurationName(scraperName); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository, never()).create(scraperName, true); + verify(scraperConfigRepository).activate(scraperName); + } - @Test - void whenDeactivateNotExistsScraperThanThrowConfigNotExistsException() { - //given - final String scraperName = "sc2"; - //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(false); - //then - assertThrows(ScraperConfigurationByNameNotExists.class, () -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); - - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName); - verify(scraperConfigRepository, never()).deactivate(scraperName); + @Test + void whenDeactivateNotExistsScraperThanThrowConfigNotExistsException() { + //given + final String scraperName = "sc2"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(false); + //then + assertThrows(ScraperConfigurationByNameNotExists.class, () -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository, never()).create(scraperName, true); + verify(scraperConfigRepository, never()).deactivate(scraperName); + } + + @Test + void whenDeactivateExistsScraperThanFinishSuccessfully() { + //given + final String scraperName = "sc3"; + //when + when(scraperConfigRepository.exists(scraperName)).thenReturn(true); + //then + assertDoesNotThrow(() -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); + + verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository, never()).create(scraperName, true); + verify(scraperConfigRepository).deactivate(scraperName); + } } - @Test - void whenDeactivateExistsScraperThanFinishSuccessfully() { - //given - final String scraperName = "sc3"; - //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(true); - //then - assertDoesNotThrow(() -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); - - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName); - verify(scraperConfigRepository).deactivate(scraperName); + @Nested + class LastScrapedEventMarkerTest { + @Test + void whenFoundLastScrapedEventMarkerThenReturnNotEmptyOptional() { + //given + final String scraperName = "scm4"; + final LastScrapedEventMarker lastScrapedEventMarker = new LastScrapedEventMarker(scraperName, LocalDateTime.now().minusDays(2), "Event Title", "Marker", false); + //when + when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) + .thenReturn(Optional.of(lastScrapedEventMarker)); + //then + + assertDoesNotThrow(() -> { + final Optional lastScrapedEventMarkerResult = scraperFacade.lastScrapedEventMarkerByConfigurationName(scraperName); + assertThat(lastScrapedEventMarkerResult) + .isNotEmpty() + .get() + .isEqualTo(lastScrapedEventMarker); + }); + + verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); + } + + @Test + void whenNotFoundLastScrapedEventMarkerThenReturnOptionalEmpty() { + //given + final String scraperName = "scm5"; + //when + when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) + .thenReturn(Optional.empty()); + //then + + assertDoesNotThrow(() -> { + final Optional lastScrapedEventMarkerResult = scraperFacade.lastScrapedEventMarkerByConfigurationName(scraperName); + assertThat(lastScrapedEventMarkerResult) + .isEmpty(); + }); + + verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); + } + + @Test + void whenCreateLastScrapedEventMarkerThenReplaceNotCompleteOne() { + //given + final String scraperConfigurationName = "name1"; + final LocalDateTime runTime = LocalDateTime.now(); + + final LastScrapedEventMarker previousLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime.minusDays(1), "Title 1", "Marker", false); + final LastScrapedEventMarker newLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime, "Title 2", "Marker 2", false); + + //when + when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, false)) + .thenReturn(Optional.of(previousLastScrapedEventMarker)); + + //then + scraperFacade.saveLastScrapedEventMarker(scraperConfigurationName, runTime, newLastScrapedEventMarker.eventTitle(), newLastScrapedEventMarker.marker()); + + verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperConfigurationName, false); + verify(lastScrapedEventMarkerRepository).drop(previousLastScrapedEventMarker); + + verify(lastScrapedEventMarkerRepository).store(lastScrapedEventMarkerArgumentCaptor.capture()); + + assertThat(lastScrapedEventMarkerArgumentCaptor.getValue()) + .isEqualTo(newLastScrapedEventMarker); + + } } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java index 9b321ed..24441fb 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java @@ -33,6 +33,8 @@ class ScraperRunServiceTest { private EventFacadeAdapter eventFacadeAdapter; @Mock private DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; + @Mock + private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; @Captor private ArgumentCaptor> scrapedEventListCaptor; @Captor @@ -57,7 +59,7 @@ void whenListContainsScraperWithoutConfigThenCreateNewConfig() { final List pageScrapers = Arrays.asList(activeScraper, noConfigScraper1, noConfigScraper2, inactiveScraper); final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraperConfig, inactiveScraperConfig)); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper.configurationName()).thenReturn(activeScraperName); @@ -70,10 +72,10 @@ void whenListContainsScraperWithoutConfigThenCreateNewConfig() { scraperRunService.start(); verify(scraperConfigRepository).allScraperConfigs(); - verify(scraperConfigRepository).create(noConfigScraper1Name); - verify(scraperConfigRepository).create(noConfigScraper2Name); - verify(scraperConfigRepository, never()).create(activeScraperName); - verify(scraperConfigRepository, never()).create(inactiveScraperName); + verify(scraperConfigRepository).create(noConfigScraper1Name, true); + verify(scraperConfigRepository).create(noConfigScraper2Name, true); + verify(scraperConfigRepository, never()).create(activeScraperName, true); + verify(scraperConfigRepository, never()).create(inactiveScraperName, true); } @Test @@ -93,7 +95,7 @@ void whenListContainsInactiveScraperThenSkipRunIt() { final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2, inactiveScraper); final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config, inactiveScraperConfig)); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); @@ -105,8 +107,7 @@ void whenListContainsInactiveScraperThenSkipRunIt() { //then scraperRunService.start(); - verify(scraperConfigRepository).allScraperConfigs(); - verify(scraperConfigRepository, never()).create(anyString()); + verify(scraperConfigRepository, never()).create(anyString(), anyBoolean()); verify(activeScraper1).scrap(); verify(activeScraper2).scrap(); verify(inactiveScraper, never()).scrap(); @@ -150,7 +151,7 @@ void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() throws EventDateInPastEx final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); @@ -216,7 +217,7 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsDuplicatedEventCandida analyzedEventCandidate3.addDuplicateCandidate(scrapedEvent1); final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); @@ -301,7 +302,7 @@ void whenScrapedEventContainsDuplicatedWithEventThenSaveAsDuplicatedEventCandida analyzedEventCandidate2.addDuplicateEvent(existsEvent); final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); @@ -340,4 +341,62 @@ void whenScrapedEventContainsDuplicatedWithEventThenSaveAsDuplicatedEventCandida tuple(scrapedEvent2, 0, 1, "1012") ); } + + @Test + void whenScrapedEventSavedThenMakeLastScrapedEventMarkerDraftActive() throws EventDateInPastException { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); + final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) + .build(); + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + .title("Title2") + .description("Description2") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) + .build(); + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + .title("Title1") + .description("Description1") + .scrapedTime(LocalDateTime.now()) + .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Eden Prime", "Addres 134", "location 166")) + .build(); + final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); + + final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); + final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); + final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); + + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + + when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); + + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + when(eventCandidateAnalyzer.analyze(scrapedEventList)) + .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); + + //then + scraperRunService.start(); + + verify(lastScrapedEventMarkerRepository).makeDraftActive(List.of(activeScraperName1,activeScraperName2)); + } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java index 582fe5a..0a748fb 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java @@ -2,6 +2,7 @@ import hub.event.scrapers.core.exceptions.EventDateEndDateTimeBeforeStartDateTimeException; import hub.event.scrapers.core.exceptions.EventDateInPastException; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.time.LocalDate; @@ -11,136 +12,141 @@ class SingleEventDateWithLocationTest { - @Test - void whenBuildSingleTypedDateWithIncorrectInputThenThrows() { - final LocalDate startDate = LocalDate.now().plusDays(2); - final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); - final LocalTime startTime = LocalTime.of(14, 20); - final LocalDate incorrectEndDate = LocalDate.now().plusDays(1); - final LocalDate correctEndDate = LocalDate.now().plusDays(2); - final LocalDate endDateInPast = LocalDate.of(2022, 7, 12); - final LocalTime incorrectEndTime = LocalTime.of(10, 10); - final LocalTime correctEndTime = LocalTime.of(20, 10); - final String city = "Thessia"; - final String address = "Nightmare Street 102/34"; - final String locationName = "Black hole mirror club"; - - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); - - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); - - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); + @Nested + class SingleTypedEventDateTest { + @Test + void whenBuildWithIncorrectInputThenThrows() { + final LocalDate startDate = LocalDate.now().plusDays(2); + final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); + final LocalTime startTime = LocalTime.of(14, 20); + final LocalDate incorrectEndDate = LocalDate.now().plusDays(1); + final LocalDate correctEndDate = LocalDate.now().plusDays(2); + final LocalDate endDateInPast = LocalDate.of(2022, 7, 12); + final LocalTime incorrectEndTime = LocalTime.of(10, 10); + final LocalTime correctEndTime = LocalTime.of(20, 10); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); + + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); + + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); + } + + @Test + void whenCorrectInputThenBuildCorrectly() { + final LocalDate startDate = LocalDate.now().plusDays(2); + final LocalTime startTime = LocalTime.of(10, 20); + final LocalDate endDate = LocalDate.now().plusDays(2); + final LocalTime endTime = LocalTime.of(20, 10); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation singleDateContainsStartDateAndTime = SingleEventDateWithLocation.single(startDate, startTime, city, address, locationName); + assertNotNull(singleDateContainsStartDateAndTime); + + assertEquals(startDate, singleDateContainsStartDateAndTime.startDate()); + assertEquals(startTime, singleDateContainsStartDateAndTime.startTime()); + assertEquals(city, singleDateContainsStartDateAndTime.city()); + assertEquals(address, singleDateContainsStartDateAndTime.address()); + assertEquals(locationName, singleDateContainsStartDateAndTime.locationName()); + assertNull(singleDateContainsStartDateAndTime.endDate()); + assertNull(singleDateContainsStartDateAndTime.endTime()); + assertTrue(singleDateContainsStartDateAndTime.isSingleDate()); + assertFalse(singleDateContainsStartDateAndTime.isPeriodDate()); + + }); + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, startTime, endDate, endTime, city, address, locationName); + assertNotNull(fullEventDate); + + assertEquals(startDate, fullEventDate.startDate()); + assertEquals(startTime, fullEventDate.startTime()); + assertEquals(city, fullEventDate.city()); + assertEquals(endDate, fullEventDate.endDate()); + assertEquals(endTime, fullEventDate.endTime()); + assertEquals(address, fullEventDate.address()); + assertEquals(locationName, fullEventDate.locationName()); + assertTrue(fullEventDate.isSingleDate()); + assertFalse(fullEventDate.isPeriodDate()); + }); + + } } - @Test - void whenBuildPeriodTypedDateWithIncorrectInputThenThrows() { - final LocalDate startDate = LocalDate.now().plusDays(2); - final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); - final LocalTime startTime = LocalTime.of(14, 20); - final LocalDate incorrectEndDate = LocalDate.now().plusDays(1); - final LocalDate correctEndDate = LocalDate.now().plusDays(2); - final LocalDate endDateInPast = LocalDate.of(2022, 2, 12); - final LocalTime incorrectEndTime = LocalTime.of(10, 10); - final LocalTime correctEndTime = LocalTime.of(20, 10); - final String city = "Thessia"; - final String address = "Nightmare Street 102/34"; - final String locationName = "Black hole mirror club"; - - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); - - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); + @Nested + class PeriodTypedDateTest { + @Test + void whenBuildWithIncorrectInputThenThrows() { + final LocalDate startDate = LocalDate.now().plusDays(2); + final LocalDate startDateInPast = LocalDate.of(2022, 1, 12); + final LocalTime startTime = LocalTime.of(14, 20); + final LocalDate incorrectEndDate = LocalDate.now().plusDays(1); + final LocalDate correctEndDate = LocalDate.now().plusDays(2); + final LocalDate endDateInPast = LocalDate.of(2022, 2, 12); + final LocalTime incorrectEndTime = LocalTime.of(10, 10); + final LocalTime correctEndTime = LocalTime.of(20, 10); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); + + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); + } + + + @Test + void whenCorrectInputThenBuildCorrectly() { + final LocalDate startDate = LocalDate.now().plusDays(2); + final LocalTime startTime = LocalTime.of(14, 0); + final LocalDate endDate = LocalDate.now().plusDays(10); + final LocalTime endTime = LocalTime.of(16, 30); + final String city = "Thessia"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, startTime, endDate, city, address, locationName); + + assertNotNull(periodDateContainsStartDateAndTime); + + assertEquals(startDate, periodDateContainsStartDateAndTime.startDate()); + assertEquals(startTime, periodDateContainsStartDateAndTime.startTime()); + assertEquals(city, periodDateContainsStartDateAndTime.city()); + assertEquals(endDate, periodDateContainsStartDateAndTime.endDate()); + assertEquals(address, periodDateContainsStartDateAndTime.address()); + assertEquals(locationName, periodDateContainsStartDateAndTime.locationName()); + assertNull(periodDateContainsStartDateAndTime.endTime()); + assertTrue(periodDateContainsStartDateAndTime.isPeriodDate()); + assertFalse(periodDateContainsStartDateAndTime.isSingleDate()); + }); + + assertDoesNotThrow(() -> { + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, city, address, locationName); + + assertNotNull(fullEventDate); + + assertEquals(startDate, fullEventDate.startDate()); + assertEquals(startTime, fullEventDate.startTime()); + assertEquals(city, fullEventDate.city()); + assertEquals(endDate, fullEventDate.endDate()); + assertEquals(endTime, fullEventDate.endTime()); + assertEquals(address, fullEventDate.address()); + assertEquals(locationName, fullEventDate.locationName()); + assertTrue(fullEventDate.isPeriodDate()); + assertFalse(fullEventDate.isSingleDate()); + }); + } } - - @Test - void whenCorrectInputThenSingleTypedEventDateBuildCorrectly() { - final LocalDate startDate = LocalDate.now().plusDays(2); - final LocalTime startTime = LocalTime.of(10, 20); - final LocalDate endDate = LocalDate.now().plusDays(2); - final LocalTime endTime = LocalTime.of(20, 10); - final String city = "Thessia"; - final String address = "Nightmare Street 102/34"; - final String locationName = "Black hole mirror club"; - - assertDoesNotThrow(() -> { - SingleEventDateWithLocation singleDateContainsStartDateAndTime = SingleEventDateWithLocation.single(startDate, startTime, city, address, locationName); - assertNotNull(singleDateContainsStartDateAndTime); - - assertEquals(startDate, singleDateContainsStartDateAndTime.startDate()); - assertEquals(startTime, singleDateContainsStartDateAndTime.startTime()); - assertEquals(city, singleDateContainsStartDateAndTime.city()); - assertEquals(address, singleDateContainsStartDateAndTime.address()); - assertEquals(locationName, singleDateContainsStartDateAndTime.locationName()); - assertNull(singleDateContainsStartDateAndTime.endDate()); - assertNull(singleDateContainsStartDateAndTime.endTime()); - assertTrue(singleDateContainsStartDateAndTime.isSingleDate()); - assertFalse(singleDateContainsStartDateAndTime.isPeriodDate()); - - }); - - assertDoesNotThrow(() -> { - SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, startTime, endDate, endTime, city, address, locationName); - assertNotNull(fullEventDate); - - assertEquals(startDate, fullEventDate.startDate()); - assertEquals(startTime, fullEventDate.startTime()); - assertEquals(city, fullEventDate.city()); - assertEquals(endDate, fullEventDate.endDate()); - assertEquals(endTime, fullEventDate.endTime()); - assertEquals(address, fullEventDate.address()); - assertEquals(locationName, fullEventDate.locationName()); - assertTrue(fullEventDate.isSingleDate()); - assertFalse(fullEventDate.isPeriodDate()); - }); - - - } - - @Test - void whenCorrectInputThenPeriodTypedEventDateBuildCorrectly() { - final LocalDate startDate = LocalDate.now().plusDays(2); - final LocalTime startTime = LocalTime.of(14, 0); - final LocalDate endDate = LocalDate.now().plusDays(10); - final LocalTime endTime = LocalTime.of(16, 30); - final String city = "Thessia"; - final String address = "Nightmare Street 102/34"; - final String locationName = "Black hole mirror club"; - - assertDoesNotThrow(() -> { - SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, startTime, endDate, city, address, locationName); - - assertNotNull(periodDateContainsStartDateAndTime); - - assertEquals(startDate, periodDateContainsStartDateAndTime.startDate()); - assertEquals(startTime, periodDateContainsStartDateAndTime.startTime()); - assertEquals(city, periodDateContainsStartDateAndTime.city()); - assertEquals(endDate, periodDateContainsStartDateAndTime.endDate()); - assertEquals(address, periodDateContainsStartDateAndTime.address()); - assertEquals(locationName, periodDateContainsStartDateAndTime.locationName()); - assertNull(periodDateContainsStartDateAndTime.endTime()); - assertTrue(periodDateContainsStartDateAndTime.isPeriodDate()); - assertFalse(periodDateContainsStartDateAndTime.isSingleDate()); - }); - - assertDoesNotThrow(() -> { - SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, city, address, locationName); - - assertNotNull(fullEventDate); - - assertEquals(startDate, fullEventDate.startDate()); - assertEquals(startTime, fullEventDate.startTime()); - assertEquals(city, fullEventDate.city()); - assertEquals(endDate, fullEventDate.endDate()); - assertEquals(endTime, fullEventDate.endTime()); - assertEquals(address, fullEventDate.address()); - assertEquals(locationName, fullEventDate.locationName()); - assertTrue(fullEventDate.isPeriodDate()); - assertFalse(fullEventDate.isSingleDate()); - }); - } - } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java b/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java index 04767c7..1da4813 100644 --- a/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java +++ b/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java @@ -43,12 +43,12 @@ void saveTest() { assertThat(lastScrapedEventMarkerEntityCaptor.getAllValues()) .extracting( LastScrapedEventMarkerEntity::getScraperConfigurationName, - LastScrapedEventMarkerEntity::getEventDate, + LastScrapedEventMarkerEntity::getRunDateTime, LastScrapedEventMarkerEntity::getEventTitle, LastScrapedEventMarkerEntity::getMarker ).contains( - tuple(eventMarker1.scraperConfigurationName(), eventMarker1.eventDate(), eventMarker1.eventTitle(), eventMarker1.marker()), - tuple(eventMarker2.scraperConfigurationName(), eventMarker2.eventDate(), eventMarker2.eventTitle(), eventMarker2.marker()) + tuple(eventMarker1.scraperConfigurationName(), eventMarker1.runDateTime(), eventMarker1.eventTitle(), eventMarker1.marker()), + tuple(eventMarker2.scraperConfigurationName(), eventMarker2.runDateTime(), eventMarker2.eventTitle(), eventMarker2.marker()) ); } @@ -63,7 +63,7 @@ void whenFindByNotExistsScraperConfigurationNameThenReturnEmptyMarker() { .thenReturn(Optional.empty()); //then - final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName); + final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, true); assertThat(lastScrapedEventMarker).isEmpty(); verify(lastScrapedEventMarkerEntityRepository).findByScraperConfigurationName(scraperConfigurationName); @@ -77,22 +77,23 @@ void whenFindByExistsScraperConfigurationNameThenReturnMarker() { final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); lastScrapedEventMarkerEntity.setMarker("maker1"); - lastScrapedEventMarkerEntity.setEventDate(localDateTime); + lastScrapedEventMarkerEntity.setRunDateTime(localDateTime); lastScrapedEventMarkerEntity.setEventTitle("title1"); lastScrapedEventMarkerEntity.setScraperConfigurationName("exists_scraper"); + lastScrapedEventMarkerEntity.setComplete(true); //when when(lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(scraperConfigurationName)) .thenReturn(Optional.of(lastScrapedEventMarkerEntity)); //then - final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName); + final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, true); assertThat(lastScrapedEventMarker).isNotEmpty() .get() .extracting( LastScrapedEventMarker::scraperConfigurationName, - LastScrapedEventMarker::eventDate, + LastScrapedEventMarker::runDateTime, LastScrapedEventMarker::eventTitle, LastScrapedEventMarker::marker ) diff --git a/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java b/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java index d452c00..71d4714 100644 --- a/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java +++ b/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java @@ -41,7 +41,7 @@ void logError() { verify(scraperRunLogRepository).save(scraperRunErrorLogArgumentCaptor.capture()); - final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.capture(); + final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.getValue(); assertNotNull(scraperRunErrorLog); assertEquals(configurationName, scraperRunErrorLog.configurationName()); @@ -66,7 +66,7 @@ void logStatus() { verify(scraperRunLogRepository).save(scraperRunStatusLogArgumentCaptor.capture()); - final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.capture(); + final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.getValue(); assertNotNull(scraperRunStatusLog); assertEquals(configurationName, scraperRunStatusLog.configurationName()); From f2654222e3c0f91c587fa0e61f71331dddb460e0 Mon Sep 17 00:00:00 2001 From: Dawid Olejnik Date: Thu, 6 Oct 2022 22:40:04 +0200 Subject: [PATCH 04/10] kret review --- .../event/scrapers/core/EventCandidateAnalyzer.java | 2 ++ .../hub/event/scrapers/core/EventFacadeAdapter.java | 2 +- .../hub/event/scrapers/core/PageScraperPort.java | 7 +++++-- .../java/hub/event/scrapers/core/ScrapedEvent.java | 10 +++++++++- .../java/hub/event/scrapers/core/ScraperConfig.java | 2 +- .../hub/event/scrapers/core/ScraperRunService.java | 13 +++++++++---- .../datewithlocation/EventDateWithLocation.java | 2 ++ .../scrapers/core/runlog/ScraperRunErrorLog.java | 2 +- 8 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java b/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java index 56fb81c..e680ace 100644 --- a/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java +++ b/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java @@ -4,6 +4,8 @@ class EventCandidateAnalyzer { List analyze(List scrapedEventList) { + // nie idź do event modułu żeby zdecydować o zapisie lub pominięciu. + // niech event moduł sam zdecyduje co mu się podoba //TODO replace this just map implementation by real one return scrapedEventList.stream() .map(this::mapAnalyzedEventCandidate) diff --git a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java index 341e5eb..c977875 100644 --- a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java +++ b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java @@ -4,6 +4,6 @@ class EventFacadeAdapter { public void saveAll(List scrapedEventList) { - //TODO integration with EventFacade from event module + //TODO integration with EventFacade from event module - mapping to event module API dto } } diff --git a/src/main/java/hub/event/scrapers/core/PageScraperPort.java b/src/main/java/hub/event/scrapers/core/PageScraperPort.java index 13799d0..ae72d59 100644 --- a/src/main/java/hub/event/scrapers/core/PageScraperPort.java +++ b/src/main/java/hub/event/scrapers/core/PageScraperPort.java @@ -3,6 +3,9 @@ import java.util.Collection; public interface PageScraperPort { - String configurationName(); + default String configurationName() { + // chyba załatwi sprawę w modularnym monolicie + return this.getClass().getName(); + } Collection scrap(); -} \ No newline at end of file +} diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java index 67e0ce2..f2af8ef 100644 --- a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java +++ b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java @@ -17,6 +17,14 @@ public class ScrapedEvent { private List types; private SingleEventDateWithLocation singleEventDateWithLocation; private MultipleEventDateWithLocations multipleEventDateWithLocations; + + // nowa klasa na powtarzający się event + // Period - od kiedy do kiedy + // List - lista dat kiedy + // Interval - co ile się powtarza + // można na razie nie robić + + // tu lepiej Instant private LocalDateTime scrapedTime; private ScrapedEvent(UUID uuid, String title, String description, String sourceLink, SingleEventDateWithLocation singleEventDateWithLocation, MultipleEventDateWithLocations multipleEventDateWithLocations, LocalDateTime scrapedTime, Map metadata, List types) { @@ -143,4 +151,4 @@ public ScrapedEvent build() { return new ScrapedEvent(eventUuid, title, description, sourceLink, singleEventDateWithLocation, multipleEventDateWithLocations, scrapedTime, metadata, types); } } -} \ No newline at end of file +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfig.java b/src/main/java/hub/event/scrapers/core/ScraperConfig.java index 5f9fda1..bccfa86 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperConfig.java +++ b/src/main/java/hub/event/scrapers/core/ScraperConfig.java @@ -1,4 +1,4 @@ package hub.event.scrapers.core; -public record ScraperConfig(String configurationName, boolean isActive) { +public record ScraperConfig(String configurationName, boolean isActive /*, String timeZone*/) { } diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java index b56ca11..4e9b697 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunService.java +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -36,10 +36,13 @@ class ScraperRunService { // @Scheduled(cron = "0 0 * * *") void start() { final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); - - final List scrapersWithoutConfig = getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); - for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { - saveActiveConfigAndAppendToConfigList(pageScraperPort, scraperConfigs); + { // zamiast tego kodu można zrobić akcje która się dzieje po tym jak apka wstanie (register new scraper) + final List scrapersWithoutConfig = + getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); + for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { + // raczej nie mutuj kolekcji, tylko zwróć nowy config którego potem wykorzystasz + saveActiveConfigAndAppendToConfigList(pageScraperPort, scraperConfigs); + } } final List pageScrapersToRun = getActiveScrapersThatShouldBeRun(scraperConfigs); @@ -49,6 +52,7 @@ void start() { final List notDuplicateScrapedEvents = extractNotDuplicateScrapedEvents(analyzedEventCandidates); final List analyzedEventCandidateMarkedAsDuplicate = extractDuplicateScrapedEvents(analyzedEventCandidates); + // do przemyślenia czy w ogóle potrzebne final List duplicatedEventCandidates = mapAnalyzedEventCandidateToDuplicatedEventCandidate(analyzedEventCandidateMarkedAsDuplicate); if (!notDuplicateScrapedEvents.isEmpty()) { @@ -70,6 +74,7 @@ private List getScraperThatConfigNotFoundByConfigurationName(Co .toList(); return pageScrapers.stream() + // filtruj w sqlu .filter(pageScraper -> !availableScraperConfigByName.contains(pageScraper.configurationName())) .toList(); } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java index 25e2b42..6d6f8ba 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java @@ -11,6 +11,8 @@ class EventDateWithLocation { private final LocalDate endDate; private final LocalTime startTime; private final LocalTime endTime; + // potrzebna dodatkowo strefa czasowa / zamień LocalDateTime na ZonedDateTime, + // chcemy wiedzieć dokładnie kiedy event się odbywa private final EventLocation eventLocation; diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java index ed21e60..dbe7713 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java @@ -2,5 +2,5 @@ import java.time.LocalDateTime; -public record ScraperRunErrorLog(String configurationName, LocalDateTime time, String errorCode, String description) { +public record ScraperRunErrorLog(String configurationName, /* Instant */ LocalDateTime time, String errorCode, String description) { } From 006f426422e91bacbee349b3ed9855842343d53b Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Mon, 10 Oct 2022 22:32:10 +0200 Subject: [PATCH 05/10] feat: 17 refactor and rebuild package scrapers.core --- .../hub/event/adminapi/AdminController.java | 4 - .../event/newsletter/NewsLetterService.java | 3 - .../scrapers/core/AnalyzedEventCandidate.java | 49 --- .../core/DuplicatedEventCandidate.java | 29 -- .../DuplicatedEventCandidateRepository.java | 12 - .../scrapers/core/EventCandidateAnalyzer.java | 18 -- .../scrapers/core/EventFacadeAdapter.java | 3 + .../hub/event/scrapers/core/ExistsEvent.java | 9 - .../LastScrapedEventMarkerEntity.java | 33 +- ...astScrapedEventMarkerEntityRepository.java | 14 +- .../LastScrapedEventMarkerJpaRepository.java | 18 ++ .../LastScrapedEventMarkerRepository.java | 2 + .../event/scrapers/core/PageScraperPort.java | 47 ++- .../hub/event/scrapers/core/ScrapedEvent.java | 65 ++-- .../event/scrapers/core/ScraperConfig.java | 4 - .../scrapers/core/ScraperConfigEntity.java | 49 +++ .../core/ScraperConfigEntityRepository.java | 50 +++ .../core/ScraperConfigJpaRepository.java | 14 + .../core/ScraperConfigRepository.java | 12 +- .../event/scrapers/core/ScraperFacade.java | 37 +-- .../scrapers/core/ScraperQueryFacade.java | 22 ++ .../core/ScraperRunErrorJpaRepository.java | 6 + .../core/ScraperRunErrorLogEntity.java | 61 ++++ .../core/ScraperRunLogJpaRepository.java | 7 + .../core/ScraperRunLogRepository.java | 186 +++++++++++ .../scrapers/core/ScraperRunService.java | 79 +---- .../core/ScraperRunStatusLogEntity.java | 69 +++++ .../EventDateWithLocation.java | 22 +- .../MultipleEventDateWithLocations.java | 13 +- .../SingleEventDateWithLocation.java | 47 ++- .../LastScrapedEventMarkerJpaRepository.java | 10 - ...hCommand.java => ErrorLogSearchQuery.java} | 14 +- .../core/runlog/ScraperRunErrorLog.java | 4 +- .../core/runlog/ScraperRunLogFacade.java | 35 --- .../core/runlog/ScraperRunLogRepository.java | 14 - .../core/runlog/ScraperRunStatusLog.java | 4 +- .../core/runlog/StatusLogSearchCommand.java | 123 -------- .../core/runlog/StatusLogSearchQuery.java | 161 ++++++++++ .../{ => scraper}/LastScrapedEventMarker.java | 12 +- .../scrapers/core/scraper/ScraperConfig.java | 6 + .../event/scrapers/ebilet/EbiletScraper.java | 8 +- .../empikbilet/EmpikBiletScraper.java | 10 +- .../scrapers/goingapp/GoingAppScraper.java | 9 +- .../kupbilecik/KupBilecikScraper.java | 9 +- .../scrapers/proanima/ProanimaScraper.java | 9 +- .../java/hub/event/users/UserService.java | 2 - .../hub/event/usersapi/UserController.java | 5 - src/main/resources/application.properties | 5 +- .../LastScrapedEventMarkerRepositoryTest.java | 12 +- .../scrapers/core/PageScraperPortTest.java | 168 ++++++++++ .../core/ScrapedEventBuilderTest.java | 34 +- .../scrapers/core/ScraperFacadeTest.java | 80 +---- .../scrapers/core/ScraperRunServiceTest.java | 293 ++---------------- .../MultipleEventDateWithLocationsTest.java | 16 +- .../SingleEventDateWithLocationTest.java | 33 +- .../core/runlog/ScraperRunLogFacadeTest.java | 80 ----- 56 files changed, 1088 insertions(+), 1052 deletions(-) delete mode 100644 src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java delete mode 100644 src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java delete mode 100644 src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java delete mode 100644 src/main/java/hub/event/scrapers/core/ExistsEvent.java rename src/main/java/hub/event/scrapers/core/{entityrepository => }/LastScrapedEventMarkerEntity.java (52%) rename src/main/java/hub/event/scrapers/core/{entityrepository => }/LastScrapedEventMarkerEntityRepository.java (86%) create mode 100644 src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfig.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java delete mode 100644 src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java rename src/main/java/hub/event/scrapers/core/runlog/{ErrorLogSearchCommand.java => ErrorLogSearchQuery.java} (62%) delete mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java delete mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java rename src/main/java/hub/event/scrapers/core/{ => scraper}/LastScrapedEventMarker.java (86%) create mode 100644 src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java rename src/test/java/hub/event/scrapers/core/{entityrepository => }/LastScrapedEventMarkerRepositoryTest.java (90%) create mode 100644 src/test/java/hub/event/scrapers/core/PageScraperPortTest.java delete mode 100644 src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java diff --git a/src/main/java/hub/event/adminapi/AdminController.java b/src/main/java/hub/event/adminapi/AdminController.java index 3790a86..4c62a2e 100644 --- a/src/main/java/hub/event/adminapi/AdminController.java +++ b/src/main/java/hub/event/adminapi/AdminController.java @@ -1,9 +1,5 @@ package hub.event.adminapi; -import hub.event.auth.AuthService; -import hub.event.users.UserService; -import hub.event.statistics.StatsService; - class AdminController { diff --git a/src/main/java/hub/event/newsletter/NewsLetterService.java b/src/main/java/hub/event/newsletter/NewsLetterService.java index dcc1424..99addcf 100644 --- a/src/main/java/hub/event/newsletter/NewsLetterService.java +++ b/src/main/java/hub/event/newsletter/NewsLetterService.java @@ -1,7 +1,4 @@ package hub.event.newsletter; -import hub.event.mailmodule.MailService; -import hub.event.events.EventService; - public class NewsLetterService { } diff --git a/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java b/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java deleted file mode 100644 index 534f17c..0000000 --- a/src/main/java/hub/event/scrapers/core/AnalyzedEventCandidate.java +++ /dev/null @@ -1,49 +0,0 @@ -package hub.event.scrapers.core; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -class AnalyzedEventCandidate { - private final List duplicateWithCandidates; - private final List duplicateWithEvents; - private final ScrapedEvent scrapedEvent; - - AnalyzedEventCandidate(ScrapedEvent scrapedEvent) { - this.scrapedEvent = scrapedEvent; - this.duplicateWithCandidates = new ArrayList<>(); - this.duplicateWithEvents = new ArrayList<>(); - } - - ScrapedEvent scrapedEvent() { - return scrapedEvent; - } - - void addDuplicateCandidate(ScrapedEvent scrapedEvent) { - duplicateWithCandidates.add(scrapedEvent); - } - - void addDuplicateEvent(ExistsEvent existsEvent) { - duplicateWithEvents.add(existsEvent); - } - - boolean isDuplicate() { - return !duplicateWithCandidates.isEmpty() || !duplicateWithEvents.isEmpty(); - } - - boolean isNotDuplicate() { - return duplicateWithCandidates.isEmpty() && duplicateWithEvents.isEmpty(); - } - - public List duplicateCandidateUUIDsList() { - return duplicateWithCandidates.stream() - .map(ScrapedEvent::uuid) - .toList(); - } - - public List duplicateEventIdList() { - return duplicateWithEvents.stream() - .map(ExistsEvent::eventId) - .toList(); - } -} diff --git a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java deleted file mode 100644 index d2bb474..0000000 --- a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidate.java +++ /dev/null @@ -1,29 +0,0 @@ -package hub.event.scrapers.core; - -import java.util.List; -import java.util.UUID; - -class DuplicatedEventCandidate { - private ScrapedEvent scrapedEvent; - private List duplicatedCandidates; - private List duplicatedEvents; - - public DuplicatedEventCandidate(ScrapedEvent scrapedEvent, List duplicatedCandidates, List duplicatedEvents) { - this.scrapedEvent = scrapedEvent; - this.duplicatedCandidates = duplicatedCandidates; - this.duplicatedEvents = duplicatedEvents; - } - - - ScrapedEvent getScrapedEvent() { - return scrapedEvent; - } - - List getDuplicatedCandidates() { - return duplicatedCandidates; - } - - List getDuplicatedEvents() { - return duplicatedEvents; - } -} diff --git a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java b/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java deleted file mode 100644 index 20557fb..0000000 --- a/src/main/java/hub/event/scrapers/core/DuplicatedEventCandidateRepository.java +++ /dev/null @@ -1,12 +0,0 @@ -package hub.event.scrapers.core; - -import java.util.List; -import java.util.UUID; - -interface DuplicatedEventCandidateRepository { - void saveAll(List candidates); - - List getAllDuplicatedEventCandidates(); - - boolean deleteDuplicatedEventCandidateByUuid(UUID eventCandidateUuid); -} diff --git a/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java b/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java deleted file mode 100644 index e680ace..0000000 --- a/src/main/java/hub/event/scrapers/core/EventCandidateAnalyzer.java +++ /dev/null @@ -1,18 +0,0 @@ -package hub.event.scrapers.core; - -import java.util.List; - -class EventCandidateAnalyzer { - List analyze(List scrapedEventList) { - // nie idź do event modułu żeby zdecydować o zapisie lub pominięciu. - // niech event moduł sam zdecyduje co mu się podoba - //TODO replace this just map implementation by real one - return scrapedEventList.stream() - .map(this::mapAnalyzedEventCandidate) - .toList(); - } - - private AnalyzedEventCandidate mapAnalyzedEventCandidate(ScrapedEvent scrapedEvent) { - return new AnalyzedEventCandidate(scrapedEvent); - } -} diff --git a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java index c977875..331d8d0 100644 --- a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java +++ b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java @@ -1,7 +1,10 @@ package hub.event.scrapers.core; +import org.springframework.stereotype.Repository; + import java.util.List; +@Repository class EventFacadeAdapter { public void saveAll(List scrapedEventList) { //TODO integration with EventFacade from event module - mapping to event module API dto diff --git a/src/main/java/hub/event/scrapers/core/ExistsEvent.java b/src/main/java/hub/event/scrapers/core/ExistsEvent.java deleted file mode 100644 index 09242e6..0000000 --- a/src/main/java/hub/event/scrapers/core/ExistsEvent.java +++ /dev/null @@ -1,9 +0,0 @@ -package hub.event.scrapers.core; - -import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; -import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; - -record ExistsEvent(int eventId, String title, String description, - SingleEventDateWithLocation singleEventDateWithLocation, - MultipleEventDateWithLocations multipleEventDateWithLocations) { -} diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java similarity index 52% rename from src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java rename to src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java index 6339efd..7602e05 100644 --- a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntity.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java @@ -1,62 +1,65 @@ -package hub.event.scrapers.core.entityrepository; +package hub.event.scrapers.core; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; -import java.time.LocalDateTime; +import javax.persistence.IdClass; +import java.io.Serializable; +import java.time.Instant; @Entity -class LastScrapedEventMarkerEntity { +@IdClass(LastScrapedEventMarkerEntity.class) +class LastScrapedEventMarkerEntity implements Serializable { @Id private String scraperConfigurationName; @Column(nullable = false) - private LocalDateTime runDateTime; + private Instant runDateTime; private String eventTitle; @Column(nullable = false) private String marker; - @Column(nullable = false) + @Id private Boolean complete; LastScrapedEventMarkerEntity() { } - public String getScraperConfigurationName() { + String getScraperConfigurationName() { return scraperConfigurationName; } - public void setScraperConfigurationName(String scraperConfigurationName) { + void setScraperConfigurationName(String scraperConfigurationName) { this.scraperConfigurationName = scraperConfigurationName; } - public LocalDateTime getRunDateTime() { + Instant getRunDateTime() { return runDateTime; } - public void setRunDateTime(LocalDateTime runDateTime) { + void setRunDateTime(Instant runDateTime) { this.runDateTime = runDateTime; } - public String getEventTitle() { + String getEventTitle() { return eventTitle; } - public void setEventTitle(String eventTitle) { + void setEventTitle(String eventTitle) { this.eventTitle = eventTitle; } - public String getMarker() { + String getMarker() { return marker; } - public void setMarker(String marker) { + void setMarker(String marker) { this.marker = marker; } - public Boolean getComplete() { + Boolean getComplete() { return complete; } - public void setComplete(Boolean complete) { + void setComplete(Boolean complete) { this.complete = complete; } } diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java similarity index 86% rename from src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java rename to src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java index 8bbab1a..8a63dbd 100644 --- a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerEntityRepository.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java @@ -1,10 +1,9 @@ -package hub.event.scrapers.core.entityrepository; +package hub.event.scrapers.core; -import hub.event.scrapers.core.LastScrapedEventMarker; -import hub.event.scrapers.core.LastScrapedEventMarkerRepository; +import hub.event.scrapers.core.scraper.LastScrapedEventMarker; import org.springframework.stereotype.Repository; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -23,12 +22,12 @@ public void store(LastScrapedEventMarker lastScrapedEventMarker) { @Override public void drop(LastScrapedEventMarker lastScrapedEventMarker) { - + lastScrapedEventMarkerJpaRepository.deleteById(lastScrapedEventMarker.scraperConfigurationName()); } @Override public void makeDraftActive(List configurationNameList) { - + lastScrapedEventMarkerJpaRepository.updateSetAllActiveById(configurationNameList); } public Optional findByScraperConfigurationName(String configurationName, boolean complete) { @@ -47,11 +46,12 @@ private LastScrapedEventMarkerEntity mapToEntity(LastScrapedEventMarker lastScra private LastScrapedEventMarker mapToMaker(LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity) { final String marker = lastScrapedEventMarkerEntity.getMarker(); - final LocalDateTime date = lastScrapedEventMarkerEntity.getRunDateTime(); + final Instant date = lastScrapedEventMarkerEntity.getRunDateTime(); final String title = lastScrapedEventMarkerEntity.getEventTitle(); final String configurationName = lastScrapedEventMarkerEntity.getScraperConfigurationName(); final Boolean isComplete = lastScrapedEventMarkerEntity.getComplete(); return new LastScrapedEventMarker(configurationName, date, title, marker, isComplete); } + } diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java new file mode 100644 index 0000000..0da50a2 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java @@ -0,0 +1,18 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.List; +import java.util.Optional; + +interface LastScrapedEventMarkerJpaRepository extends JpaRepository { + + Optional findByScraperConfigurationName(String configurationName); + + @Modifying + @Query("update LastScrapedEventMarkerEntity e set e.complete=true where e.scraperConfigurationName in :ids") + void updateSetAllActiveById(@Param("ids") List configurationNameList); +} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java index 34bb2c2..0d0bad3 100644 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java @@ -1,5 +1,7 @@ package hub.event.scrapers.core; +import hub.event.scrapers.core.scraper.LastScrapedEventMarker; + import java.util.List; import java.util.Optional; diff --git a/src/main/java/hub/event/scrapers/core/PageScraperPort.java b/src/main/java/hub/event/scrapers/core/PageScraperPort.java index ae72d59..564b970 100644 --- a/src/main/java/hub/event/scrapers/core/PageScraperPort.java +++ b/src/main/java/hub/event/scrapers/core/PageScraperPort.java @@ -1,11 +1,52 @@ package hub.event.scrapers.core; +import hub.event.scrapers.core.runlog.ScraperRunErrorLog; +import hub.event.scrapers.core.runlog.ScraperRunStatusLog; +import hub.event.scrapers.core.scraper.LastScrapedEventMarker; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.Instant; +import java.time.ZoneId; import java.util.Collection; +import java.util.Optional; + +public abstract class PageScraperPort { + @Autowired + private ScraperRunLogRepository scraperRunLogRepository; + @Autowired + private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; -public interface PageScraperPort { - default String configurationName() { + String configurationName() { // chyba załatwi sprawę w modularnym monolicie return this.getClass().getName(); } - Collection scrap(); + + protected abstract Collection scrap(); + + ZoneId timeZone() { + return ZoneId.systemDefault(); + } + + void logError(Instant time, String errorCode, String description) { + final ScraperRunErrorLog scraperRunErrorLog = new ScraperRunErrorLog(configurationName(), time, errorCode, description); + scraperRunLogRepository.save(scraperRunErrorLog); + } + + void logStatus(Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { + final ScraperRunStatusLog scraperRunStatusLog = new ScraperRunStatusLog(configurationName(), startTime, finishTime, scannedEventCount, errorCount); + scraperRunLogRepository.save(scraperRunStatusLog); + } + + Optional lastScrapedEventMarkerByConfigurationName() { + return lastScrapedEventMarkerRepository.findByScraperConfigurationName(configurationName(), true); + } + + void saveLastScrapedEventMarker(Instant runDateTime, String eventTitle, String marker) { + final Optional savedScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(configurationName(), false); + savedScrapedEventMarker.ifPresent(lastScrapedEventMarkerRepository::drop); + + final LastScrapedEventMarker newScrapedEventMarkerToSave = new LastScrapedEventMarker(configurationName(), runDateTime, eventTitle, marker); + lastScrapedEventMarkerRepository.store(newScrapedEventMarkerToSave); + } + } diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java index f2af8ef..a59ecd8 100644 --- a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java +++ b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java @@ -3,17 +3,14 @@ import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; -import java.time.LocalDateTime; import java.util.*; public class ScrapedEvent { - private UUID uuid; private Map metadata; private String title; private String description; private String sourceLink; - private List types; private SingleEventDateWithLocation singleEventDateWithLocation; private MultipleEventDateWithLocations multipleEventDateWithLocations; @@ -24,11 +21,7 @@ public class ScrapedEvent { // Interval - co ile się powtarza // można na razie nie robić - // tu lepiej Instant - private LocalDateTime scrapedTime; - - private ScrapedEvent(UUID uuid, String title, String description, String sourceLink, SingleEventDateWithLocation singleEventDateWithLocation, MultipleEventDateWithLocations multipleEventDateWithLocations, LocalDateTime scrapedTime, Map metadata, List types) { - this.uuid = uuid; + private ScrapedEvent(String title, String description, String sourceLink, SingleEventDateWithLocation singleEventDateWithLocation, MultipleEventDateWithLocations multipleEventDateWithLocations, Map metadata, List types) { this.metadata = metadata; this.title = title; this.description = description; @@ -36,14 +29,17 @@ private ScrapedEvent(UUID uuid, String title, String description, String sourceL this.types = types; this.singleEventDateWithLocation = singleEventDateWithLocation; this.multipleEventDateWithLocations = multipleEventDateWithLocations; - this.scrapedTime = scrapedTime; } private ScrapedEvent() { } - public static ScrapedEventBuilder builder() { - return new ScrapedEventBuilder(); + public static ScrapedEventBuilder builder(SingleEventDateWithLocation singleEventDateWithLocation) { + return new ScrapedEventBuilder(singleEventDateWithLocation); + } + + public static ScrapedEventBuilder builder(MultipleEventDateWithLocations multipleEventDateWithLocations) { + return new ScrapedEventBuilder(multipleEventDateWithLocations); } String title() { @@ -70,24 +66,15 @@ MultipleEventDateWithLocations multipleEventDateWithLocations() { return multipleEventDateWithLocations; } - LocalDateTime scrapedTime() { - return scrapedTime; - } - boolean hasMultipleDateAndLocations() { return Objects.nonNull(multipleEventDateWithLocations); } - UUID uuid() { - return uuid; - } - List types() { return types; } - static class ScrapedEventBuilder { - + public static class ScrapedEventBuilder { private final Map metadata; private final List types; private String title; @@ -95,13 +82,22 @@ static class ScrapedEventBuilder { private String sourceLink; private SingleEventDateWithLocation singleEventDateWithLocation; private MultipleEventDateWithLocations multipleEventDateWithLocations; - private LocalDateTime scrapedTime; - ScrapedEventBuilder() { + private ScrapedEventBuilder() { this.metadata = new HashMap<>(); this.types = new ArrayList<>(); } + private ScrapedEventBuilder(SingleEventDateWithLocation singleEventDateWithLocation) { + this(); + this.singleEventDateWithLocation = singleEventDateWithLocation; + } + + public ScrapedEventBuilder(MultipleEventDateWithLocations multipleEventDateWithLocations) { + this(); + this.multipleEventDateWithLocations = multipleEventDateWithLocations; + } + public ScrapedEventBuilder title(String title) { this.title = title; return this; @@ -121,34 +117,13 @@ public ScrapedEventBuilder metadata(String key, String value) { this.metadata.put(key, value); return this; } - - public ScrapedEventBuilder date(SingleEventDateWithLocation singleEventDateWithLocation) { - if (Objects.isNull(multipleEventDateWithLocations)) { - this.singleEventDateWithLocation = singleEventDateWithLocation; - } - - return this; - } - - public ScrapedEventBuilder date(MultipleEventDateWithLocations multipleEventDateWithLocations) { - this.singleEventDateWithLocation = null; - this.multipleEventDateWithLocations = multipleEventDateWithLocations; - return this; - } - - public ScrapedEventBuilder scrapedTime(LocalDateTime scrapedTime) { - this.scrapedTime = scrapedTime; - return this; - } - public ScrapedEventBuilder type(String type) { this.types.add(type); return this; } public ScrapedEvent build() { - final UUID eventUuid = UUID.randomUUID(); - return new ScrapedEvent(eventUuid, title, description, sourceLink, singleEventDateWithLocation, multipleEventDateWithLocations, scrapedTime, metadata, types); + return new ScrapedEvent(title, description, sourceLink, singleEventDateWithLocation, multipleEventDateWithLocations, metadata, types); } } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfig.java b/src/main/java/hub/event/scrapers/core/ScraperConfig.java deleted file mode 100644 index bccfa86..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperConfig.java +++ /dev/null @@ -1,4 +0,0 @@ -package hub.event.scrapers.core; - -public record ScraperConfig(String configurationName, boolean isActive /*, String timeZone*/) { -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java b/src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java new file mode 100644 index 0000000..b215d81 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java @@ -0,0 +1,49 @@ +package hub.event.scrapers.core; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; + +@Entity +class ScraperConfigEntity { + private String scraperName; + private String timeZone; + private boolean activeState; + @Id + private String configurationName; + @Column(nullable = false) + private boolean isActive; + + public ScraperConfigEntity() { + } + + public ScraperConfigEntity(String scraperName,String timeZone, boolean activeState) { + this.scraperName = scraperName; + this.timeZone = timeZone; + this.activeState = activeState; + } + + String getConfigurationName() { + return configurationName; + } + + void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + String getTimeZone() { + return timeZone; + } + + void setTimeZone(String timeZone) { + this.timeZone = timeZone; + } + + boolean isActive() { + return isActive; + } + + void setActive(boolean active) { + isActive = active; + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java new file mode 100644 index 0000000..65130e1 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java @@ -0,0 +1,50 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.scraper.ScraperConfig; +import org.springframework.stereotype.Repository; + +import java.time.ZoneId; +import java.util.List; + +@Repository +class ScraperConfigEntityRepository implements ScraperConfigRepository { + private final ScraperConfigJpaRepository scraperConfigJpaRepository; + + ScraperConfigEntityRepository(ScraperConfigJpaRepository scraperConfigJpaRepository) { + this.scraperConfigJpaRepository = scraperConfigJpaRepository; + } + + @Override + public boolean exists(String scraperName) { + return scraperConfigJpaRepository.existsById(scraperName); + } + + @Override + public void create(String scraperName, ZoneId timeZone ,boolean activeState) { + final ScraperConfigEntity scraperConfigEntity = new ScraperConfigEntity(scraperName, timeZone.toString(), activeState); + scraperConfigJpaRepository.save(scraperConfigEntity); + } + + @Override + public void activate(String scraperConfigurationName) { + scraperConfigJpaRepository.setActiveState(scraperConfigurationName, true); + } + + @Override + public void deactivate(String scraperConfigurationName) { + scraperConfigJpaRepository.setActiveState(scraperConfigurationName, false); + } + + @Override + public List allScraperConfigs() { + return scraperConfigJpaRepository.findAll() + .stream() + .map(this::mapToScraperConfig) + .toList(); + } + + private ScraperConfig mapToScraperConfig(ScraperConfigEntity scraperConfigEntity) { + ZoneId timeZone = ZoneId.of(scraperConfigEntity.getTimeZone()); + return new ScraperConfig(scraperConfigEntity.getConfigurationName(), timeZone, scraperConfigEntity.isActive()); + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java new file mode 100644 index 0000000..136d0f5 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java @@ -0,0 +1,14 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +interface ScraperConfigJpaRepository extends JpaRepository { + + @Modifying + @Query("update ScraperConfigEntity e set e.activeState = :state where e.scraperName = :name" ) + void setActiveState(@Param("name") String scraperConfigurationName, @Param("state") boolean activeState); + +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java index 01080d2..b81d1b0 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java @@ -1,16 +1,20 @@ package hub.event.scrapers.core; -import java.util.Collection; -interface ScraperConfigRepository { +import hub.event.scrapers.core.scraper.ScraperConfig; + +import java.time.ZoneId; +import java.util.List; + +public interface ScraperConfigRepository { boolean exists(String scraperName); - void create(String scraperName, boolean activeState); + void create(String scraperName, ZoneId timeZone, boolean activeState); void activate(String scraperConfigurationName); void deactivate(String scraperConfigurationName); - Collection allScraperConfigs(); + List allScraperConfigs(); } diff --git a/src/main/java/hub/event/scrapers/core/ScraperFacade.java b/src/main/java/hub/event/scrapers/core/ScraperFacade.java index 017352c..86e34ed 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperFacade.java +++ b/src/main/java/hub/event/scrapers/core/ScraperFacade.java @@ -1,32 +1,25 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; -import org.apache.commons.lang3.NotImplementedException; import org.springframework.stereotype.Service; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.time.ZoneId; @Service public class ScraperFacade { - private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; - private final ScraperConfigRepository scraperConfigRepository; - private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; + private final ScraperConfigRepository scraperConfigRepository; - public ScraperFacade(LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, ScraperConfigRepository scraperConfigRepository, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository) { - this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; + public ScraperFacade(ScraperConfigRepository scraperConfigRepository) { this.scraperConfigRepository = scraperConfigRepository; - this.duplicatedEventCandidateRepository = duplicatedEventCandidateRepository; } public void activateScraperByConfigurationName(String scraperName) { + //TODO nightmare throw exception if config not exists if (scraperConfigRepository.exists(scraperName)) { scraperConfigRepository.activate(scraperName); } else { - scraperConfigRepository.create(scraperName, true); + scraperConfigRepository.create(scraperName, ZoneId.systemDefault(),true); } } @@ -38,24 +31,4 @@ public void deactivateScraperByConfigurationName(String scraperName) throws Scra scraperConfigRepository.deactivate(scraperName); } - - public Optional lastScrapedEventMarkerByConfigurationName(String scraperConfigurationName) { - return lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, true); - } - - public void saveLastScrapedEventMarker(String scraperConfigurationName, LocalDateTime runDateTime, String eventTitle, String marker) { - final Optional savedScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, false); - savedScrapedEventMarker.ifPresent(lastScrapedEventMarkerRepository::drop); - - final LastScrapedEventMarker newScrapedEventMarkerToSave = new LastScrapedEventMarker(scraperConfigurationName, runDateTime, eventTitle, marker); - lastScrapedEventMarkerRepository.store(newScrapedEventMarkerToSave); - } - - public List getAllDuplicatedEventCandidates() { - return duplicatedEventCandidateRepository.getAllDuplicatedEventCandidates(); - } - - public boolean deleteDuplicatedEventCandidateByUuid(UUID eventCandidateUuid) { - return duplicatedEventCandidateRepository.deleteDuplicatedEventCandidateByUuid(eventCandidateUuid); - } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java b/src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java new file mode 100644 index 0000000..50e2d5b --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java @@ -0,0 +1,22 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.runlog.*; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class ScraperQueryFacade { + private final ScraperRunLogRepository scraperRunLogRepository; + + public ScraperQueryFacade(ScraperRunLogRepository scraperRunLogRepository) { + this.scraperRunLogRepository = scraperRunLogRepository; + } + public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { + return scraperRunLogRepository.findAllErrorLog(errorLogSearchQuery); + } + public List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { + return scraperRunLogRepository.findAllStatusLog(statusLogSearchQuery); + } + +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java b/src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java new file mode 100644 index 0000000..9c82a7c --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java @@ -0,0 +1,6 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; + +interface ScraperRunErrorJpaRepository extends JpaRepository { +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java b/src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java new file mode 100644 index 0000000..6d54081 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java @@ -0,0 +1,61 @@ +package hub.event.scrapers.core; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import java.time.Instant; + +@Entity +class ScraperRunErrorLogEntity { + @Id + private String configurationName; + @Column(nullable = false) + private Instant time; + + @Column(nullable = false) + private String errorCode; + private String description; + + public ScraperRunErrorLogEntity() { + } + + ScraperRunErrorLogEntity(String configurationName, Instant time, String errorCode, String description) { + + this.configurationName = configurationName; + this.time = time; + this.errorCode = errorCode; + this.description = description; + } + + String getConfigurationName() { + return configurationName; + } + + void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + Instant getTime() { + return time; + } + + void setTime(Instant time) { + this.time = time; + } + + String getErrorCode() { + return errorCode; + } + + void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + String getDescription() { + return description; + } + + void setDescription(String description) { + this.description = description; + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java b/src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java new file mode 100644 index 0000000..eaa8feb --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; + +interface ScraperRunLogJpaRepository extends JpaRepository, JpaSpecificationExecutor { +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java b/src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java new file mode 100644 index 0000000..c4e94dc --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java @@ -0,0 +1,186 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.runlog.ErrorLogSearchQuery; +import hub.event.scrapers.core.runlog.ScraperRunErrorLog; +import hub.event.scrapers.core.runlog.ScraperRunStatusLog; +import hub.event.scrapers.core.runlog.StatusLogSearchQuery; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Repository; + +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +@Repository +class ScraperRunLogRepository { + + private ScraperRunLogJpaRepository scraperRunLogJpaRepository; + private ScraperRunErrorJpaRepository scraperRunErrorJpaRepository; + + void save(ScraperRunStatusLog scraperRunStatusLog) { + ScraperRunStatusLogEntity scraperRunStatusLogEntity = mapToEntity(scraperRunStatusLog); + scraperRunLogJpaRepository.save(scraperRunStatusLogEntity); + } + + void save(ScraperRunErrorLog scraperRunError) { + ScraperRunErrorLogEntity scraperRunErrorLogEntity = mapToEntity(scraperRunError); + scraperRunErrorJpaRepository.save(scraperRunErrorLogEntity); + } + + List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { + final Sort sort = Sort.by(Sort.Direction.ASC, "configurationName"); + final Specification findAllSpecification = new FindAllErrorLogSpecification(errorLogSearchQuery); + final Pageable pageable = extractPageSettings(errorLogSearchQuery, sort); + + return findAllErrorLog(findAllSpecification, pageable, sort) + .stream() + .map(this::mapToLog) + .toList(); + + } + + List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { + final Sort sort = Sort.by(Sort.Direction.ASC, "configurationName"); + final Specification findAllSpecification = new FindAllStatusLogSpecification(statusLogSearchQuery); + final Pageable pageable = extractPageSettings(statusLogSearchQuery, sort); + + return findAllStatusLog(findAllSpecification, pageable, sort) + .stream() + .map(this::mapToLog) + .toList(); + } + + private List findAllStatusLog(Specification findAllSpecification, Pageable pageable, Sort sort) { + if (Objects.isNull(pageable)) { + return scraperRunLogJpaRepository.findAll(findAllSpecification, sort); + } + return scraperRunLogJpaRepository.findAll(findAllSpecification, pageable).toList(); + } + + private List findAllErrorLog(Specification findAllSpecification, Pageable pageable, Sort sort) { + return null; + } + + private ScraperRunStatusLog mapToLog(ScraperRunStatusLogEntity scraperRunStatusLogEntity) { + return new ScraperRunStatusLog( + scraperRunStatusLogEntity.getConfigurationName(), + scraperRunStatusLogEntity.getStartTime(), + scraperRunStatusLogEntity.getFinishTime(), + scraperRunStatusLogEntity.getScannedEventCount(), + scraperRunStatusLogEntity.getErrorCount() + ); + } + + private ScraperRunErrorLog mapToLog(ScraperRunErrorLogEntity scraperRunErrorLogEntity) { + return null; + } + + private ScraperRunStatusLogEntity mapToEntity(ScraperRunStatusLog scraperRunStatusLog) { + return new ScraperRunStatusLogEntity( + scraperRunStatusLog.configurationName(), + scraperRunStatusLog.startTime(), + scraperRunStatusLog.finishTime(), + scraperRunStatusLog.scannedEventCount(), + scraperRunStatusLog.errorCount() + ); + } + + private ScraperRunErrorLogEntity mapToEntity(ScraperRunErrorLog scraperRunError) { + return new ScraperRunErrorLogEntity( + scraperRunError.configurationName(), + scraperRunError.time(), + scraperRunError.errorCode(), + scraperRunError.description() + ); + } + + private Pageable extractPageSettings(StatusLogSearchQuery statusLogSearchQuery, Sort sort) { + return statusLogSearchQuery.hasPageSetting() + ? PageRequest.of(statusLogSearchQuery.page(), statusLogSearchQuery.pageSize(), sort) + : null; + } + + private Pageable extractPageSettings(ErrorLogSearchQuery errorLogSearchQuery, Sort sort) { + return null; + } + + private static class FindAllStatusLogSpecification implements Specification { + + private final StatusLogSearchQuery command; + + public FindAllStatusLogSpecification(StatusLogSearchQuery statusLogSearchQuery) { + + this.command = statusLogSearchQuery; + } + + + @Override + public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { + List outputPredicates = new ArrayList<>(); + + if (command.hasConfigurationNames()) { + outputPredicates.add(criteriaBuilder.in(root.get("configurationName")).value(command.configurationNames())); + } + + if (Objects.nonNull(command.startTimeFrom())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("startTime"), command.startTimeFrom())); + } + + if (Objects.nonNull(command.startTimeTo())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("startTime"), command.startTimeTo())); + } + + if (Objects.nonNull(command.finishTimeFrom())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("finishTime"), command.finishTimeFrom())); + } + + if (Objects.nonNull(command.finishTimeTo())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("finishTime"), command.finishTimeTo())); + } + + if (Objects.nonNull(command.hasScannedEvent())) { + final Predicate predicate = Boolean.TRUE.equals(command.hasScannedEvent()) + ? criteriaBuilder.greaterThan(root.get("scannedEventCount"), 0) + : criteriaBuilder.equal(root.get("scannedEventCount"), 0); + + outputPredicates.add(predicate); + } else if (Objects.nonNull(command.scannedEventCount())) { + outputPredicates.add(criteriaBuilder.equal(root.get("scannedEventCount"), command.scannedEventCount())); + } + + if (Objects.nonNull(command.hasErrors())) { + final Predicate predicate = Boolean.TRUE.equals(command.hasErrors()) + ? criteriaBuilder.greaterThan(root.get("errorCount"), 0) + : criteriaBuilder.equal(root.get("errorCount"), 0); + + outputPredicates.add(predicate); + } else if (Objects.nonNull(command.errorCount())) { + outputPredicates.add(criteriaBuilder.equal(root.get("errorCount"), command.scannedEventCount())); + } + + return outputPredicates.stream() + .reduce(criteriaBuilder::and) + .orElse(null); + } + } + + private static class FindAllErrorLogSpecification implements Specification { + private final ErrorLogSearchQuery errorLogSearchQuery; + + public FindAllErrorLogSpecification(ErrorLogSearchQuery errorLogSearchQuery) { + this.errorLogSearchQuery = errorLogSearchQuery; + } + + @Override + public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { + return null; + } + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java index 4e9b697..fb60d5e 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunService.java +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -1,14 +1,15 @@ package hub.event.scrapers.core; +import hub.event.scrapers.core.scraper.ScraperConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; @@ -16,52 +17,38 @@ @EnableScheduling class ScraperRunService { private final ScraperConfigRepository scraperConfigRepository; - private final EventCandidateAnalyzer eventCandidateAnalyzer; private final EventFacadeAdapter eventFacadeAdapter; - private final DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; private final List pageScrapers; @Autowired - ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventCandidateAnalyzer eventCandidateAnalyzer, EventFacadeAdapter eventFacadeAdapter, DuplicatedEventCandidateRepository duplicatedEventCandidateRepository, LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, List pageScrapers) { + ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventFacadeAdapter eventFacadeAdapter, LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, List pageScrapers) { this.scraperConfigRepository = scraperConfigRepository; - this.eventCandidateAnalyzer = eventCandidateAnalyzer; this.eventFacadeAdapter = eventFacadeAdapter; - this.duplicatedEventCandidateRepository = duplicatedEventCandidateRepository; this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; this.pageScrapers = pageScrapers; } @Scheduled(cron = "${scrapers.run.cron.expression}") // @Scheduled(cron = "0 0 * * *") - void start() { + + @PostConstruct + void createScrapersConfigsIfMissing() { final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); - { // zamiast tego kodu można zrobić akcje która się dzieje po tym jak apka wstanie (register new scraper) - final List scrapersWithoutConfig = - getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); - for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { - // raczej nie mutuj kolekcji, tylko zwróć nowy config którego potem wykorzystasz - saveActiveConfigAndAppendToConfigList(pageScraperPort, scraperConfigs); - } + final List scrapersWithoutConfig = getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); + + for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { + scraperConfigRepository.create(pageScraperPort.configurationName(), pageScraperPort.timeZone(),true); } + } + + void start() { + final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); final List pageScrapersToRun = getActiveScrapersThatShouldBeRun(scraperConfigs); final List scrapedEventList = runScrapersForEvents(pageScrapersToRun); - final List analyzedEventCandidates = eventCandidateAnalyzer.analyze(scrapedEventList); - final List notDuplicateScrapedEvents = extractNotDuplicateScrapedEvents(analyzedEventCandidates); - final List analyzedEventCandidateMarkedAsDuplicate = extractDuplicateScrapedEvents(analyzedEventCandidates); - - // do przemyślenia czy w ogóle potrzebne - final List duplicatedEventCandidates = mapAnalyzedEventCandidateToDuplicatedEventCandidate(analyzedEventCandidateMarkedAsDuplicate); - - if (!notDuplicateScrapedEvents.isEmpty()) { - eventFacadeAdapter.saveAll(notDuplicateScrapedEvents); - } - - if (!duplicatedEventCandidates.isEmpty()) { - duplicatedEventCandidateRepository.saveAll(duplicatedEventCandidates); - } + eventFacadeAdapter.saveAll(scrapedEventList); final List runScraperConfigurationsNames = getRunScraperConfigurationsNames(pageScrapersToRun); lastScrapedEventMarkerRepository.makeDraftActive(runScraperConfigurationsNames); @@ -74,18 +61,10 @@ private List getScraperThatConfigNotFoundByConfigurationName(Co .toList(); return pageScrapers.stream() - // filtruj w sqlu .filter(pageScraper -> !availableScraperConfigByName.contains(pageScraper.configurationName())) .toList(); } - private void saveActiveConfigAndAppendToConfigList(PageScraperPort pageScraperPort, Collection scraperConfigs) { - scraperConfigRepository.create(pageScraperPort.configurationName(), true); - final ScraperConfig scraperConfig = new ScraperConfig(pageScraperPort.configurationName(), true); - - scraperConfigs.add(scraperConfig); - } - private List getActiveScrapersThatShouldBeRun(Collection scraperConfigs) { final Map configStatusByScraperConfigurationNameMap = scraperConfigs.stream() .collect(Collectors.toMap(ScraperConfig::configurationName, ScraperConfig::isActive)); @@ -102,34 +81,6 @@ private List runScrapersForEvents(List pageScrape .toList(); } - private List extractNotDuplicateScrapedEvents(List analyzedEventCandidates) { - return analyzedEventCandidates.stream() - .filter(AnalyzedEventCandidate::isNotDuplicate) - .map(AnalyzedEventCandidate::scrapedEvent) - .toList(); - } - - private List extractDuplicateScrapedEvents(List analyzedEventCandidates) { - return analyzedEventCandidates.stream() - .filter(AnalyzedEventCandidate::isDuplicate) - .toList(); - } - - private List mapAnalyzedEventCandidateToDuplicatedEventCandidate(List analyzedEventCandidateMarkedAsDuplicate) { - return analyzedEventCandidateMarkedAsDuplicate.stream() - .map(this::mapToDuplicatedEventCandidate) - .toList(); - } - - private DuplicatedEventCandidate mapToDuplicatedEventCandidate(AnalyzedEventCandidate analyzedEventCandidate) { - - final ScrapedEvent scrapedEvent = analyzedEventCandidate.scrapedEvent(); - final List duplicateCandidateUUIDsList = analyzedEventCandidate.duplicateCandidateUUIDsList(); - final List duplicateEventIdList = analyzedEventCandidate.duplicateEventIdList(); - - return new DuplicatedEventCandidate(scrapedEvent, duplicateCandidateUUIDsList, duplicateEventIdList); - } - private List getRunScraperConfigurationsNames(List pageScrapersToRun) { return pageScrapersToRun.stream() .map(PageScraperPort::configurationName) diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java b/src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java new file mode 100644 index 0000000..7391608 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java @@ -0,0 +1,69 @@ +package hub.event.scrapers.core; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import java.time.Instant; + +@Entity +class ScraperRunStatusLogEntity { + @Id + private String configurationName; + @Column(nullable = false) + private Instant startTime; + @Column(nullable = false) + private Instant finishTime; + @Column(nullable = false) + private Integer scannedEventCount; + @Column(nullable = false) + private Integer errorCount; + + + ScraperRunStatusLogEntity(String configurationName, Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { + this.configurationName = configurationName; + this.startTime = startTime; + this.finishTime = finishTime; + this.scannedEventCount = scannedEventCount; + this.errorCount = errorCount; + } + + String getConfigurationName() { + return configurationName; + } + + void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + Instant getStartTime() { + return startTime; + } + + void setStartTime(Instant startTime) { + this.startTime = startTime; + } + + Instant getFinishTime() { + return finishTime; + } + + void setFinishTime(Instant finishTime) { + this.finishTime = finishTime; + } + + Integer getScannedEventCount() { + return scannedEventCount; + } + + void setScannedEventCount(Integer scannedEventCount) { + this.scannedEventCount = scannedEventCount; + } + + Integer getErrorCount() { + return errorCount; + } + + void setErrorCount(Integer errorCount) { + this.errorCount = errorCount; + } +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java index 6d6f8ba..b14c390 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/EventDateWithLocation.java @@ -3,29 +3,31 @@ import hub.event.scrapers.core.exceptions.EventDateInPastException; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZoneId; class EventDateWithLocation { private final LocalDate startDate; private final LocalDate endDate; private final LocalTime startTime; private final LocalTime endTime; - // potrzebna dodatkowo strefa czasowa / zamień LocalDateTime na ZonedDateTime, - // chcemy wiedzieć dokładnie kiedy event się odbywa + + private final ZoneId timeZone; private final EventLocation eventLocation; - public EventDateWithLocation(LocalDate startDate, LocalTime startTime,LocalDate endDate, LocalTime endTime, String city, String address, String locationName) { + public EventDateWithLocation(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, ZoneId timeZone, String city, String address, String locationName) { this.startDate = startDate; this.endDate = endDate; this.startTime = startTime; this.endTime = endTime; + this.timeZone = timeZone; this.eventLocation = new EventLocation(city, address, locationName); } - public EventDateWithLocation(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { + public EventDateWithLocation(LocalDate date, LocalTime time, ZoneId timeZone, String city, String address, String locationName) throws EventDateInPastException { inputDateValidation(date); + this.timeZone = timeZone; this.startDate = date; this.endDate = null; this.startTime = time; @@ -61,9 +63,13 @@ String locationName() { return eventLocation.name(); } - private void inputDateValidation(LocalDate date) throws EventDateInPastException { - if(date.isBefore(LocalDate.now())){ - throw new EventDateInPastException(date); + ZoneId timeZone() { + return timeZone; } + + private void inputDateValidation(LocalDate date) throws EventDateInPastException { + if (date.isBefore(LocalDate.now())) { + throw new EventDateInPastException(date); + } } } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java index f12e4bd..9ee6f0e 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java @@ -4,6 +4,7 @@ import java.time.LocalDate; import java.time.LocalTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; @@ -14,18 +15,18 @@ private MultipleEventDateWithLocations() { eventDateWithLocations = new ArrayList<>(); } - private MultipleEventDateWithLocations(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { + private MultipleEventDateWithLocations(LocalDate date, LocalTime time, ZoneId timeZone, String city, String address, String locationName) throws EventDateInPastException { this(); - final EventDateWithLocation eventDateWithLocation = new EventDateWithLocation(date, time, city, address, locationName); + final EventDateWithLocation eventDateWithLocation = new EventDateWithLocation(date, time, timeZone, city, address, locationName); this.eventDateWithLocations.add(eventDateWithLocation); } - public static MultipleEventDateWithLocations create(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { - return new MultipleEventDateWithLocations(date, time, city, address, locationName); + public static MultipleEventDateWithLocations create(LocalDate date, LocalTime time, ZoneId timeZone, String city, String address, String locationName) throws EventDateInPastException { + return new MultipleEventDateWithLocations(date, time, timeZone,city, address, locationName); } - public MultipleEventDateWithLocations add(LocalDate date, LocalTime time, String city, String address, String locationName) throws EventDateInPastException { - final EventDateWithLocation eventDateWithLocation = new EventDateWithLocation(date, time, city, address, locationName); + public MultipleEventDateWithLocations add(LocalDate date, LocalTime time, ZoneId timeZone, String city, String address, String locationName) throws EventDateInPastException { + final EventDateWithLocation eventDateWithLocation = new EventDateWithLocation(date, time, timeZone, city, address, locationName); this.eventDateWithLocations.add(eventDateWithLocation); return this; } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java index 081707e..f1f75b2 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java @@ -6,38 +6,35 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZoneId; public class SingleEventDateWithLocation { - private EventDateType eventDateType; - private EventDateWithLocation eventDateWithLocation; + private final EventDateType eventDateType; + private final EventDateWithLocation eventDateWithLocation; - private SingleEventDateWithLocation() { - - } - - private SingleEventDateWithLocation(EventDateType eventDateType, LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, String city, String address, String locationName) { + private SingleEventDateWithLocation(EventDateType eventDateType, LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, ZoneId timeZone, String city, String address, String locationName) { this.eventDateType = eventDateType; - this.eventDateWithLocation = new EventDateWithLocation(startDate, startTime, endDate, endTime, city, address, locationName); + this.eventDateWithLocation = new EventDateWithLocation(startDate, startTime, endDate, endTime, timeZone, city, address, locationName); } - public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, String city, String address, String locationName) throws EventDateInPastException { - validateStartDateTime(startDate,startTime); - return new SingleEventDateWithLocation(EventDateType.SINGLE, startDate, startTime, null, null, city, address, locationName); + public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, ZoneId timeZone, String city, String address, String locationName) throws EventDateInPastException { + validateStartDateTime(startDate, startTime); + return new SingleEventDateWithLocation(EventDateType.SINGLE, startDate, startTime, null, null, timeZone, city, address, locationName); } - public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + public static SingleEventDateWithLocation single(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, ZoneId timeZone, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { validateStartDateTimeAndEndDateTime(startDate, startTime, endDate, endTime); - return new SingleEventDateWithLocation(EventDateType.SINGLE, startDate, startTime, endDate, endTime, city, address, locationName); + return new SingleEventDateWithLocation(EventDateType.SINGLE, startDate, startTime, endDate, endTime, timeZone, city, address, locationName); } - public static SingleEventDateWithLocation period(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { - validateStartDateTimeAndEndDateTime(startDate,startTime,endDate,endTime); - return new SingleEventDateWithLocation(EventDateType.PERIOD, startDate, startTime, endDate, endTime, city, address, locationName); + public static SingleEventDateWithLocation period(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime, ZoneId timeZone, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + validateStartDateTimeAndEndDateTime(startDate, startTime, endDate, endTime); + return new SingleEventDateWithLocation(EventDateType.PERIOD, startDate, startTime, endDate, endTime, timeZone, city, address, locationName); } - public static SingleEventDateWithLocation period(LocalDate startDate, LocalTime startTime, LocalDate endDate, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { - validateStartDateTimeAndEndDateTime(startDate,startTime,endDate,startTime); - return new SingleEventDateWithLocation(EventDateType.PERIOD, startDate, startTime, endDate, null, city, address, locationName); + public static SingleEventDateWithLocation period(LocalDate startDate, LocalTime startTime, LocalDate endDate, ZoneId timeZone, String city, String address, String locationName) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { + validateStartDateTimeAndEndDateTime(startDate, startTime, endDate, startTime); + return new SingleEventDateWithLocation(EventDateType.PERIOD, startDate, startTime, endDate, null, timeZone, city, address, locationName); } LocalDate startDate() { @@ -82,14 +79,14 @@ private static void validateStartDateTimeAndEndDateTime(LocalDate startDate, Loc final LocalDateTime endLocalDateTime = LocalDateTime.of(endDate, endTime); final LocalDateTime now = LocalDateTime.now(); - if(startLocalDateTime.isBefore(now)){ - throw new EventDateInPastException(startLocalDateTime); + if (startLocalDateTime.isBefore(now)) { + throw new EventDateInPastException(startLocalDateTime); } - if(endLocalDateTime.isBefore(now)){ - throw new EventDateInPastException(endLocalDateTime); + if (endLocalDateTime.isBefore(now)) { + throw new EventDateInPastException(endLocalDateTime); } - if(endLocalDateTime.isBefore(startLocalDateTime)){ + if (endLocalDateTime.isBefore(startLocalDateTime)) { throw new EventDateEndDateTimeBeforeStartDateTimeException(startLocalDateTime, endLocalDateTime); } } @@ -98,7 +95,7 @@ private static void validateStartDateTime(LocalDate startDate, LocalTime startTi final LocalDateTime startLocalDateTime = LocalDateTime.of(startDate, startTime); final LocalDateTime now = LocalDateTime.now(); - if(startLocalDateTime.isBefore(now)){ + if (startLocalDateTime.isBefore(now)) { throw new EventDateInPastException(startLocalDateTime); } diff --git a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java b/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java deleted file mode 100644 index 118da53..0000000 --- a/src/main/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerJpaRepository.java +++ /dev/null @@ -1,10 +0,0 @@ -package hub.event.scrapers.core.entityrepository; - -import org.springframework.data.jpa.repository.JpaRepository; - -import java.util.Optional; - -interface LastScrapedEventMarkerJpaRepository extends JpaRepository { - - Optional findByScraperConfigurationName(String configurationName); -} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java b/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java similarity index 62% rename from src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java rename to src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java index 97653d9..6cbfb2e 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchCommand.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java @@ -1,16 +1,16 @@ package hub.event.scrapers.core.runlog; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.List; -public class ErrorLogSearchCommand { +public class ErrorLogSearchQuery { private final List configurationNames; - private final LocalDateTime fromDate; - private final LocalDateTime toDate; + private final Instant fromDate; + private final Instant toDate; private final List errorCodes; private final String description; - public ErrorLogSearchCommand(List configurationNames, LocalDateTime fromDate, LocalDateTime toDate, List errorCodes, String description) { + public ErrorLogSearchQuery(List configurationNames, Instant fromDate, Instant toDate, List errorCodes, String description) { this.configurationNames = configurationNames; this.fromDate = fromDate; this.toDate = toDate; @@ -22,11 +22,11 @@ List configurationNames() { return configurationNames; } - LocalDateTime fromDate() { + Instant fromDate() { return fromDate; } - LocalDateTime toDate() { + Instant toDate() { return toDate; } diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java index dbe7713..3ed632b 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLog.java @@ -1,6 +1,6 @@ package hub.event.scrapers.core.runlog; -import java.time.LocalDateTime; +import java.time.Instant; -public record ScraperRunErrorLog(String configurationName, /* Instant */ LocalDateTime time, String errorCode, String description) { +public record ScraperRunErrorLog(String configurationName, Instant time, String errorCode, String description) { } diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java deleted file mode 100644 index 0570c16..0000000 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogFacade.java +++ /dev/null @@ -1,35 +0,0 @@ -package hub.event.scrapers.core.runlog; - -import org.springframework.stereotype.Service; - -import java.time.LocalDateTime; -import java.util.List; - -@Service -public class ScraperRunLogFacade { - private final ScraperRunLogRepository scraperRunLogRepository; - - public ScraperRunLogFacade(ScraperRunLogRepository scraperRunLogRepository) { - this.scraperRunLogRepository = scraperRunLogRepository; - } - - - public void logError(String configurationName, LocalDateTime time, String errorCode, String description) { - final ScraperRunErrorLog scraperRunErrorLog = new ScraperRunErrorLog(configurationName, time, errorCode, description); - scraperRunLogRepository.save(scraperRunErrorLog); - } - - public void logStatus(String configurationName, LocalDateTime startTime, LocalDateTime finishTime, Integer scannedEventCount, Integer errorCount) { - final ScraperRunStatusLog scraperRunStatusLog = new ScraperRunStatusLog(configurationName, startTime, finishTime, scannedEventCount, errorCount); - scraperRunLogRepository.save(scraperRunStatusLog); - } - - public List findAllErrorLog(ErrorLogSearchCommand errorLogSearchCommand) { - return scraperRunLogRepository.findAllErrorLog(errorLogSearchCommand); - } - public List findAllStatusLog(StatusLogSearchCommand statusLogSearchCommand) { - return scraperRunLogRepository.findAllStatusLog(statusLogSearchCommand); - } - - -} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java deleted file mode 100644 index 8c6265d..0000000 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunLogRepository.java +++ /dev/null @@ -1,14 +0,0 @@ -package hub.event.scrapers.core.runlog; - -import java.util.List; - -interface ScraperRunLogRepository { - - void save(ScraperRunStatusLog scraperRunStatusLog); - - void save (ScraperRunErrorLog scraperRunError); - - List findAllErrorLog(ErrorLogSearchCommand errorLogSearchCommand); - - List findAllStatusLog(StatusLogSearchCommand statusLogSearchCommand); -} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java index 0636fac..fe5d4ce 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLog.java @@ -1,7 +1,7 @@ package hub.event.scrapers.core.runlog; -import java.time.LocalDateTime; +import java.time.Instant; -public record ScraperRunStatusLog(String configurationName, LocalDateTime startTime, LocalDateTime finishTime, +public record ScraperRunStatusLog(String configurationName, Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { } diff --git a/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java deleted file mode 100644 index def9a67..0000000 --- a/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchCommand.java +++ /dev/null @@ -1,123 +0,0 @@ -package hub.event.scrapers.core.runlog; - -import java.time.LocalDateTime; -import java.util.List; - -public class StatusLogSearchCommand { - private final List configurationNames; - private final LocalDateTime startTime; - private final LocalDateTime finishTime; - private final Integer scannedEventCount; - private final Integer errorCount; - private final Boolean hasErrors; - private final Integer limit; - private final Integer offset; - - private StatusLogSearchCommand(List configurationNames, LocalDateTime startTime, LocalDateTime finishTime, Integer scannedEventCount, Integer errorCount, Boolean hasErrors, Integer offset, Integer limit) { - this.configurationNames = configurationNames; - this.startTime = startTime; - this.finishTime = finishTime; - this.scannedEventCount = scannedEventCount; - this.errorCount = errorCount; - this.hasErrors = hasErrors; - this.limit = limit; - this.offset = offset; - } - - public StatusLogSearchCommandBuilder builder() { - return new StatusLogSearchCommandBuilder(); - } - - List configurationNames() { - return configurationNames; - } - - LocalDateTime startTime() { - return startTime; - } - - LocalDateTime finishTime() { - return finishTime; - } - - Integer scannedEventCount() { - return scannedEventCount; - } - - Integer errorCount() { - return errorCount; - } - - Boolean hasErrors() { - return hasErrors; - } - - Integer limit() { - return limit; - } - - Integer offset() { - return offset; - } - - class StatusLogSearchCommandBuilder { - List configurationNames; - LocalDateTime startTime; - LocalDateTime finishTime; - Integer scannedEventCount; - Integer errorCount; - Boolean hasErrors; - Integer limit; - Integer offset; - - StatusLogSearchCommandBuilder() { - this.limit = 10; - this.offset = 0; - } - - public StatusLogSearchCommandBuilder setConfigurationNames(List configurationNames) { - this.configurationNames = configurationNames; - return this; - } - - public StatusLogSearchCommandBuilder setStartTime(LocalDateTime startTime) { - this.startTime = startTime; - return this; - } - - public StatusLogSearchCommandBuilder setFinishTime(LocalDateTime finishTime) { - this.finishTime = finishTime; - return this; - } - - public StatusLogSearchCommandBuilder setScannedEventCount(Integer scannedEventCount) { - this.scannedEventCount = scannedEventCount; - return this; - } - - public StatusLogSearchCommandBuilder setErrorCount(Integer errorCount) { - this.errorCount = errorCount; - return this; - } - - public StatusLogSearchCommandBuilder setHasErrors(Boolean hasErrors) { - this.hasErrors = hasErrors; - return this; - } - - public StatusLogSearchCommandBuilder setLimit(Integer limit) { - this.limit = limit; - return this; - } - - public StatusLogSearchCommandBuilder setOffset(Integer offset) { - this.offset = offset; - return this; - } - - public StatusLogSearchCommand build() { - return new StatusLogSearchCommand(configurationNames, startTime, finishTime, scannedEventCount, errorCount, hasErrors, offset, limit); - } - } - -} diff --git a/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java new file mode 100644 index 0000000..89886e2 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java @@ -0,0 +1,161 @@ +package hub.event.scrapers.core.runlog; + +import java.time.Instant; +import java.util.List; +import java.util.Objects; + +public class StatusLogSearchQuery { + private final List configurationNames; + private final Instant startTimeTo; + private final Instant startTimeFrom; + private final Instant finishTimeTo; + private final Instant finishTimeFrom; + private final Integer scannedEventCount; + private final Integer errorCount; + private final Boolean hasErrors; + private final Boolean hasScannedEvent; + private final Integer pageSize; + private final Integer page; + + private StatusLogSearchQuery(List configurationNames, Instant startTimeTo, Instant startTimeFrom, Instant finishTimeTo, Instant finishTime, Integer scannedEventCount, Integer errorCount, Boolean hasErrors, Boolean hasScannedEvent, Integer page, Integer pageSize) { + this.configurationNames = configurationNames; + this.startTimeTo = startTimeTo; + this.startTimeFrom = startTimeFrom; + this.finishTimeTo = finishTimeTo; + this.finishTimeFrom = finishTime; + this.scannedEventCount = scannedEventCount; + this.errorCount = errorCount; + this.hasErrors = hasErrors; + this.hasScannedEvent = hasScannedEvent; + this.pageSize = pageSize; + this.page = page; + } + + public StatusLogSearchCommandBuilder builder() { + return new StatusLogSearchCommandBuilder(); + } + + public List configurationNames() { + return configurationNames; + } + + public Instant startTimeFrom() { + return startTimeFrom; + } + + public Instant finishTimeFrom() { + return finishTimeFrom; + } + + public Instant startTimeTo() { + return startTimeTo; + } + + public Instant finishTimeTo() { + return finishTimeTo; + } + + public Integer scannedEventCount() { + return scannedEventCount; + } + + public Integer errorCount() { + return errorCount; + } + + public Boolean hasErrors() { + return hasErrors; + } + + public Boolean hasScannedEvent() { + return hasScannedEvent; + } + + public Integer pageSize() { + return pageSize; + } + + public Integer page() { + return page; + } + + public boolean hasConfigurationNames() { + return Objects.nonNull(configurationNames) && !configurationNames.isEmpty(); + } + + public boolean hasPageSetting() { + return Objects.nonNull(page) && Objects.nonNull(pageSize); + } + + static class StatusLogSearchCommandBuilder { + private List configurationNames; + private Instant startTimeTo; + private Instant startTimeFrom; + private Instant finishTimeTo; + private Instant finishTimeFrom; + private Integer scannedEventCount; + private Integer errorCount; + private Boolean hasErrors; + private Boolean hasScannedEvent; + private Integer pageSize; + private Integer page; + + StatusLogSearchCommandBuilder() { + } + + public StatusLogSearchCommandBuilder configurationNames(List configurationNames) { + this.configurationNames = configurationNames; + return this; + } + + public StatusLogSearchCommandBuilder startTimeTo(Instant startTimeTo) { + this.startTimeTo = startTimeTo; + return this; + } + + public StatusLogSearchCommandBuilder startTimeFrom(Instant startTimeFrom) { + this.startTimeFrom = startTimeFrom; + return this; + } + + public StatusLogSearchCommandBuilder finishTimeTo(Instant finishTimeTo) { + this.finishTimeTo = finishTimeTo; + return this; + } + + public StatusLogSearchCommandBuilder finishTimeFrom(Instant finishTimeFrom) { + this.finishTimeFrom = finishTimeFrom; + return this; + } + + public StatusLogSearchCommandBuilder scannedEventCount(Integer scannedEventCount) { + this.scannedEventCount = scannedEventCount; + return this; + } + + public StatusLogSearchCommandBuilder errorCount(Integer errorCount) { + this.errorCount = errorCount; + return this; + } + + public StatusLogSearchCommandBuilder hasErrors(Boolean hasErrors) { + this.hasErrors = hasErrors; + return this; + } + + public StatusLogSearchCommandBuilder page(Integer page) { + this.page = page; + return this; + } + + public StatusLogSearchCommandBuilder pageSize(Integer pageSize) { + this.pageSize = pageSize; + return this; + } + + public StatusLogSearchQuery build() { + return new StatusLogSearchQuery(configurationNames, startTimeTo, startTimeFrom, finishTimeTo, finishTimeFrom, scannedEventCount, errorCount, hasErrors, hasScannedEvent, page, pageSize); + } + } + +} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java b/src/main/java/hub/event/scrapers/core/scraper/LastScrapedEventMarker.java similarity index 86% rename from src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java rename to src/main/java/hub/event/scrapers/core/scraper/LastScrapedEventMarker.java index f926583..bcf0ae8 100644 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarker.java +++ b/src/main/java/hub/event/scrapers/core/scraper/LastScrapedEventMarker.java @@ -1,20 +1,20 @@ -package hub.event.scrapers.core; +package hub.event.scrapers.core.scraper; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.Objects; public class LastScrapedEventMarker { private final String scraperConfigurationName; - private final LocalDateTime runDateTime; + private final Instant runDateTime; private final String eventTitle; private final String marker; private final boolean complete; - public LastScrapedEventMarker(String scraperConfigurationName, LocalDateTime runDateTime, String eventTitle, String marker) { + public LastScrapedEventMarker(String scraperConfigurationName, Instant runDateTime, String eventTitle, String marker) { this(scraperConfigurationName, runDateTime, eventTitle,marker,false); } - public LastScrapedEventMarker(String scraperConfigurationName, LocalDateTime runDateTime, String eventTitle, String marker, boolean complete) { + public LastScrapedEventMarker(String scraperConfigurationName, Instant runDateTime, String eventTitle, String marker, boolean complete) { this.scraperConfigurationName = scraperConfigurationName; this.runDateTime = runDateTime; this.eventTitle = eventTitle; @@ -26,7 +26,7 @@ public String scraperConfigurationName() { return scraperConfigurationName; } - public LocalDateTime runDateTime() { + public Instant runDateTime() { return runDateTime; } diff --git a/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java b/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java new file mode 100644 index 0000000..b615ecf --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java @@ -0,0 +1,6 @@ +package hub.event.scrapers.core.scraper; + +import java.time.ZoneId; + +public record ScraperConfig(String configurationName, ZoneId timeZone, boolean isActive) { +} diff --git a/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java b/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java index 0edbf35..f09b470 100644 --- a/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java +++ b/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java @@ -6,14 +6,10 @@ import java.util.Collection; import java.util.Collections; -class EbiletScraper implements PageScraperPort { - @Override - public String configurationName() { - return null; - } +class EbiletScraper extends PageScraperPort { @Override - public Collection scrap() { + protected Collection scrap() { return Collections.emptyList(); } } diff --git a/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java b/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java index 87bb513..44c502d 100644 --- a/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java +++ b/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java @@ -5,16 +5,12 @@ import java.util.Collection; import java.util.Collections; -import java.util.List; -class EmpikBiletScraper implements PageScraperPort { - @Override - public String configurationName() { - return null; - } +class EmpikBiletScraper extends PageScraperPort { + @Override - public Collection scrap() { + protected Collection scrap() { return Collections.emptyList(); } } diff --git a/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java b/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java index 869fe1d..76971a2 100644 --- a/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java +++ b/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java @@ -7,15 +7,10 @@ import java.util.Collections; -class GoingAppScraper implements PageScraperPort { +class GoingAppScraper extends PageScraperPort { @Override - public String configurationName() { - return null; - } - - @Override - public Collection scrap() { + protected Collection scrap() { return Collections.emptyList(); } } diff --git a/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java b/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java index 696501e..6924298 100644 --- a/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java +++ b/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java @@ -7,16 +7,11 @@ import java.util.Collections; -class KupBilecikScraper implements PageScraperPort { +class KupBilecikScraper extends PageScraperPort { @Override - public String configurationName() { - return null; - } - - @Override - public Collection scrap() { + protected Collection scrap() { return Collections.emptyList(); } } diff --git a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java index 461c8ed..6d25c72 100644 --- a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java +++ b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java @@ -6,14 +6,9 @@ import java.util.Collection; import java.util.Collections; -class ProanimaScraper implements PageScraperPort { +class ProanimaScraper extends PageScraperPort { @Override - public String configurationName() { - return null; - } - - @Override - public Collection scrap() { + protected Collection scrap() { return Collections.emptyList(); } } diff --git a/src/main/java/hub/event/users/UserService.java b/src/main/java/hub/event/users/UserService.java index 12d4b75..191e24a 100644 --- a/src/main/java/hub/event/users/UserService.java +++ b/src/main/java/hub/event/users/UserService.java @@ -1,6 +1,4 @@ package hub.event.users; -import hub.event.auth.AuthService; - public class UserService { } diff --git a/src/main/java/hub/event/usersapi/UserController.java b/src/main/java/hub/event/usersapi/UserController.java index 8531ee7..1263f68 100644 --- a/src/main/java/hub/event/usersapi/UserController.java +++ b/src/main/java/hub/event/usersapi/UserController.java @@ -1,9 +1,4 @@ package hub.event.usersapi; -import hub.event.auth.AuthService; -import hub.event.users.UserService; -import hub.event.newsletter.NewsLetterService; -import hub.event.events.EventService; - class UserController { } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1fcf242..3edc9c0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,6 +2,7 @@ spring.profiles.active=prod spring.liquibase.change-log=classpath:database/changelog-root.xml spring.jpa.hibernate.show-sql=true +scrapers.run.cron.expression=0 0 0 * * * #--- spring.config.activate.on-profile=prod @@ -19,6 +20,4 @@ spring.datasource.driver-class-name=org.h2.Driver spring.datasource.url=jdbc:h2:mem:devdb spring.datasource.username=sa spring.datasource.password=sa -spring.h2.console.enabled=true - -scrapers.run.cron.expression=0 0 * * * \ No newline at end of file +spring.h2.console.enabled=true \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java b/src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java similarity index 90% rename from src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java rename to src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java index 1da4813..06741fd 100644 --- a/src/test/java/hub/event/scrapers/core/entityrepository/LastScrapedEventMarkerRepositoryTest.java +++ b/src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java @@ -1,6 +1,6 @@ -package hub.event.scrapers.core.entityrepository; +package hub.event.scrapers.core; -import hub.event.scrapers.core.LastScrapedEventMarker; +import hub.event.scrapers.core.scraper.LastScrapedEventMarker; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -9,7 +9,9 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; @@ -30,8 +32,8 @@ class LastScrapedEventMarkerRepositoryTest { @Test void saveTest() { //given - LastScrapedEventMarker eventMarker1 = new LastScrapedEventMarker("config1", LocalDateTime.now(), "title1", "marker1"); - LastScrapedEventMarker eventMarker2 = new LastScrapedEventMarker("config2", LocalDateTime.now(), "title2", "marker2"); + LastScrapedEventMarker eventMarker1 = new LastScrapedEventMarker("config1", LocalDateTime.now().toInstant(ZoneOffset.UTC), "title1", "marker1"); + LastScrapedEventMarker eventMarker2 = new LastScrapedEventMarker("config2", LocalDateTime.now().toInstant(ZoneOffset.UTC), "title2", "marker2"); //then @@ -73,7 +75,7 @@ void whenFindByNotExistsScraperConfigurationNameThenReturnEmptyMarker() { void whenFindByExistsScraperConfigurationNameThenReturnMarker() { //given final String scraperConfigurationName = "exists_scraper"; - final LocalDateTime localDateTime = LocalDateTime.now(); + final Instant localDateTime = LocalDateTime.now().toInstant(ZoneOffset.UTC); final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); lastScrapedEventMarkerEntity.setMarker("maker1"); diff --git a/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java b/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java new file mode 100644 index 0000000..a81a560 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java @@ -0,0 +1,168 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.runlog.ScraperRunErrorLog; +import hub.event.scrapers.core.runlog.ScraperRunStatusLog; +import hub.event.scrapers.core.scraper.LastScrapedEventMarker; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PageScraperPortTest { + @Mock + private ScraperRunLogRepository scraperRunLogRepository; + + @Mock + private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; + + @Captor + ArgumentCaptor lastScrapedEventMarkerArgumentCaptor; + + @InjectMocks + private PageScraperPort pageScraperPort = new PageScraperPort() { + @Override + protected Collection scrap() { + return Collections.emptyList(); + } + }; + + @Captor + private ArgumentCaptor scraperRunStatusLogArgumentCaptor; + @Captor + private ArgumentCaptor scraperRunErrorLogArgumentCaptor; + + @Test + void logError() { + //given + final String configurationName = pageScraperPort.configurationName(); + final Instant time = Instant.now(); + final String errorCode = "ER1"; + final String description = "Error 1 server is down"; + + //then + pageScraperPort.logError(time, errorCode, description); + + verify(scraperRunLogRepository).save(scraperRunErrorLogArgumentCaptor.capture()); + + final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.getValue(); + + assertNotNull(scraperRunErrorLog); + assertEquals(configurationName, scraperRunErrorLog.configurationName()); + assertEquals(time, scraperRunErrorLog.time()); + assertEquals(errorCode, scraperRunErrorLog.errorCode()); + assertEquals(description, scraperRunErrorLog.description()); + + } + + @Test + void logStatus() { + //given + final String configurationName = pageScraperPort.configurationName(); + final Instant startTime = Instant.now(); + final Instant finishTime = LocalDateTime.now().plusMinutes(10).toInstant(ZoneOffset.UTC); + final Integer scannedEventCount = 23; + final Integer errorCount = 1; + + + //then + pageScraperPort.logStatus(startTime, finishTime, scannedEventCount, errorCount); + + verify(scraperRunLogRepository).save(scraperRunStatusLogArgumentCaptor.capture()); + + final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.getValue(); + + assertNotNull(scraperRunStatusLog); + assertEquals(configurationName, scraperRunStatusLog.configurationName()); + assertEquals(startTime, scraperRunStatusLog.startTime()); + assertEquals(finishTime, scraperRunStatusLog.finishTime()); + assertEquals(scannedEventCount, scraperRunStatusLog.scannedEventCount()); + assertEquals(errorCount, scraperRunStatusLog.errorCount()); + + } + + @Nested + class LastScrapedEventMarkerTest { + @Test + void whenFoundLastScrapedEventMarkerThenReturnNotEmptyOptional() { + //given + final String scraperName = pageScraperPort.configurationName(); + final LastScrapedEventMarker lastScrapedEventMarker = new LastScrapedEventMarker(scraperName, LocalDateTime.now().minusDays(2).toInstant(ZoneOffset.UTC), "Event Title", "Marker", false); + //when + when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) + .thenReturn(Optional.of(lastScrapedEventMarker)); + //then + + assertDoesNotThrow(() -> { + final Optional lastScrapedEventMarkerResult = pageScraperPort.lastScrapedEventMarkerByConfigurationName(); + assertThat(lastScrapedEventMarkerResult) + .isNotEmpty() + .get() + .isEqualTo(lastScrapedEventMarker); + }); + + verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); + } + + @Test + void whenNotFoundLastScrapedEventMarkerThenReturnOptionalEmpty() { + //given + final String scraperName = pageScraperPort.configurationName(); + //when + when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) + .thenReturn(Optional.empty()); + //then + + assertDoesNotThrow(() -> { + final Optional lastScrapedEventMarkerResult = pageScraperPort.lastScrapedEventMarkerByConfigurationName(); + assertThat(lastScrapedEventMarkerResult) + .isEmpty(); + }); + + verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); + } + + @Test + void whenCreateLastScrapedEventMarkerThenReplaceNotCompleteOne() { + //given + final String scraperConfigurationName = pageScraperPort.configurationName(); + final LocalDateTime runTime = LocalDateTime.now(); + + final LastScrapedEventMarker previousLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime.minusDays(1).toInstant(ZoneOffset.UTC), "Title 1", "Marker", false); + final LastScrapedEventMarker newLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime.toInstant(ZoneOffset.UTC), "Title 2", "Marker 2", false); + + //when + when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, false)) + .thenReturn(Optional.of(previousLastScrapedEventMarker)); + + //then + pageScraperPort.saveLastScrapedEventMarker(runTime.toInstant(ZoneOffset.UTC), newLastScrapedEventMarker.eventTitle(), newLastScrapedEventMarker.marker()); + + verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperConfigurationName, false); + verify(lastScrapedEventMarkerRepository).drop(previousLastScrapedEventMarker); + + verify(lastScrapedEventMarkerRepository).store(lastScrapedEventMarkerArgumentCaptor.capture()); + + assertThat(lastScrapedEventMarkerArgumentCaptor.getValue()) + .isEqualTo(newLastScrapedEventMarker); + + } + } + +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java b/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java index a2fb9cd..a7c0d70 100644 --- a/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java +++ b/src/test/java/hub/event/scrapers/core/ScrapedEventBuilderTest.java @@ -7,8 +7,8 @@ import org.junit.jupiter.api.Test; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZoneId; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @@ -25,19 +25,19 @@ void whenUseBuilderThenEventIsBuiltCorrectly() throws EventDateInPastException, final String location = "Blue Bar"; final String description = "example description"; final Map metadata = Map.of("MetaKey1", "MetaValue1", "MetaKey2", "MetaValue2", "MetaKey3", "MetaValue3"); - final LocalDateTime scrapedTime = LocalDateTime.of(2022, 8, 21, 10, 0); - final LocalDate startDate = LocalDate.of(2022, 10, 12); + final LocalDate startDate = LocalDate.now().plusDays(2); final LocalTime startTime = LocalTime.of(14, 0); - final LocalDate endDate = LocalDate.of(2022, 10, 16); + final LocalDate endDate = LocalDate.now().plusDays(4); final LocalTime endTime = LocalTime.of(22, 30); + final ZoneId timeZone = ZoneId.systemDefault(); - final SingleEventDateWithLocation singleEventDateWithLocation = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, city, address, location); + final SingleEventDateWithLocation singleEventDateWithLocation = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, timeZone, city, address, location); - final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(startDate, startTime, city, address, location); + final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(startDate, startTime, timeZone, city, address, location); //then - ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(singleEventDateWithLocation) .title(title) .description(description) .sourceLink(sourceLink) @@ -46,30 +46,16 @@ void whenUseBuilderThenEventIsBuiltCorrectly() throws EventDateInPastException, .metadata("MetaKey3", "MetaValue3") .type("Inline Skating") .type("Skating Workshops") - .date(singleEventDateWithLocation) - .scrapedTime(scrapedTime) .build(); - ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(multipleEventDateWithLocations) .title(title) .description(description) .sourceLink(sourceLink) - .date(multipleEventDateWithLocations) - .scrapedTime(scrapedTime) - .build(); - - ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() - .title(title) - .description(description) - .sourceLink(sourceLink) - .date(multipleEventDateWithLocations) - .date(singleEventDateWithLocation) - .scrapedTime(scrapedTime) .build(); assertNotNull(scrapedEvent1); assertNotNull(scrapedEvent2); - assertNotNull(scrapedEvent3); assertEquals(title, scrapedEvent1.title()); assertEquals(description, scrapedEvent1.description()); @@ -80,16 +66,12 @@ void whenUseBuilderThenEventIsBuiltCorrectly() throws EventDateInPastException, assertTrue(scrapedEvent1.types().contains("Skating Workshops")); assertEquals(singleEventDateWithLocation, scrapedEvent1.singleEventDateWithLocation()); assertNull(scrapedEvent1.multipleEventDateWithLocations()); - assertEquals(scrapedTime, scrapedEvent1.scrapedTime()); assertFalse(scrapedEvent1.hasMultipleDateAndLocations()); assertEquals(multipleEventDateWithLocations, scrapedEvent2.multipleEventDateWithLocations()); assertNull(scrapedEvent2.singleEventDateWithLocation()); assertTrue(scrapedEvent2.hasMultipleDateAndLocations()); - assertEquals(multipleEventDateWithLocations, scrapedEvent3.multipleEventDateWithLocations()); - assertNull(scrapedEvent3.singleEventDateWithLocation()); - assertTrue(scrapedEvent3.hasMultipleDateAndLocations()); } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java index cecfc79..6a4efd3 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java @@ -1,6 +1,7 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; +import hub.event.scrapers.core.scraper.LastScrapedEventMarker; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -10,10 +11,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.time.LocalDateTime; -import java.util.Optional; +import java.time.ZoneId; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.*; @@ -42,7 +41,7 @@ void whenActivateScraperThatNotExistThenCreateNewRecord() { scraperFacade.activateScraperByConfigurationName(scraperName); verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository).create(scraperName, true); + verify(scraperConfigRepository).create(scraperName, ZoneId.systemDefault(), true); verify(scraperConfigRepository, never()).activate(scraperName); } @@ -56,7 +55,7 @@ void whenActivateScraperThatExistThenFinishSuccessfully() { scraperFacade.activateScraperByConfigurationName(scraperName); verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName, true); + verify(scraperConfigRepository, never()).create(scraperName, ZoneId.systemDefault(), true); verify(scraperConfigRepository).activate(scraperName); } @@ -70,7 +69,7 @@ void whenDeactivateNotExistsScraperThanThrowConfigNotExistsException() { assertThrows(ScraperConfigurationByNameNotExists.class, () -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName, true); + verify(scraperConfigRepository, never()).create(scraperName, ZoneId.systemDefault(), true); verify(scraperConfigRepository, never()).deactivate(scraperName); } @@ -84,76 +83,9 @@ void whenDeactivateExistsScraperThanFinishSuccessfully() { assertDoesNotThrow(() -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName, true); + verify(scraperConfigRepository, never()).create(scraperName, ZoneId.systemDefault(), true); verify(scraperConfigRepository).deactivate(scraperName); } } - @Nested - class LastScrapedEventMarkerTest { - @Test - void whenFoundLastScrapedEventMarkerThenReturnNotEmptyOptional() { - //given - final String scraperName = "scm4"; - final LastScrapedEventMarker lastScrapedEventMarker = new LastScrapedEventMarker(scraperName, LocalDateTime.now().minusDays(2), "Event Title", "Marker", false); - //when - when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) - .thenReturn(Optional.of(lastScrapedEventMarker)); - //then - - assertDoesNotThrow(() -> { - final Optional lastScrapedEventMarkerResult = scraperFacade.lastScrapedEventMarkerByConfigurationName(scraperName); - assertThat(lastScrapedEventMarkerResult) - .isNotEmpty() - .get() - .isEqualTo(lastScrapedEventMarker); - }); - - verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); - } - - @Test - void whenNotFoundLastScrapedEventMarkerThenReturnOptionalEmpty() { - //given - final String scraperName = "scm5"; - //when - when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) - .thenReturn(Optional.empty()); - //then - - assertDoesNotThrow(() -> { - final Optional lastScrapedEventMarkerResult = scraperFacade.lastScrapedEventMarkerByConfigurationName(scraperName); - assertThat(lastScrapedEventMarkerResult) - .isEmpty(); - }); - - verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); - } - - @Test - void whenCreateLastScrapedEventMarkerThenReplaceNotCompleteOne() { - //given - final String scraperConfigurationName = "name1"; - final LocalDateTime runTime = LocalDateTime.now(); - - final LastScrapedEventMarker previousLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime.minusDays(1), "Title 1", "Marker", false); - final LastScrapedEventMarker newLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime, "Title 2", "Marker 2", false); - - //when - when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, false)) - .thenReturn(Optional.of(previousLastScrapedEventMarker)); - - //then - scraperFacade.saveLastScrapedEventMarker(scraperConfigurationName, runTime, newLastScrapedEventMarker.eventTitle(), newLastScrapedEventMarker.marker()); - - verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperConfigurationName, false); - verify(lastScrapedEventMarkerRepository).drop(previousLastScrapedEventMarker); - - verify(lastScrapedEventMarkerRepository).store(lastScrapedEventMarkerArgumentCaptor.capture()); - - assertThat(lastScrapedEventMarkerArgumentCaptor.getValue()) - .isEqualTo(newLastScrapedEventMarker); - - } - } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java index 24441fb..3578fbf 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java @@ -2,6 +2,7 @@ import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; import hub.event.scrapers.core.exceptions.EventDateInPastException; +import hub.event.scrapers.core.scraper.ScraperConfig; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -10,16 +11,12 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) @@ -28,55 +25,11 @@ class ScraperRunServiceTest { @Mock private ScraperConfigRepository scraperConfigRepository; @Mock - private EventCandidateAnalyzer eventCandidateAnalyzer; - @Mock private EventFacadeAdapter eventFacadeAdapter; @Mock - private DuplicatedEventCandidateRepository duplicatedEventCandidateRepository; - @Mock private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; @Captor private ArgumentCaptor> scrapedEventListCaptor; - @Captor - private ArgumentCaptor> duplicatedEventCandidateListCaptor; - - @Test - void whenListContainsScraperWithoutConfigThenCreateNewConfig() { - //given - final PageScraperPort activeScraper = mock(PageScraperPort.class); - final PageScraperPort noConfigScraper1 = mock(PageScraperPort.class); - final PageScraperPort noConfigScraper2 = mock(PageScraperPort.class); - final PageScraperPort inactiveScraper = mock(PageScraperPort.class); - - - final String activeScraperName = "active1"; - final String inactiveScraperName = "inactive2"; - final String noConfigScraper1Name = "no-config-1"; - final String noConfigScraper2Name = "no-config-2-Atr-45"; - - final ScraperConfig activeScraperConfig = new ScraperConfig(activeScraperName, true); - final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, false); - - final List pageScrapers = Arrays.asList(activeScraper, noConfigScraper1, noConfigScraper2, inactiveScraper); - final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraperConfig, inactiveScraperConfig)); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); - - //when - when(activeScraper.configurationName()).thenReturn(activeScraperName); - when(noConfigScraper1.configurationName()).thenReturn(noConfigScraper1Name); - when(noConfigScraper2.configurationName()).thenReturn(noConfigScraper2Name); - when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); - when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - - //then - scraperRunService.start(); - - verify(scraperConfigRepository).allScraperConfigs(); - verify(scraperConfigRepository).create(noConfigScraper1Name, true); - verify(scraperConfigRepository).create(noConfigScraper2Name, true); - verify(scraperConfigRepository, never()).create(activeScraperName, true); - verify(scraperConfigRepository, never()).create(inactiveScraperName, true); - } @Test void whenListContainsInactiveScraperThenSkipRunIt() { @@ -89,94 +42,36 @@ void whenListContainsInactiveScraperThenSkipRunIt() { final String activeScraperName2 = "active2"; final String inactiveScraperName = "inactive2"; - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, false); + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, ZoneId.systemDefault(), true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, ZoneId.systemDefault(), true); + final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, ZoneId.systemDefault(), false); final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2, inactiveScraper); - final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config, inactiveScraperConfig)); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); + final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config, inactiveScraperConfig)); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); when(activeScraper2.configurationName()).thenReturn(activeScraperName2); when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); - when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - - //then - scraperRunService.start(); - - verify(scraperConfigRepository, never()).create(anyString(), anyBoolean()); - verify(activeScraper1).scrap(); - verify(activeScraper2).scrap(); - verify(inactiveScraper, never()).scrap(); - } - - @Test - void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() throws EventDateInPastException { - //given - final PageScraperPort activeScraper1 = mock(PageScraperPort.class); - final PageScraperPort activeScraper2 = mock(PageScraperPort.class); - - final String activeScraperName1 = "active1"; - final String activeScraperName2 = "active2"; - - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); - - final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() - .title("Title1") - .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) - .build(); - final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() - .title("Title2") - .description("Description2") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) - .build(); - final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() - .title("Title1") - .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Eden Prime", "Addres 134", "location 166")) - .build(); - final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); - - final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); - final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); - final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); - - final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); - - //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - - when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); + when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); - //then scraperRunService.start(); + verify(scraperConfigRepository, never()).create(anyString(), any(ZoneId.class), anyBoolean()); verify(activeScraper1).scrap(); verify(activeScraper2).scrap(); - verify(eventCandidateAnalyzer).analyze(scrapedEventList); - verify(eventFacadeAdapter).saveAll(scrapedEventList); - verifyNoInteractions(duplicatedEventCandidateRepository); + verify(inactiveScraper, never()).scrap(); } @Test - void whenScrapedEventContainsDuplicatedCandidateThenSaveAsDuplicatedEventCandidate() throws EventDateInPastException { + void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() throws EventDateInPastException { //given final PageScraperPort activeScraper1 = mock(PageScraperPort.class); final PageScraperPort activeScraper2 = mock(PageScraperPort.class); @@ -184,40 +79,26 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsDuplicatedEventCandida final String activeScraperName1 = "active1"; final String activeScraperName2 = "active2"; - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, ZoneId.systemDefault(), true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, ZoneId.systemDefault(), true); + final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); - final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Palaven", "Addres 1", "location 2")) .title("Title1") .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); - final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Thessia", "Addres 2", "location 22")) .title("Title2") .description("Description2") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) .build(); - final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Eden Prime", "Addres 134", "location 166")) .title("Title1") .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); - final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); - final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); - final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); - final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); - - analyzedEventCandidate1.addDuplicateCandidate(scrapedEvent3); - analyzedEventCandidate3.addDuplicateCandidate(scrapedEvent1); - final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); @@ -228,118 +109,12 @@ void whenScrapedEventContainsDuplicatedCandidateThenSaveAsDuplicatedEventCandida when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); - //then scraperRunService.start(); verify(activeScraper1).scrap(); verify(activeScraper2).scrap(); - verify(eventCandidateAnalyzer).analyze(scrapedEventList); - verify(eventFacadeAdapter).saveAll(scrapedEventListCaptor.capture()); - verify(duplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); - - assertThat(scrapedEventListCaptor.getValue()) - .hasSize(1) - .contains(scrapedEvent2) - .doesNotContain(scrapedEvent1, scrapedEvent3); - - assertThat(duplicatedEventCandidateListCaptor.getValue()) - .hasSize(2) - .extracting( - DuplicatedEventCandidate::getScrapedEvent, - eventCandidate -> eventCandidate.getDuplicatedCandidates().size(), - eventCandidate -> eventCandidate.getDuplicatedEvents().size() - ) - .contains( - tuple(scrapedEvent1, 1, 0), - tuple(scrapedEvent3, 1, 0) - ); - - } - - @Test - void whenScrapedEventContainsDuplicatedWithEventThenSaveAsDuplicatedEventCandidate() throws EventDateInPastException { - //given - final PageScraperPort activeScraper1 = mock(PageScraperPort.class); - final PageScraperPort activeScraper2 = mock(PageScraperPort.class); - - final String activeScraperName1 = "active1"; - final String activeScraperName2 = "active2"; - - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); - - final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() - .title("Title1") - .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) - .build(); - final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() - .title("Title2") - .description("Description2") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) - .build(); - final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() - .title("Title3") - .description("Description3") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Eden Prime", "Addres 133", "location 2")) - .build(); - - final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); - - final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); - final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); - final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); - - final ExistsEvent existsEvent = new ExistsEvent(1012, "title", "description", null, null); - - analyzedEventCandidate2.addDuplicateEvent(existsEvent); - - final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); - - //when - when(activeScraper1.configurationName()).thenReturn(activeScraperName1); - when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - - when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); - - when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - - when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); - - //then - scraperRunService.start(); - - verify(activeScraper1).scrap(); - verify(activeScraper2).scrap(); - verify(eventCandidateAnalyzer).analyze(scrapedEventList); - verify(eventFacadeAdapter).saveAll(scrapedEventListCaptor.capture()); - verify(duplicatedEventCandidateRepository).saveAll(duplicatedEventCandidateListCaptor.capture()); - - assertThat(scrapedEventListCaptor.getValue()) - .hasSize(2) - .contains(scrapedEvent1, scrapedEvent3) - .doesNotContain(scrapedEvent2); - - assertThat(duplicatedEventCandidateListCaptor.getValue()) - .hasSize(1) - .extracting( - DuplicatedEventCandidate::getScrapedEvent, - eventCandidate -> eventCandidate.getDuplicatedCandidates().size(), - eventCandidate -> eventCandidate.getDuplicatedEvents().size(), - eventCandidate -> eventCandidate.getDuplicatedEvents().stream().map(String::valueOf).collect(Collectors.joining(","))) - .contains( - tuple(scrapedEvent2, 0, 1, "1012") - ); + verify(eventFacadeAdapter).saveAll(scrapedEventList); } @Test @@ -351,36 +126,25 @@ void whenScrapedEventSavedThenMakeLastScrapedEventMarkerDraftActive() throws Eve final String activeScraperName1 = "active1"; final String activeScraperName2 = "active2"; - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, true); - final Collection scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); + final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, ZoneId.systemDefault(), true); + final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, ZoneId.systemDefault(), true); + final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); - final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder() + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Palaven", "Addres 1", "location 2")) .title("Title1") .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Palaven", "Addres 1", "location 2")) .build(); - final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder() + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Thessia", "Addres 2", "location 22")) .title("Title2") .description("Description2") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Thessia", "Addres 2", "location 22")) .build(); - final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder() + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Eden Prime", "Addres 134", "location 166")) .title("Title1") .description("Description1") - .scrapedTime(LocalDateTime.now()) - .date(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), "Eden Prime", "Addres 134", "location 166")) .build(); - final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); - - final AnalyzedEventCandidate analyzedEventCandidate1 = new AnalyzedEventCandidate(scrapedEvent1); - final AnalyzedEventCandidate analyzedEventCandidate2 = new AnalyzedEventCandidate(scrapedEvent2); - final AnalyzedEventCandidate analyzedEventCandidate3 = new AnalyzedEventCandidate(scrapedEvent3); final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventCandidateAnalyzer, eventFacadeAdapter, duplicatedEventCandidateRepository, lastScrapedEventMarkerRepository, pageScrapers); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers); //when when(activeScraper1.configurationName()).thenReturn(activeScraperName1); @@ -391,12 +155,9 @@ void whenScrapedEventSavedThenMakeLastScrapedEventMarkerDraftActive() throws Eve when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - when(eventCandidateAnalyzer.analyze(scrapedEventList)) - .thenReturn(Arrays.asList(analyzedEventCandidate1, analyzedEventCandidate2, analyzedEventCandidate3)); - //then scraperRunService.start(); - verify(lastScrapedEventMarkerRepository).makeDraftActive(List.of(activeScraperName1,activeScraperName2)); + verify(lastScrapedEventMarkerRepository).makeDraftActive(List.of(activeScraperName1, activeScraperName2)); } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java index 53bb1df..73001ff 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java @@ -6,6 +6,7 @@ import java.time.LocalDate; import java.time.LocalTime; +import java.time.ZoneId; import static org.assertj.core.api.Assertions.*; @@ -19,13 +20,14 @@ void whenBuildWithIncorrectInputThenThrows() throws EventDateInPastException { final String city = "Thessia"; final String address= "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; - final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(date, time, city, address, locationName); + final ZoneId timeZone = ZoneId.systemDefault(); + final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(date, time, timeZone, city, address, locationName); assertThatExceptionOfType(EventDateInPastException.class) - .isThrownBy(() -> MultipleEventDateWithLocations.create(dateInPast, time, city, address, locationName)); + .isThrownBy(() -> MultipleEventDateWithLocations.create(dateInPast, time, timeZone, city, address, locationName)); assertThatExceptionOfType(EventDateInPastException.class) - .isThrownBy(() -> multipleEventDateWithLocations.add(dateInPast, time, city, address, locationName)); + .isThrownBy(() -> multipleEventDateWithLocations.add(dateInPast, time, timeZone, city, address, locationName)); } @Test @@ -43,15 +45,15 @@ void whenCorrectInputThenBuildCorrectly() { final String locationName = "Black hole mirror club"; Assertions.assertThatNoException().isThrownBy(() -> { - MultipleEventDateWithLocations multipleDate = MultipleEventDateWithLocations.create(date1, time1, city1, address, locationName) - .add(date2, time2, city2, address, locationName) - .add(date3, time3, city3, address, locationName); + MultipleEventDateWithLocations multipleDate = MultipleEventDateWithLocations.create(date1, time1, ZoneId.systemDefault(), city1, address, locationName) + .add(date2, time2, ZoneId.systemDefault(), city2, address, locationName) + .add(date3, time3, ZoneId.systemDefault(), city3, address, locationName); assertThat(multipleDate).isNotNull(); assertThat(multipleDate.eventDateWithLocations()).isNotNull() .hasSize(3) - .extracting(EventDateWithLocation::startDate, EventDateWithLocation::startTime, EventDateWithLocation::city, EventDateWithLocation::address, EventDateWithLocation::locationName) + .extracting(EventDateWithLocation::startDate, EventDateWithLocation::startTime, EventDateWithLocation::city, EventDateWithLocation::address, EventDateWithLocation::locationName) .contains(tuple(date1, time1, city1, address, locationName), tuple(date2, time2, city2, address, locationName), tuple(date3, time3, city3, address, locationName) diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java index 0a748fb..6cd2340 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java @@ -7,6 +7,7 @@ import java.time.LocalDate; import java.time.LocalTime; +import java.time.ZoneId; import static org.junit.jupiter.api.Assertions.*; @@ -27,14 +28,15 @@ void whenBuildWithIncorrectInputThenThrows() { final String city = "Thessia"; final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; + final ZoneId timeZone = ZoneId.systemDefault(); - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, incorrectEndDate, correctEndTime, timeZone, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, correctEndDate, incorrectEndTime, timeZone, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, timeZone, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDateInPast, startTime, correctEndDate, correctEndTime, timeZone, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.single(startDate, startTime, endDateInPast, correctEndTime, timeZone, city, address, locationName)); } @Test @@ -46,9 +48,10 @@ void whenCorrectInputThenBuildCorrectly() { final String city = "Thessia"; final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; + final ZoneId timeZone = ZoneId.systemDefault(); assertDoesNotThrow(() -> { - SingleEventDateWithLocation singleDateContainsStartDateAndTime = SingleEventDateWithLocation.single(startDate, startTime, city, address, locationName); + SingleEventDateWithLocation singleDateContainsStartDateAndTime = SingleEventDateWithLocation.single(startDate, startTime, timeZone, city, address, locationName); assertNotNull(singleDateContainsStartDateAndTime); assertEquals(startDate, singleDateContainsStartDateAndTime.startDate()); @@ -64,7 +67,7 @@ void whenCorrectInputThenBuildCorrectly() { }); assertDoesNotThrow(() -> { - SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, startTime, endDate, endTime, city, address, locationName); + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.single(startDate, startTime, endDate, endTime, timeZone, city, address, locationName); assertNotNull(fullEventDate); assertEquals(startDate, fullEventDate.startDate()); @@ -96,13 +99,14 @@ void whenBuildWithIncorrectInputThenThrows() { final String city = "Thessia"; final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; + final ZoneId timeZone = ZoneId.systemDefault(); - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, incorrectEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, correctEndDate, incorrectEndTime, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, incorrectEndDate, correctEndTime, timeZone, city, address, locationName)); + assertThrows(EventDateEndDateTimeBeforeStartDateTimeException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, correctEndDate, incorrectEndTime, timeZone, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, correctEndTime, city, address, locationName)); - assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, endDateInPast, correctEndTime, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, timeZone, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDateInPast, startTime, correctEndDate, correctEndTime, timeZone, city, address, locationName)); + assertThrows(EventDateInPastException.class, () -> SingleEventDateWithLocation.period(startDate, startTime, endDateInPast, correctEndTime, timeZone, city, address, locationName)); } @@ -115,9 +119,10 @@ void whenCorrectInputThenBuildCorrectly() { final String city = "Thessia"; final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; + final ZoneId timeZone = ZoneId.systemDefault(); assertDoesNotThrow(() -> { - SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, startTime, endDate, city, address, locationName); + SingleEventDateWithLocation periodDateContainsStartDateAndTime = SingleEventDateWithLocation.period(startDate, startTime, endDate, timeZone, city, address, locationName); assertNotNull(periodDateContainsStartDateAndTime); @@ -133,7 +138,7 @@ void whenCorrectInputThenBuildCorrectly() { }); assertDoesNotThrow(() -> { - SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, city, address, locationName); + SingleEventDateWithLocation fullEventDate = SingleEventDateWithLocation.period(startDate, startTime, endDate, endTime, timeZone, city, address, locationName); assertNotNull(fullEventDate); diff --git a/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java b/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java deleted file mode 100644 index 71d4714..0000000 --- a/src/test/java/hub/event/scrapers/core/runlog/ScraperRunLogFacadeTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package hub.event.scrapers.core.runlog; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.time.LocalDateTime; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.verify; - -@ExtendWith(MockitoExtension.class) -class ScraperRunLogFacadeTest { - - @Mock - private ScraperRunLogRepository scraperRunLogRepository; - - @InjectMocks - private ScraperRunLogFacade scraperRunLogFacade; - - @Captor - private ArgumentCaptor scraperRunStatusLogArgumentCaptor; - @Captor - private ArgumentCaptor scraperRunErrorLogArgumentCaptor; - - @Test - void logError() { - //given - final String configurationName = "scraper1"; - final LocalDateTime time = LocalDateTime.now(); - final String errorCode = "ER1"; - final String description = "Error 1 server is down"; - - //then - scraperRunLogFacade.logError(configurationName, time, errorCode, description); - - verify(scraperRunLogRepository).save(scraperRunErrorLogArgumentCaptor.capture()); - - final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.getValue(); - - assertNotNull(scraperRunErrorLog); - assertEquals(configurationName, scraperRunErrorLog.configurationName()); - assertEquals(time, scraperRunErrorLog.time()); - assertEquals(errorCode, scraperRunErrorLog.errorCode()); - assertEquals(description, scraperRunErrorLog.description()); - - } - - @Test - void logStatus() { - //given - final String configurationName = "scraper1"; - final LocalDateTime startTime = LocalDateTime.now(); - final LocalDateTime finishTime = LocalDateTime.now().plusMinutes(10); - final Integer scannedEventCount = 23; - final Integer errorCount = 1; - - - //then - scraperRunLogFacade.logStatus(configurationName, startTime, finishTime, scannedEventCount, errorCount); - - verify(scraperRunLogRepository).save(scraperRunStatusLogArgumentCaptor.capture()); - - final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.getValue(); - - assertNotNull(scraperRunStatusLog); - assertEquals(configurationName, scraperRunStatusLog.configurationName()); - assertEquals(startTime, scraperRunStatusLog.startTime()); - assertEquals(finishTime, scraperRunStatusLog.finishTime()); - assertEquals(scannedEventCount, scraperRunStatusLog.scannedEventCount()); - assertEquals(errorCount, scraperRunStatusLog.errorCount()); - - } - -} \ No newline at end of file From d2022f75ffe6553f455fc08eb5b7f3da3697b67f Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Sun, 16 Oct 2022 14:06:05 +0200 Subject: [PATCH 06/10] feat: 17 added jpa repositories implementation, added missing tests --- pom.xml | 10 +- .../core/EntityLastScrapedEventMarker.java | 86 ++++ .../scrapers/core/EntityScraperConfig.java | 72 +++ .../core/EntityScraperRunErrorLog.java | 85 ++++ .../core/EntityScraperRunStatusLog.java | 97 ++++ .../JpaLastScrapedEventMarkerRepository.java | 21 + .../core/JpaScraperConfigRepository.java | 17 + ...java => JpaScraperRunErrorRepository.java} | 2 +- .../core/JpaScraperRunLogRepository.java | 7 + .../core/LastScrapedEventMarkerEntity.java | 65 --- ...astScrapedEventMarkerEntityRepository.java | 57 --- .../LastScrapedEventMarkerJpaRepository.java | 18 - .../LastScrapedEventMarkerRepository.java | 51 +- .../event/scrapers/core/PageScraperPort.java | 27 +- .../hub/event/scrapers/core/ScrapedEvent.java | 19 +- .../scrapers/core/ScraperConfigEntity.java | 49 -- .../core/ScraperConfigEntityRepository.java | 50 -- .../core/ScraperConfigJpaRepository.java | 14 - .../core/ScraperConfigRepository.java | 41 +- .../event/scrapers/core/ScraperFacade.java | 28 +- .../scrapers/core/ScraperIdNameCache.java | 37 ++ .../scrapers/core/ScraperLogQueryFacade.java | 22 + .../scrapers/core/ScraperLogRepository.java | 248 +++++++++ .../scrapers/core/ScraperQueryFacade.java | 22 - .../core/ScraperRunErrorJpaRepository.java | 6 - .../core/ScraperRunErrorLogEntity.java | 61 --- .../core/ScraperRunLogRepository.java | 186 ------- .../scrapers/core/ScraperRunService.java | 23 +- .../core/ScraperRunStatusLogEntity.java | 69 --- .../SingleEventDateWithLocation.java | 4 + .../core/runlog/ErrorLogSearchQuery.java | 92 +++- .../core/runlog/StatusLogSearchQuery.java | 51 +- .../scrapers/core/scraper/ScraperConfig.java | 2 +- .../event/scrapers/ebilet/EbiletScraper.java | 2 + .../empikbilet/EmpikBiletScraper.java | 2 + .../scrapers/goingapp/GoingAppScraper.java | 2 + .../kupbilecik/KupBilecikScraper.java | 3 +- .../scrapers/proanima/ProanimaScraper.java | 9 +- src/main/resources/application.properties | 5 +- .../resources/database/change/change_2.yaml | 197 ++++++++ .../LastScrapedEventMarkerRepositoryTest.java | 124 ++--- .../scrapers/core/PageScraperPortTest.java | 140 +++--- .../core/ScraperConfigRepositoryTest.java | 78 +++ .../scrapers/core/ScraperFacadeTest.java | 58 +-- .../core/ScraperLogQueryFacadeTest.java | 473 ++++++++++++++++++ .../scrapers/core/ScraperRunServiceTest.java | 340 ++++++++----- .../MultipleEventDateWithLocationsTest.java | 19 +- .../SingleEventDateWithLocationTest.java | 4 + .../database/scrapers/core/test_data_init.sql | 32 ++ 49 files changed, 2138 insertions(+), 989 deletions(-) create mode 100644 src/main/java/hub/event/scrapers/core/EntityLastScrapedEventMarker.java create mode 100644 src/main/java/hub/event/scrapers/core/EntityScraperConfig.java create mode 100644 src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java create mode 100644 src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java create mode 100644 src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/JpaScraperConfigRepository.java rename src/main/java/hub/event/scrapers/core/{ScraperRunLogJpaRepository.java => JpaScraperRunErrorRepository.java} (53%) create mode 100644 src/main/java/hub/event/scrapers/core/JpaScraperRunLogRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java delete mode 100644 src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java create mode 100644 src/main/java/hub/event/scrapers/core/ScraperLogRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java create mode 100644 src/main/resources/database/change/change_2.yaml create mode 100644 src/test/java/hub/event/scrapers/core/ScraperConfigRepositoryTest.java create mode 100644 src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java create mode 100644 src/test/resources/database/scrapers/core/test_data_init.sql diff --git a/pom.xml b/pom.xml index 6d5487f..cbb9b71 100644 --- a/pom.xml +++ b/pom.xml @@ -21,10 +21,6 @@ org.springframework.boot spring-boot-starter-data-jpa - - org.springframework.boot - spring-boot-starter-quartz - org.springframework.boot spring-boot-starter-web @@ -62,6 +58,12 @@ 3.23.1 test + + org.yaml + snakeyaml + 1.33 + + diff --git a/src/main/java/hub/event/scrapers/core/EntityLastScrapedEventMarker.java b/src/main/java/hub/event/scrapers/core/EntityLastScrapedEventMarker.java new file mode 100644 index 0000000..98ab746 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/EntityLastScrapedEventMarker.java @@ -0,0 +1,86 @@ +package hub.event.scrapers.core; + +import javax.persistence.*; +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +@Entity(name = "scraper_scraped_event_maker") +class EntityLastScrapedEventMarker implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer markerId; + @Column(nullable = false) + private Instant runTime; + private String eventTitle; + @Column(nullable = false) + private String marker; + @Column(nullable = false) + private Boolean isComplete; + @Column(nullable = false) + private Integer scraperId; + + EntityLastScrapedEventMarker() { + } + + Integer getMarkerId() { + return markerId; + } + + void setMarkerId(Integer markerId) { + this.markerId = markerId; + } + + Instant getRunTime() { + return runTime; + } + + void setRunTime(Instant runTime) { + this.runTime = runTime; + } + + String getEventTitle() { + return eventTitle; + } + + void setEventTitle(String eventTitle) { + this.eventTitle = eventTitle; + } + + String getMarker() { + return marker; + } + + void setMarker(String marker) { + this.marker = marker; + } + + Boolean getComplete() { + return isComplete; + } + + void setComplete(Boolean complete) { + isComplete = complete; + } + + Integer getScraperId() { + return scraperId; + } + + void setScraperId(Integer scraperId) { + this.scraperId = scraperId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityLastScrapedEventMarker that = (EntityLastScrapedEventMarker) o; + return Objects.equals(markerId, that.markerId) && Objects.equals(runTime, that.runTime) && Objects.equals(eventTitle, that.eventTitle) && Objects.equals(marker, that.marker) && Objects.equals(isComplete, that.isComplete) && Objects.equals(scraperId, that.scraperId); + } + + @Override + public int hashCode() { + return Objects.hash(markerId, runTime, eventTitle, marker, isComplete, scraperId); + } +} diff --git a/src/main/java/hub/event/scrapers/core/EntityScraperConfig.java b/src/main/java/hub/event/scrapers/core/EntityScraperConfig.java new file mode 100644 index 0000000..73ddbb9 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/EntityScraperConfig.java @@ -0,0 +1,72 @@ +package hub.event.scrapers.core; + +import javax.persistence.*; +import java.io.Serializable; +import java.util.Objects; + +@Entity(name = "scraper_config") +class EntityScraperConfig implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer scraperId; + @Column(unique = true) + private String configurationName; + @Column(nullable = false) + private String timeZone; + @Column(nullable = false) + private boolean isActive; + + EntityScraperConfig() { + } + + EntityScraperConfig(String configurationName, String timeZone, boolean isActive) { + this.configurationName = configurationName; + this.timeZone = timeZone; + this.isActive = isActive; + } + + Integer getScraperId() { + return scraperId; + } + + void setScraperId(Integer scraperId) { + this.scraperId = scraperId; + } + + String getConfigurationName() { + return configurationName; + } + + void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + String getTimeZone() { + return timeZone; + } + + void setTimeZone(String timeZone) { + this.timeZone = timeZone; + } + + boolean isActive() { + return isActive; + } + + void setActive(boolean active) { + isActive = active; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityScraperConfig that = (EntityScraperConfig) o; + return isActive == that.isActive && Objects.equals(scraperId, that.scraperId) && Objects.equals(configurationName, that.configurationName) && Objects.equals(timeZone, that.timeZone); + } + + @Override + public int hashCode() { + return Objects.hash(scraperId, configurationName, timeZone, isActive); + } +} diff --git a/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java new file mode 100644 index 0000000..33dff8e --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java @@ -0,0 +1,85 @@ +package hub.event.scrapers.core; + +import javax.persistence.*; +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +@Entity(name = "Scraper_error_log") +class EntityScraperRunErrorLog implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer logId; + + @Column(nullable = false, name = "error_time") + private Instant time; + + @Column(nullable = false) + private String errorCode; + private String description; + @Column(nullable = false) + private Integer scraperId; + + EntityScraperRunErrorLog() { + } + + EntityScraperRunErrorLog(Integer scraperId, Instant time, String errorCode, String description) { + this.scraperId = scraperId; + this.time = time; + this.errorCode = errorCode; + this.description = description; + } + + Integer getLogId() { + return logId; + } + + void setLogId(Integer logId) { + this.logId = logId; + } + + Instant getTime() { + return time; + } + + void setTime(Instant time) { + this.time = time; + } + + String getErrorCode() { + return errorCode; + } + + void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + String getDescription() { + return description; + } + + void setDescription(String description) { + this.description = description; + } + + Integer getScraperId() { + return scraperId; + } + + void setScraperId(Integer scraperId) { + this.scraperId = scraperId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityScraperRunErrorLog that = (EntityScraperRunErrorLog) o; + return Objects.equals(logId, that.logId) && Objects.equals(time, that.time) && Objects.equals(errorCode, that.errorCode) && Objects.equals(description, that.description) && Objects.equals(scraperId, that.scraperId); + } + + @Override + public int hashCode() { + return Objects.hash(logId, time, errorCode, description, scraperId); + } +} diff --git a/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java b/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java new file mode 100644 index 0000000..89bc013 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java @@ -0,0 +1,97 @@ +package hub.event.scrapers.core; + +import javax.persistence.*; +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +@Entity(name = "Scraper_status_log") +class EntityScraperRunStatusLog implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer logId; + @Column(nullable = false) + private Instant startTime; + @Column(nullable = false) + private Instant finishTime; + @Column(nullable = false) + private Integer scannedEventCount; + + @Column(nullable = false) + private Integer errorCount; + + @Column(nullable = false) + private Integer scraperId; + + EntityScraperRunStatusLog() { + } + + EntityScraperRunStatusLog(Integer scraperId, Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { + this.scraperId = scraperId; + this.startTime = startTime; + this.finishTime = finishTime; + this.scannedEventCount = scannedEventCount; + this.errorCount = errorCount; + } + + Integer getLogId() { + return logId; + } + + void setLogId(Integer logId) { + this.logId = logId; + } + + Instant getStartTime() { + return startTime; + } + + void setStartTime(Instant startTime) { + this.startTime = startTime; + } + + Instant getFinishTime() { + return finishTime; + } + + void setFinishTime(Instant finishTime) { + this.finishTime = finishTime; + } + + Integer getScannedEventCount() { + return scannedEventCount; + } + + void setScannedEventCount(Integer scannedEventCount) { + this.scannedEventCount = scannedEventCount; + } + + Integer getErrorCount() { + return errorCount; + } + + void setErrorCount(Integer errorCount) { + this.errorCount = errorCount; + } + + Integer getScraperId() { + return scraperId; + } + + void setScraperId(Integer scraperId) { + this.scraperId = scraperId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityScraperRunStatusLog that = (EntityScraperRunStatusLog) o; + return Objects.equals(logId, that.logId) && Objects.equals(startTime, that.startTime) && Objects.equals(finishTime, that.finishTime) && Objects.equals(scannedEventCount, that.scannedEventCount) && Objects.equals(errorCount, that.errorCount) && Objects.equals(scraperId, that.scraperId); + } + + @Override + public int hashCode() { + return Objects.hash(logId, startTime, finishTime, scannedEventCount, errorCount, scraperId); + } +} diff --git a/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java new file mode 100644 index 0000000..00f1c1d --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java @@ -0,0 +1,21 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import javax.transaction.Transactional; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +interface JpaLastScrapedEventMarkerRepository extends JpaRepository { + + Optional findByScraperId(Integer scraperId); + + @Modifying + @Query("UPDATE scraper_scraped_event_maker m SET m.isComplete = true WHERE m.isComplete = false AND m.runTime >= :date AND m.scraperId IN :configurationIds") + @Transactional + void updateSetAllActiveById(@Param("configurationIds") List configurationIds, @Param("date")Instant date); +} diff --git a/src/main/java/hub/event/scrapers/core/JpaScraperConfigRepository.java b/src/main/java/hub/event/scrapers/core/JpaScraperConfigRepository.java new file mode 100644 index 0000000..542e64e --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/JpaScraperConfigRepository.java @@ -0,0 +1,17 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import javax.transaction.Transactional; + +interface JpaScraperConfigRepository extends JpaRepository { + + @Modifying + @Transactional + @Query("update scraper_config e set e.isActive = :state where e.scraperId = :id") + void setActiveState(@Param("id") Integer scraperId, @Param("state") boolean activeState); + +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java b/src/main/java/hub/event/scrapers/core/JpaScraperRunErrorRepository.java similarity index 53% rename from src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java rename to src/main/java/hub/event/scrapers/core/JpaScraperRunErrorRepository.java index eaa8feb..10f9606 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunLogJpaRepository.java +++ b/src/main/java/hub/event/scrapers/core/JpaScraperRunErrorRepository.java @@ -3,5 +3,5 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; -interface ScraperRunLogJpaRepository extends JpaRepository, JpaSpecificationExecutor { +interface JpaScraperRunErrorRepository extends JpaRepository, JpaSpecificationExecutor { } diff --git a/src/main/java/hub/event/scrapers/core/JpaScraperRunLogRepository.java b/src/main/java/hub/event/scrapers/core/JpaScraperRunLogRepository.java new file mode 100644 index 0000000..6c38f4d --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/JpaScraperRunLogRepository.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; + +interface JpaScraperRunLogRepository extends JpaRepository, JpaSpecificationExecutor { +} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java deleted file mode 100644 index 7602e05..0000000 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntity.java +++ /dev/null @@ -1,65 +0,0 @@ -package hub.event.scrapers.core; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.IdClass; -import java.io.Serializable; -import java.time.Instant; - -@Entity -@IdClass(LastScrapedEventMarkerEntity.class) -class LastScrapedEventMarkerEntity implements Serializable { - @Id - private String scraperConfigurationName; - @Column(nullable = false) - private Instant runDateTime; - private String eventTitle; - @Column(nullable = false) - private String marker; - @Id - private Boolean complete; - - LastScrapedEventMarkerEntity() { - } - - String getScraperConfigurationName() { - return scraperConfigurationName; - } - - void setScraperConfigurationName(String scraperConfigurationName) { - this.scraperConfigurationName = scraperConfigurationName; - } - - Instant getRunDateTime() { - return runDateTime; - } - - void setRunDateTime(Instant runDateTime) { - this.runDateTime = runDateTime; - } - - String getEventTitle() { - return eventTitle; - } - - void setEventTitle(String eventTitle) { - this.eventTitle = eventTitle; - } - - String getMarker() { - return marker; - } - - void setMarker(String marker) { - this.marker = marker; - } - - Boolean getComplete() { - return complete; - } - - void setComplete(Boolean complete) { - this.complete = complete; - } -} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java deleted file mode 100644 index 8a63dbd..0000000 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerEntityRepository.java +++ /dev/null @@ -1,57 +0,0 @@ -package hub.event.scrapers.core; - -import hub.event.scrapers.core.scraper.LastScrapedEventMarker; -import org.springframework.stereotype.Repository; - -import java.time.Instant; -import java.util.List; -import java.util.Optional; - -@Repository -class LastScrapedEventMarkerEntityRepository implements LastScrapedEventMarkerRepository { - private final LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerJpaRepository; - - LastScrapedEventMarkerEntityRepository(LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository) { - this.lastScrapedEventMarkerJpaRepository = lastScrapedEventMarkerEntityRepository; - } - - public void store(LastScrapedEventMarker lastScrapedEventMarker) { - final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = mapToEntity(lastScrapedEventMarker); - lastScrapedEventMarkerJpaRepository.save(lastScrapedEventMarkerEntity); - } - - @Override - public void drop(LastScrapedEventMarker lastScrapedEventMarker) { - lastScrapedEventMarkerJpaRepository.deleteById(lastScrapedEventMarker.scraperConfigurationName()); - } - - @Override - public void makeDraftActive(List configurationNameList) { - lastScrapedEventMarkerJpaRepository.updateSetAllActiveById(configurationNameList); - } - - public Optional findByScraperConfigurationName(String configurationName, boolean complete) { - final Optional scraperConfigurationName = lastScrapedEventMarkerJpaRepository.findByScraperConfigurationName(configurationName); - return scraperConfigurationName.map(this::mapToMaker); - } - - private LastScrapedEventMarkerEntity mapToEntity(LastScrapedEventMarker lastScrapedEventMarker) { - final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); - lastScrapedEventMarkerEntity.setMarker(lastScrapedEventMarker.marker()); - lastScrapedEventMarkerEntity.setEventTitle(lastScrapedEventMarker.eventTitle()); - lastScrapedEventMarkerEntity.setRunDateTime(lastScrapedEventMarker.runDateTime()); - lastScrapedEventMarkerEntity.setScraperConfigurationName(lastScrapedEventMarker.scraperConfigurationName()); - return lastScrapedEventMarkerEntity; - } - - private LastScrapedEventMarker mapToMaker(LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity) { - final String marker = lastScrapedEventMarkerEntity.getMarker(); - final Instant date = lastScrapedEventMarkerEntity.getRunDateTime(); - final String title = lastScrapedEventMarkerEntity.getEventTitle(); - final String configurationName = lastScrapedEventMarkerEntity.getScraperConfigurationName(); - final Boolean isComplete = lastScrapedEventMarkerEntity.getComplete(); - - return new LastScrapedEventMarker(configurationName, date, title, marker, isComplete); - } - -} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java deleted file mode 100644 index 0da50a2..0000000 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerJpaRepository.java +++ /dev/null @@ -1,18 +0,0 @@ -package hub.event.scrapers.core; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; - -import java.util.List; -import java.util.Optional; - -interface LastScrapedEventMarkerJpaRepository extends JpaRepository { - - Optional findByScraperConfigurationName(String configurationName); - - @Modifying - @Query("update LastScrapedEventMarkerEntity e set e.complete=true where e.scraperConfigurationName in :ids") - void updateSetAllActiveById(@Param("ids") List configurationNameList); -} diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java index 0d0bad3..43fdf15 100644 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java @@ -1,17 +1,58 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.scraper.LastScrapedEventMarker; +import org.springframework.stereotype.Repository; +import java.time.Instant; +import java.time.ZonedDateTime; import java.util.List; import java.util.Optional; -public interface LastScrapedEventMarkerRepository { - void store(LastScrapedEventMarker lastScrapedEventMarker); +@Repository +class LastScrapedEventMarkerRepository { + private final JpaLastScrapedEventMarkerRepository jpaLastScrapedEventMarkerRepository; + private final ScraperIdNameCache scraperIdNameCache; - void drop(LastScrapedEventMarker lastScrapedEventMarker); + LastScrapedEventMarkerRepository(JpaLastScrapedEventMarkerRepository lastScrapedEventMarkerEntityRepository, ScraperIdNameCache scraperIdNameCache) { + this.jpaLastScrapedEventMarkerRepository = lastScrapedEventMarkerEntityRepository; + this.scraperIdNameCache = scraperIdNameCache; + } - void makeDraftActive(List configurationNameList); + void store(LastScrapedEventMarker lastScrapedEventMarker) { + final EntityLastScrapedEventMarker entityLastScrapedEventMarker = mapToEntity(lastScrapedEventMarker); + jpaLastScrapedEventMarkerRepository.save(entityLastScrapedEventMarker); + } - Optional findByScraperConfigurationName(String configurationName, boolean complete); + void markAllMarkersByIdsAsActive(List ids) { + jpaLastScrapedEventMarkerRepository.updateSetAllActiveById(ids, ZonedDateTime.now().minusDays(1).toInstant()); + } + + Optional findLastCompletedByScraperConfigurationId(Integer scraperId) { + final Optional scraperConfigurationName = jpaLastScrapedEventMarkerRepository.findByScraperId(scraperId); + return scraperConfigurationName.map(this::mapToMaker); + } + + private EntityLastScrapedEventMarker mapToEntity(LastScrapedEventMarker lastScrapedEventMarker) { + final Integer scraperId = scraperIdNameCache.getIdByScraperName(lastScrapedEventMarker.scraperConfigurationName()); + final EntityLastScrapedEventMarker entityLastScrapedEventMarker = new EntityLastScrapedEventMarker(); + + entityLastScrapedEventMarker.setMarker(lastScrapedEventMarker.marker()); + entityLastScrapedEventMarker.setEventTitle(lastScrapedEventMarker.eventTitle()); + entityLastScrapedEventMarker.setRunTime(lastScrapedEventMarker.runDateTime()); + entityLastScrapedEventMarker.setScraperId(scraperId); + entityLastScrapedEventMarker.setComplete(lastScrapedEventMarker.complete()); + + return entityLastScrapedEventMarker; + } + + private LastScrapedEventMarker mapToMaker(EntityLastScrapedEventMarker entityLastScrapedEventMarker) { + final String marker = entityLastScrapedEventMarker.getMarker(); + final Instant date = entityLastScrapedEventMarker.getRunTime(); + final String title = entityLastScrapedEventMarker.getEventTitle(); + final String configurationName = scraperIdNameCache.getScraperNameById(entityLastScrapedEventMarker.getScraperId()); + final Boolean isComplete = entityLastScrapedEventMarker.getComplete(); + + return new LastScrapedEventMarker(configurationName, date, title, marker, isComplete); + } } diff --git a/src/main/java/hub/event/scrapers/core/PageScraperPort.java b/src/main/java/hub/event/scrapers/core/PageScraperPort.java index 564b970..629f639 100644 --- a/src/main/java/hub/event/scrapers/core/PageScraperPort.java +++ b/src/main/java/hub/event/scrapers/core/PageScraperPort.java @@ -12,39 +12,40 @@ public abstract class PageScraperPort { @Autowired - private ScraperRunLogRepository scraperRunLogRepository; + private ScraperLogRepository scraperLogRepository; @Autowired private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; - String configurationName() { + @Autowired + private ScraperIdNameCache scraperIdNameCache; + + protected String configurationName() { // chyba załatwi sprawę w modularnym monolicie return this.getClass().getName(); } protected abstract Collection scrap(); - ZoneId timeZone() { + protected ZoneId timeZone() { return ZoneId.systemDefault(); } - void logError(Instant time, String errorCode, String description) { + protected void logError(Instant time, String errorCode, String description) { final ScraperRunErrorLog scraperRunErrorLog = new ScraperRunErrorLog(configurationName(), time, errorCode, description); - scraperRunLogRepository.save(scraperRunErrorLog); + scraperLogRepository.save(scraperRunErrorLog); } - void logStatus(Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { + protected void logStatus(Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { final ScraperRunStatusLog scraperRunStatusLog = new ScraperRunStatusLog(configurationName(), startTime, finishTime, scannedEventCount, errorCount); - scraperRunLogRepository.save(scraperRunStatusLog); + scraperLogRepository.save(scraperRunStatusLog); } - Optional lastScrapedEventMarkerByConfigurationName() { - return lastScrapedEventMarkerRepository.findByScraperConfigurationName(configurationName(), true); + protected Optional lastScrapedEventMarkerByConfigurationName() { + Integer id = scraperIdNameCache.getIdByScraperName(configurationName()); + return lastScrapedEventMarkerRepository.findLastCompletedByScraperConfigurationId(id); } - void saveLastScrapedEventMarker(Instant runDateTime, String eventTitle, String marker) { - final Optional savedScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(configurationName(), false); - savedScrapedEventMarker.ifPresent(lastScrapedEventMarkerRepository::drop); - + protected void saveLastScrapedEventMarker(Instant runDateTime, String eventTitle, String marker) { final LastScrapedEventMarker newScrapedEventMarkerToSave = new LastScrapedEventMarker(configurationName(), runDateTime, eventTitle, marker); lastScrapedEventMarkerRepository.store(newScrapedEventMarkerToSave); } diff --git a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java index a59ecd8..85a7387 100644 --- a/src/main/java/hub/event/scrapers/core/ScrapedEvent.java +++ b/src/main/java/hub/event/scrapers/core/ScrapedEvent.java @@ -7,13 +7,13 @@ public class ScrapedEvent { - private Map metadata; - private String title; - private String description; - private String sourceLink; - private List types; - private SingleEventDateWithLocation singleEventDateWithLocation; - private MultipleEventDateWithLocations multipleEventDateWithLocations; + private final Map metadata; + private final String title; + private final String description; + private final String sourceLink; + private final List types; + private final SingleEventDateWithLocation singleEventDateWithLocation; + private final MultipleEventDateWithLocations multipleEventDateWithLocations; // nowa klasa na powtarzający się event // Period - od kiedy do kiedy @@ -31,9 +31,6 @@ private ScrapedEvent(String title, String description, String sourceLink, Single this.multipleEventDateWithLocations = multipleEventDateWithLocations; } - private ScrapedEvent() { - } - public static ScrapedEventBuilder builder(SingleEventDateWithLocation singleEventDateWithLocation) { return new ScrapedEventBuilder(singleEventDateWithLocation); } @@ -88,7 +85,7 @@ private ScrapedEventBuilder() { this.types = new ArrayList<>(); } - private ScrapedEventBuilder(SingleEventDateWithLocation singleEventDateWithLocation) { + public ScrapedEventBuilder(SingleEventDateWithLocation singleEventDateWithLocation) { this(); this.singleEventDateWithLocation = singleEventDateWithLocation; } diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java b/src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java deleted file mode 100644 index b215d81..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperConfigEntity.java +++ /dev/null @@ -1,49 +0,0 @@ -package hub.event.scrapers.core; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; - -@Entity -class ScraperConfigEntity { - private String scraperName; - private String timeZone; - private boolean activeState; - @Id - private String configurationName; - @Column(nullable = false) - private boolean isActive; - - public ScraperConfigEntity() { - } - - public ScraperConfigEntity(String scraperName,String timeZone, boolean activeState) { - this.scraperName = scraperName; - this.timeZone = timeZone; - this.activeState = activeState; - } - - String getConfigurationName() { - return configurationName; - } - - void setConfigurationName(String configurationName) { - this.configurationName = configurationName; - } - - String getTimeZone() { - return timeZone; - } - - void setTimeZone(String timeZone) { - this.timeZone = timeZone; - } - - boolean isActive() { - return isActive; - } - - void setActive(boolean active) { - isActive = active; - } -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java deleted file mode 100644 index 65130e1..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperConfigEntityRepository.java +++ /dev/null @@ -1,50 +0,0 @@ -package hub.event.scrapers.core; - -import hub.event.scrapers.core.scraper.ScraperConfig; -import org.springframework.stereotype.Repository; - -import java.time.ZoneId; -import java.util.List; - -@Repository -class ScraperConfigEntityRepository implements ScraperConfigRepository { - private final ScraperConfigJpaRepository scraperConfigJpaRepository; - - ScraperConfigEntityRepository(ScraperConfigJpaRepository scraperConfigJpaRepository) { - this.scraperConfigJpaRepository = scraperConfigJpaRepository; - } - - @Override - public boolean exists(String scraperName) { - return scraperConfigJpaRepository.existsById(scraperName); - } - - @Override - public void create(String scraperName, ZoneId timeZone ,boolean activeState) { - final ScraperConfigEntity scraperConfigEntity = new ScraperConfigEntity(scraperName, timeZone.toString(), activeState); - scraperConfigJpaRepository.save(scraperConfigEntity); - } - - @Override - public void activate(String scraperConfigurationName) { - scraperConfigJpaRepository.setActiveState(scraperConfigurationName, true); - } - - @Override - public void deactivate(String scraperConfigurationName) { - scraperConfigJpaRepository.setActiveState(scraperConfigurationName, false); - } - - @Override - public List allScraperConfigs() { - return scraperConfigJpaRepository.findAll() - .stream() - .map(this::mapToScraperConfig) - .toList(); - } - - private ScraperConfig mapToScraperConfig(ScraperConfigEntity scraperConfigEntity) { - ZoneId timeZone = ZoneId.of(scraperConfigEntity.getTimeZone()); - return new ScraperConfig(scraperConfigEntity.getConfigurationName(), timeZone, scraperConfigEntity.isActive()); - } -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java deleted file mode 100644 index 136d0f5..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperConfigJpaRepository.java +++ /dev/null @@ -1,14 +0,0 @@ -package hub.event.scrapers.core; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; - -interface ScraperConfigJpaRepository extends JpaRepository { - - @Modifying - @Query("update ScraperConfigEntity e set e.activeState = :state where e.scraperName = :name" ) - void setActiveState(@Param("name") String scraperConfigurationName, @Param("state") boolean activeState); - -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java index b81d1b0..accc246 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java +++ b/src/main/java/hub/event/scrapers/core/ScraperConfigRepository.java @@ -1,20 +1,47 @@ package hub.event.scrapers.core; - import hub.event.scrapers.core.scraper.ScraperConfig; +import org.springframework.stereotype.Repository; import java.time.ZoneId; import java.util.List; -public interface ScraperConfigRepository { +@Repository +class ScraperConfigRepository { + private final JpaScraperConfigRepository jpaScraperConfigRepository; + + ScraperConfigRepository(JpaScraperConfigRepository jpaScraperConfigRepository) { + this.jpaScraperConfigRepository = jpaScraperConfigRepository; + } + + + boolean exists(Integer scraperId) { + return jpaScraperConfigRepository.existsById(scraperId); + } - boolean exists(String scraperName); + ScraperConfig create(String scraperName, ZoneId timeZone, boolean activeState) { + final EntityScraperConfig entityScraperConfig = new EntityScraperConfig(scraperName, timeZone.toString(), activeState); + final EntityScraperConfig savedEntity = jpaScraperConfigRepository.save(entityScraperConfig); + return mapToScraperConfig(savedEntity); + } - void create(String scraperName, ZoneId timeZone, boolean activeState); + void activate(Integer scraperId) { + jpaScraperConfigRepository.setActiveState(scraperId, true); + } - void activate(String scraperConfigurationName); + void deactivate(Integer scraperId) { + jpaScraperConfigRepository.setActiveState(scraperId, false); + } - void deactivate(String scraperConfigurationName); + public List allScraperConfigs() { + return jpaScraperConfigRepository.findAll() + .stream() + .map(this::mapToScraperConfig) + .toList(); + } - List allScraperConfigs(); + private ScraperConfig mapToScraperConfig(EntityScraperConfig entityScraperConfig) { + ZoneId timeZone = ZoneId.of(entityScraperConfig.getTimeZone()); + return new ScraperConfig(entityScraperConfig.getScraperId(), entityScraperConfig.getConfigurationName(), timeZone, entityScraperConfig.isActive()); + } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperFacade.java b/src/main/java/hub/event/scrapers/core/ScraperFacade.java index 86e34ed..47e29ec 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperFacade.java +++ b/src/main/java/hub/event/scrapers/core/ScraperFacade.java @@ -3,32 +3,32 @@ import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; import org.springframework.stereotype.Service; -import java.time.ZoneId; - @Service public class ScraperFacade { private final ScraperConfigRepository scraperConfigRepository; + private final ScraperIdNameCache scraperIdNameCache; - public ScraperFacade(ScraperConfigRepository scraperConfigRepository) { + public ScraperFacade(ScraperConfigRepository scraperConfigRepository, ScraperIdNameCache scraperIdNameCache) { this.scraperConfigRepository = scraperConfigRepository; + this.scraperIdNameCache = scraperIdNameCache; } - public void activateScraperByConfigurationName(String scraperName) { - //TODO nightmare throw exception if config not exists - if (scraperConfigRepository.exists(scraperName)) { - scraperConfigRepository.activate(scraperName); - } else { - scraperConfigRepository.create(scraperName, ZoneId.systemDefault(),true); - } + public void activateScraperByConfigurationName(String scraperName) throws ScraperConfigurationByNameNotExists { + Integer scraperId = scraperIdNameCache.getIdByScraperName(scraperName); + validateScraperExists(scraperName, scraperId); + scraperConfigRepository.activate(scraperId); } public void deactivateScraperByConfigurationName(String scraperName) throws ScraperConfigurationByNameNotExists { - if (!scraperConfigRepository.exists(scraperName)) { + Integer scraperId = scraperIdNameCache.getIdByScraperName(scraperName); + validateScraperExists(scraperName, scraperId); + scraperConfigRepository.deactivate(scraperId); + } + + private void validateScraperExists(String scraperName, Integer scraperId) throws ScraperConfigurationByNameNotExists { + if (!scraperConfigRepository.exists(scraperId)) { throw new ScraperConfigurationByNameNotExists(scraperName); } - - scraperConfigRepository.deactivate(scraperName); } - } diff --git a/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java b/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java new file mode 100644 index 0000000..2124d3c --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java @@ -0,0 +1,37 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.scraper.ScraperConfig; +import org.springframework.stereotype.Component; + +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; + +@Component +class ScraperIdNameCache { + private final ConcurrentHashMap nameToIdMap; + private final ConcurrentHashMap idToNameMap; + + ScraperIdNameCache() { + nameToIdMap = new ConcurrentHashMap<>(); + idToNameMap = new ConcurrentHashMap<>(); + } + + Integer getIdByScraperName(String scraperConfigurationName) { + return nameToIdMap.get(scraperConfigurationName); + } + + String getScraperNameById(Integer scraperId) { + return idToNameMap.get(scraperId); + } + + void add(Collection scraperConfigs) { + for (ScraperConfig scraperConfig : scraperConfigs) { + add(scraperConfig); + } + } + + public void add(ScraperConfig scraperConfig) { + idToNameMap.put(scraperConfig.scraperId(), scraperConfig.configurationName()); + nameToIdMap.put(scraperConfig.configurationName(), scraperConfig.scraperId()); + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java b/src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java new file mode 100644 index 0000000..4cd298d --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java @@ -0,0 +1,22 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.runlog.*; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class ScraperLogQueryFacade { + private final ScraperLogRepository scraperLogRepository; + + public ScraperLogQueryFacade(ScraperLogRepository scraperLogRepository) { + this.scraperLogRepository = scraperLogRepository; + } + public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { + return scraperLogRepository.findAllErrorLog(errorLogSearchQuery); + } + public List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { + return scraperLogRepository.findAllStatusLog(statusLogSearchQuery); + } + +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java b/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java new file mode 100644 index 0000000..c3d74cb --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java @@ -0,0 +1,248 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.runlog.ErrorLogSearchQuery; +import hub.event.scrapers.core.runlog.ScraperRunErrorLog; +import hub.event.scrapers.core.runlog.ScraperRunStatusLog; +import hub.event.scrapers.core.runlog.StatusLogSearchQuery; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Repository; + +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +@Repository +class ScraperLogRepository { + + private final JpaScraperRunLogRepository jpaScraperRunLogRepository; + private final JpaScraperRunErrorRepository jpaScraperRunErrorRepository; + + private final ScraperIdNameCache scraperIdNameCache; + + @Autowired + public ScraperLogRepository(JpaScraperRunLogRepository jpaScraperRunLogRepository, JpaScraperRunErrorRepository jpaScraperRunErrorRepository, ScraperIdNameCache scraperIdNameCache) { + this.jpaScraperRunLogRepository = jpaScraperRunLogRepository; + this.jpaScraperRunErrorRepository = jpaScraperRunErrorRepository; + this.scraperIdNameCache = scraperIdNameCache; + } + + void save(ScraperRunStatusLog scraperRunStatusLog) { + EntityScraperRunStatusLog entityScraperRunStatusLog = mapToEntity(scraperRunStatusLog); + jpaScraperRunLogRepository.save(entityScraperRunStatusLog); + } + + void save(ScraperRunErrorLog scraperRunError) { + EntityScraperRunErrorLog entityScraperRunErrorLog = mapToEntity(scraperRunError); + jpaScraperRunErrorRepository.save(entityScraperRunErrorLog); + } + + List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { + final Sort sort = Sort.by(Sort.Direction.ASC, "scraperId"); + final Specification findAllSpecification = new FindAllErrorLogSpecification(errorLogSearchQuery, scraperIdNameCache); + final Pageable pageable = extractPageSettings(errorLogSearchQuery, sort); + + return findAllErrorLog(findAllSpecification, pageable, sort) + .stream() + .map(this::mapToLog) + .toList(); + + } + + List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { + final Sort sort = Sort.by(Sort.Direction.ASC, "scraperId"); + final Specification findAllSpecification = new FindAllStatusLogSpecification(statusLogSearchQuery, scraperIdNameCache); + final Pageable pageable = extractPageSettings(statusLogSearchQuery, sort); + + return findAllStatusLog(findAllSpecification, pageable, sort) + .stream() + .map(this::mapToLog) + .toList(); + } + + private List findAllStatusLog(Specification findAllSpecification, Pageable pageable, Sort sort) { + if (Objects.isNull(pageable)) { + return jpaScraperRunLogRepository.findAll(findAllSpecification, sort); + } + return jpaScraperRunLogRepository.findAll(findAllSpecification, pageable).toList(); + } + + private List findAllErrorLog(Specification findAllSpecification, Pageable pageable, Sort sort) { + if (Objects.isNull(pageable)) { + return jpaScraperRunErrorRepository.findAll(findAllSpecification, sort); + } + return jpaScraperRunErrorRepository.findAll(findAllSpecification, pageable).toList(); + } + + private ScraperRunStatusLog mapToLog(EntityScraperRunStatusLog entityScraperRunStatusLog) { + final String scraperName = scraperIdNameCache.getScraperNameById(entityScraperRunStatusLog.getScraperId()); + + return new ScraperRunStatusLog( + scraperName, + entityScraperRunStatusLog.getStartTime(), + entityScraperRunStatusLog.getFinishTime(), + entityScraperRunStatusLog.getScannedEventCount(), + entityScraperRunStatusLog.getErrorCount() + ); + } + + private ScraperRunErrorLog mapToLog(EntityScraperRunErrorLog entityScraperRunErrorLog) { + final String scraperName = scraperIdNameCache.getScraperNameById(entityScraperRunErrorLog.getScraperId()); + + return new ScraperRunErrorLog(scraperName, + entityScraperRunErrorLog.getTime(), + entityScraperRunErrorLog.getErrorCode(), + entityScraperRunErrorLog.getDescription()); + } + + private EntityScraperRunStatusLog mapToEntity(ScraperRunStatusLog scraperRunStatusLog) { + final Integer scraperId = scraperIdNameCache.getIdByScraperName(scraperRunStatusLog.configurationName()); + + return new EntityScraperRunStatusLog( + scraperId, + scraperRunStatusLog.startTime(), + scraperRunStatusLog.finishTime(), + scraperRunStatusLog.scannedEventCount(), + scraperRunStatusLog.errorCount() + ); + } + + private EntityScraperRunErrorLog mapToEntity(ScraperRunErrorLog scraperRunError) { + final Integer scraperId = scraperIdNameCache.getIdByScraperName(scraperRunError.configurationName()); + + return new EntityScraperRunErrorLog( + scraperId, + scraperRunError.time(), + scraperRunError.errorCode(), + scraperRunError.description() + ); + } + + private Pageable extractPageSettings(StatusLogSearchQuery statusLogSearchQuery, Sort sort) { + return statusLogSearchQuery.hasPageSetting() + ? PageRequest.of(statusLogSearchQuery.page(), statusLogSearchQuery.pageSize(), sort) + : null; + } + + private Pageable extractPageSettings(ErrorLogSearchQuery errorLogSearchQuery, Sort sort) { + return errorLogSearchQuery.hasPageSetting() + ? PageRequest.of(errorLogSearchQuery.page(), errorLogSearchQuery.pageSize(), sort) + : null; + } + + private static class FindAllStatusLogSpecification implements Specification { + + private final transient StatusLogSearchQuery searchQuery; + private final transient ScraperIdNameCache scraperIdNameCache; + + public FindAllStatusLogSpecification(StatusLogSearchQuery statusLogSearchQuery, ScraperIdNameCache scraperIdNameCache) { + + this.searchQuery = statusLogSearchQuery; + this.scraperIdNameCache = scraperIdNameCache; + } + + + @Override + public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { + List outputPredicates = new ArrayList<>(); + + if (searchQuery.hasConfigurationNames()) { + final List idsLists = searchQuery.configurationNames() + .stream() + .map(scraperIdNameCache::getIdByScraperName) + .toList(); + outputPredicates.add(criteriaBuilder.in(root.get("scraperId")).value(idsLists)); + } + + if (Objects.nonNull(searchQuery.startTimeFrom())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("startTime"), searchQuery.startTimeFrom())); + } + + if (Objects.nonNull(searchQuery.startTimeTo())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("startTime"), searchQuery.startTimeTo())); + } + + if (Objects.nonNull(searchQuery.finishTimeFrom())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("finishTime"), searchQuery.finishTimeFrom())); + } + + if (Objects.nonNull(searchQuery.finishTimeTo())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("finishTime"), searchQuery.finishTimeTo())); + } + + if (Objects.nonNull(searchQuery.hasScannedEvent())) { + final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasScannedEvent()) + ? criteriaBuilder.greaterThan(root.get("scannedEventCount"), 0) + : criteriaBuilder.or(criteriaBuilder.equal(root.get("scannedEventCount"), 0), criteriaBuilder.isNull(root.get("scannedEventCount"))); + + outputPredicates.add(predicate); + } else if (Objects.nonNull(searchQuery.scannedEventGreaterThanOrEqualTo())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("scannedEventCount"), searchQuery.scannedEventGreaterThanOrEqualTo())); + } + + if (Objects.nonNull(searchQuery.hasErrors())) { + final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasErrors()) + ? criteriaBuilder.greaterThan(root.get("errorCount"), 0) + : criteriaBuilder.or(criteriaBuilder.equal(root.get("errorCount"), 0), criteriaBuilder.isNull(root.get("errorCount"))); + + outputPredicates.add(predicate); + } else if (Objects.nonNull(searchQuery.errorCountGreaterThanOrEqualTo())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("errorCount"), searchQuery.errorCountGreaterThanOrEqualTo())); + } + + return outputPredicates.stream() + .reduce(criteriaBuilder::and) + .orElse(null); + } + } + + private static class FindAllErrorLogSpecification implements Specification { + private final transient ErrorLogSearchQuery searchQuery; + private final transient ScraperIdNameCache scraperIdNameCache; + + public FindAllErrorLogSpecification(ErrorLogSearchQuery errorLogSearchQuery, ScraperIdNameCache scraperIdNameCache) { + this.searchQuery = errorLogSearchQuery; + this.scraperIdNameCache = scraperIdNameCache; + } + + @Override + public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { + List outputPredicates = new ArrayList<>(); + + if (searchQuery.hasConfigurationNames()) { + final List idsLists = searchQuery.configurationNames() + .stream() + .map(scraperIdNameCache::getIdByScraperName) + .toList(); + outputPredicates.add(criteriaBuilder.in(root.get("scraperId")).value(idsLists)); + } + + if (Objects.nonNull(searchQuery.fromDate())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("time"), searchQuery.fromDate())); + } + + if (Objects.nonNull(searchQuery.toDate())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("time"), searchQuery.toDate())); + } + + if (Objects.nonNull(searchQuery.description())) { + outputPredicates.add(criteriaBuilder.equal(root.get("description"), searchQuery.description())); + } + + if (searchQuery.hasErrorCodes()) { + outputPredicates.add(criteriaBuilder.in(root.get("errorCode")).value(searchQuery.errorCodes())); + } + + return outputPredicates.stream() + .reduce(criteriaBuilder::and) + .orElse(null); + } + } +} diff --git a/src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java b/src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java deleted file mode 100644 index 50e2d5b..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperQueryFacade.java +++ /dev/null @@ -1,22 +0,0 @@ -package hub.event.scrapers.core; - -import hub.event.scrapers.core.runlog.*; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class ScraperQueryFacade { - private final ScraperRunLogRepository scraperRunLogRepository; - - public ScraperQueryFacade(ScraperRunLogRepository scraperRunLogRepository) { - this.scraperRunLogRepository = scraperRunLogRepository; - } - public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { - return scraperRunLogRepository.findAllErrorLog(errorLogSearchQuery); - } - public List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { - return scraperRunLogRepository.findAllStatusLog(statusLogSearchQuery); - } - -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java b/src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java deleted file mode 100644 index 9c82a7c..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperRunErrorJpaRepository.java +++ /dev/null @@ -1,6 +0,0 @@ -package hub.event.scrapers.core; - -import org.springframework.data.jpa.repository.JpaRepository; - -interface ScraperRunErrorJpaRepository extends JpaRepository { -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java b/src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java deleted file mode 100644 index 6d54081..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperRunErrorLogEntity.java +++ /dev/null @@ -1,61 +0,0 @@ -package hub.event.scrapers.core; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import java.time.Instant; - -@Entity -class ScraperRunErrorLogEntity { - @Id - private String configurationName; - @Column(nullable = false) - private Instant time; - - @Column(nullable = false) - private String errorCode; - private String description; - - public ScraperRunErrorLogEntity() { - } - - ScraperRunErrorLogEntity(String configurationName, Instant time, String errorCode, String description) { - - this.configurationName = configurationName; - this.time = time; - this.errorCode = errorCode; - this.description = description; - } - - String getConfigurationName() { - return configurationName; - } - - void setConfigurationName(String configurationName) { - this.configurationName = configurationName; - } - - Instant getTime() { - return time; - } - - void setTime(Instant time) { - this.time = time; - } - - String getErrorCode() { - return errorCode; - } - - void setErrorCode(String errorCode) { - this.errorCode = errorCode; - } - - String getDescription() { - return description; - } - - void setDescription(String description) { - this.description = description; - } -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java b/src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java deleted file mode 100644 index c4e94dc..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperRunLogRepository.java +++ /dev/null @@ -1,186 +0,0 @@ -package hub.event.scrapers.core; - -import hub.event.scrapers.core.runlog.ErrorLogSearchQuery; -import hub.event.scrapers.core.runlog.ScraperRunErrorLog; -import hub.event.scrapers.core.runlog.ScraperRunStatusLog; -import hub.event.scrapers.core.runlog.StatusLogSearchQuery; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; -import org.springframework.data.jpa.domain.Specification; -import org.springframework.stereotype.Repository; - -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Predicate; -import javax.persistence.criteria.Root; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -@Repository -class ScraperRunLogRepository { - - private ScraperRunLogJpaRepository scraperRunLogJpaRepository; - private ScraperRunErrorJpaRepository scraperRunErrorJpaRepository; - - void save(ScraperRunStatusLog scraperRunStatusLog) { - ScraperRunStatusLogEntity scraperRunStatusLogEntity = mapToEntity(scraperRunStatusLog); - scraperRunLogJpaRepository.save(scraperRunStatusLogEntity); - } - - void save(ScraperRunErrorLog scraperRunError) { - ScraperRunErrorLogEntity scraperRunErrorLogEntity = mapToEntity(scraperRunError); - scraperRunErrorJpaRepository.save(scraperRunErrorLogEntity); - } - - List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { - final Sort sort = Sort.by(Sort.Direction.ASC, "configurationName"); - final Specification findAllSpecification = new FindAllErrorLogSpecification(errorLogSearchQuery); - final Pageable pageable = extractPageSettings(errorLogSearchQuery, sort); - - return findAllErrorLog(findAllSpecification, pageable, sort) - .stream() - .map(this::mapToLog) - .toList(); - - } - - List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { - final Sort sort = Sort.by(Sort.Direction.ASC, "configurationName"); - final Specification findAllSpecification = new FindAllStatusLogSpecification(statusLogSearchQuery); - final Pageable pageable = extractPageSettings(statusLogSearchQuery, sort); - - return findAllStatusLog(findAllSpecification, pageable, sort) - .stream() - .map(this::mapToLog) - .toList(); - } - - private List findAllStatusLog(Specification findAllSpecification, Pageable pageable, Sort sort) { - if (Objects.isNull(pageable)) { - return scraperRunLogJpaRepository.findAll(findAllSpecification, sort); - } - return scraperRunLogJpaRepository.findAll(findAllSpecification, pageable).toList(); - } - - private List findAllErrorLog(Specification findAllSpecification, Pageable pageable, Sort sort) { - return null; - } - - private ScraperRunStatusLog mapToLog(ScraperRunStatusLogEntity scraperRunStatusLogEntity) { - return new ScraperRunStatusLog( - scraperRunStatusLogEntity.getConfigurationName(), - scraperRunStatusLogEntity.getStartTime(), - scraperRunStatusLogEntity.getFinishTime(), - scraperRunStatusLogEntity.getScannedEventCount(), - scraperRunStatusLogEntity.getErrorCount() - ); - } - - private ScraperRunErrorLog mapToLog(ScraperRunErrorLogEntity scraperRunErrorLogEntity) { - return null; - } - - private ScraperRunStatusLogEntity mapToEntity(ScraperRunStatusLog scraperRunStatusLog) { - return new ScraperRunStatusLogEntity( - scraperRunStatusLog.configurationName(), - scraperRunStatusLog.startTime(), - scraperRunStatusLog.finishTime(), - scraperRunStatusLog.scannedEventCount(), - scraperRunStatusLog.errorCount() - ); - } - - private ScraperRunErrorLogEntity mapToEntity(ScraperRunErrorLog scraperRunError) { - return new ScraperRunErrorLogEntity( - scraperRunError.configurationName(), - scraperRunError.time(), - scraperRunError.errorCode(), - scraperRunError.description() - ); - } - - private Pageable extractPageSettings(StatusLogSearchQuery statusLogSearchQuery, Sort sort) { - return statusLogSearchQuery.hasPageSetting() - ? PageRequest.of(statusLogSearchQuery.page(), statusLogSearchQuery.pageSize(), sort) - : null; - } - - private Pageable extractPageSettings(ErrorLogSearchQuery errorLogSearchQuery, Sort sort) { - return null; - } - - private static class FindAllStatusLogSpecification implements Specification { - - private final StatusLogSearchQuery command; - - public FindAllStatusLogSpecification(StatusLogSearchQuery statusLogSearchQuery) { - - this.command = statusLogSearchQuery; - } - - - @Override - public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { - List outputPredicates = new ArrayList<>(); - - if (command.hasConfigurationNames()) { - outputPredicates.add(criteriaBuilder.in(root.get("configurationName")).value(command.configurationNames())); - } - - if (Objects.nonNull(command.startTimeFrom())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("startTime"), command.startTimeFrom())); - } - - if (Objects.nonNull(command.startTimeTo())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("startTime"), command.startTimeTo())); - } - - if (Objects.nonNull(command.finishTimeFrom())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("finishTime"), command.finishTimeFrom())); - } - - if (Objects.nonNull(command.finishTimeTo())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("finishTime"), command.finishTimeTo())); - } - - if (Objects.nonNull(command.hasScannedEvent())) { - final Predicate predicate = Boolean.TRUE.equals(command.hasScannedEvent()) - ? criteriaBuilder.greaterThan(root.get("scannedEventCount"), 0) - : criteriaBuilder.equal(root.get("scannedEventCount"), 0); - - outputPredicates.add(predicate); - } else if (Objects.nonNull(command.scannedEventCount())) { - outputPredicates.add(criteriaBuilder.equal(root.get("scannedEventCount"), command.scannedEventCount())); - } - - if (Objects.nonNull(command.hasErrors())) { - final Predicate predicate = Boolean.TRUE.equals(command.hasErrors()) - ? criteriaBuilder.greaterThan(root.get("errorCount"), 0) - : criteriaBuilder.equal(root.get("errorCount"), 0); - - outputPredicates.add(predicate); - } else if (Objects.nonNull(command.errorCount())) { - outputPredicates.add(criteriaBuilder.equal(root.get("errorCount"), command.scannedEventCount())); - } - - return outputPredicates.stream() - .reduce(criteriaBuilder::and) - .orElse(null); - } - } - - private static class FindAllErrorLogSpecification implements Specification { - private final ErrorLogSearchQuery errorLogSearchQuery; - - public FindAllErrorLogSpecification(ErrorLogSearchQuery errorLogSearchQuery) { - this.errorLogSearchQuery = errorLogSearchQuery; - } - - @Override - public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { - return null; - } - } -} diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java index fb60d5e..f22dd9a 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunService.java +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -20,28 +20,32 @@ class ScraperRunService { private final EventFacadeAdapter eventFacadeAdapter; private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; private final List pageScrapers; + private final ScraperIdNameCache scraperIdNameCache; @Autowired - ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventFacadeAdapter eventFacadeAdapter, LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, List pageScrapers) { + ScraperRunService(ScraperConfigRepository scraperConfigRepository, EventFacadeAdapter eventFacadeAdapter, LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository, List pageScrapers, ScraperIdNameCache scraperIdNameCache) { this.scraperConfigRepository = scraperConfigRepository; this.eventFacadeAdapter = eventFacadeAdapter; this.lastScrapedEventMarkerRepository = lastScrapedEventMarkerRepository; this.pageScrapers = pageScrapers; + this.scraperIdNameCache = scraperIdNameCache; } - @Scheduled(cron = "${scrapers.run.cron.expression}") -// @Scheduled(cron = "0 0 * * *") - @PostConstruct - void createScrapersConfigsIfMissing() { + void createScrapersConfigsIfMissingAndFillIdNameCache() { final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); + scraperIdNameCache.add(scraperConfigs); + final List scrapersWithoutConfig = getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { - scraperConfigRepository.create(pageScraperPort.configurationName(), pageScraperPort.timeZone(),true); + ScraperConfig scraperConfig = scraperConfigRepository.create(pageScraperPort.configurationName(), pageScraperPort.timeZone(), true); + scraperIdNameCache.add(scraperConfig); } } + @Scheduled(cron = "${scrapers.run.cron.expression}") +// @Scheduled(cron = "0 0 * * *") void start() { final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); @@ -50,8 +54,8 @@ void start() { eventFacadeAdapter.saveAll(scrapedEventList); - final List runScraperConfigurationsNames = getRunScraperConfigurationsNames(pageScrapersToRun); - lastScrapedEventMarkerRepository.makeDraftActive(runScraperConfigurationsNames); + final List runScraperConfigurationsIds = getRunScraperConfigurationsIds(pageScrapersToRun); + lastScrapedEventMarkerRepository.markAllMarkersByIdsAsActive(runScraperConfigurationsIds); } @@ -81,9 +85,10 @@ private List runScrapersForEvents(List pageScrape .toList(); } - private List getRunScraperConfigurationsNames(List pageScrapersToRun) { + private List getRunScraperConfigurationsIds(List pageScrapersToRun) { return pageScrapersToRun.stream() .map(PageScraperPort::configurationName) + .map(scraperIdNameCache::getIdByScraperName) .toList(); } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java b/src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java deleted file mode 100644 index 7391608..0000000 --- a/src/main/java/hub/event/scrapers/core/ScraperRunStatusLogEntity.java +++ /dev/null @@ -1,69 +0,0 @@ -package hub.event.scrapers.core; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import java.time.Instant; - -@Entity -class ScraperRunStatusLogEntity { - @Id - private String configurationName; - @Column(nullable = false) - private Instant startTime; - @Column(nullable = false) - private Instant finishTime; - @Column(nullable = false) - private Integer scannedEventCount; - @Column(nullable = false) - private Integer errorCount; - - - ScraperRunStatusLogEntity(String configurationName, Instant startTime, Instant finishTime, Integer scannedEventCount, Integer errorCount) { - this.configurationName = configurationName; - this.startTime = startTime; - this.finishTime = finishTime; - this.scannedEventCount = scannedEventCount; - this.errorCount = errorCount; - } - - String getConfigurationName() { - return configurationName; - } - - void setConfigurationName(String configurationName) { - this.configurationName = configurationName; - } - - Instant getStartTime() { - return startTime; - } - - void setStartTime(Instant startTime) { - this.startTime = startTime; - } - - Instant getFinishTime() { - return finishTime; - } - - void setFinishTime(Instant finishTime) { - this.finishTime = finishTime; - } - - Integer getScannedEventCount() { - return scannedEventCount; - } - - void setScannedEventCount(Integer scannedEventCount) { - this.scannedEventCount = scannedEventCount; - } - - Integer getErrorCount() { - return errorCount; - } - - void setErrorCount(Integer errorCount) { - this.errorCount = errorCount; - } -} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java index f1f75b2..a87effa 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocation.java @@ -74,6 +74,10 @@ String locationName() { return eventDateWithLocation.locationName(); } + ZoneId timeZone(){ + return eventDateWithLocation.timeZone(); + } + private static void validateStartDateTimeAndEndDateTime(LocalDate startDate, LocalTime startTime, LocalDate endDate, LocalTime endTime) throws EventDateEndDateTimeBeforeStartDateTimeException, EventDateInPastException { final LocalDateTime startLocalDateTime = LocalDateTime.of(startDate, startTime); final LocalDateTime endLocalDateTime = LocalDateTime.of(endDate, endTime); diff --git a/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java b/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java index 6cbfb2e..ea574b4 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ErrorLogSearchQuery.java @@ -2,6 +2,7 @@ import java.time.Instant; import java.util.List; +import java.util.Objects; public class ErrorLogSearchQuery { private final List configurationNames; @@ -10,31 +11,110 @@ public class ErrorLogSearchQuery { private final List errorCodes; private final String description; - public ErrorLogSearchQuery(List configurationNames, Instant fromDate, Instant toDate, List errorCodes, String description) { + private final Integer pageSize; + private final Integer page; + + private ErrorLogSearchQuery(List configurationNames, Instant fromDate, Instant toDate, List errorCodes, String description, Integer page, Integer pageSize) { this.configurationNames = configurationNames; this.fromDate = fromDate; this.toDate = toDate; this.errorCodes = errorCodes; this.description = description; + this.pageSize = pageSize; + this.page = page; } - List configurationNames() { + + public List configurationNames() { return configurationNames; } - Instant fromDate() { + public Instant fromDate() { return fromDate; } - Instant toDate() { + public Instant toDate() { return toDate; } - List errorCodes() { + public List errorCodes() { return errorCodes; } - String description() { + public String description() { return description; } + + public Integer pageSize() { + return pageSize; + } + + public Integer page() { + return page; + } + + public boolean hasConfigurationNames() { + return Objects.nonNull(configurationNames) && !configurationNames.isEmpty(); + } + + public boolean hasErrorCodes() { + return Objects.nonNull(errorCodes) && !errorCodes.isEmpty(); + } + + public boolean hasPageSetting() { + return Objects.nonNull(page) && Objects.nonNull(pageSize); + } + + public static ErrorLogSearchQueryBuilder builder() { + return new ErrorLogSearchQueryBuilder(); + } + + public static class ErrorLogSearchQueryBuilder { + private List configurationNames; + private Instant fromDate; + private Instant toDate; + private List errorCodes; + private String description; + + private Integer pageSize; + private Integer page; + + private ErrorLogSearchQueryBuilder() { + } + + public ErrorLogSearchQueryBuilder description(String description) { + this.description = description; + return this; + } + + public ErrorLogSearchQueryBuilder errorCodes(List errorCodes) { + this.errorCodes = errorCodes; + return this; + } + + public ErrorLogSearchQueryBuilder fromDate(Instant fromDate) { + this.fromDate = fromDate; + return this; + } + + public ErrorLogSearchQueryBuilder toDate(Instant toDate) { + this.toDate = toDate; + return this; + } + + public ErrorLogSearchQueryBuilder configurationNames(List configurationNames) { + this.configurationNames = configurationNames; + return this; + } + + public ErrorLogSearchQueryBuilder page(int page, int pageSize) { + this.page = page; + this.pageSize = pageSize; + return this; + } + + public ErrorLogSearchQuery build() { + return new ErrorLogSearchQuery(configurationNames, fromDate, toDate, errorCodes, description, page, pageSize); + } + } } diff --git a/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java index 89886e2..111c0c4 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java +++ b/src/main/java/hub/event/scrapers/core/runlog/StatusLogSearchQuery.java @@ -10,28 +10,28 @@ public class StatusLogSearchQuery { private final Instant startTimeFrom; private final Instant finishTimeTo; private final Instant finishTimeFrom; - private final Integer scannedEventCount; - private final Integer errorCount; + private final Integer scannedEventGreaterThanOrEqualTo; + private final Integer errorCountGreaterThanOrEqualTo; private final Boolean hasErrors; private final Boolean hasScannedEvent; private final Integer pageSize; private final Integer page; - private StatusLogSearchQuery(List configurationNames, Instant startTimeTo, Instant startTimeFrom, Instant finishTimeTo, Instant finishTime, Integer scannedEventCount, Integer errorCount, Boolean hasErrors, Boolean hasScannedEvent, Integer page, Integer pageSize) { + private StatusLogSearchQuery(List configurationNames, Instant startTimeTo, Instant startTimeFrom, Instant finishTimeTo, Instant finishTime, Integer scannedEventGreaterThanOrEqualTo, Integer errorCountGreaterThanOrEqualTo, Boolean hasErrors, Boolean hasScannedEvent, Integer page, Integer pageSize) { this.configurationNames = configurationNames; this.startTimeTo = startTimeTo; this.startTimeFrom = startTimeFrom; this.finishTimeTo = finishTimeTo; this.finishTimeFrom = finishTime; - this.scannedEventCount = scannedEventCount; - this.errorCount = errorCount; + this.scannedEventGreaterThanOrEqualTo = scannedEventGreaterThanOrEqualTo; + this.errorCountGreaterThanOrEqualTo = errorCountGreaterThanOrEqualTo; this.hasErrors = hasErrors; this.hasScannedEvent = hasScannedEvent; this.pageSize = pageSize; this.page = page; } - public StatusLogSearchCommandBuilder builder() { + public static StatusLogSearchCommandBuilder builder() { return new StatusLogSearchCommandBuilder(); } @@ -55,12 +55,12 @@ public Instant finishTimeTo() { return finishTimeTo; } - public Integer scannedEventCount() { - return scannedEventCount; + public Integer scannedEventGreaterThanOrEqualTo() { + return scannedEventGreaterThanOrEqualTo; } - public Integer errorCount() { - return errorCount; + public Integer errorCountGreaterThanOrEqualTo() { + return errorCountGreaterThanOrEqualTo; } public Boolean hasErrors() { @@ -87,20 +87,22 @@ public boolean hasPageSetting() { return Objects.nonNull(page) && Objects.nonNull(pageSize); } - static class StatusLogSearchCommandBuilder { + public static class StatusLogSearchCommandBuilder { private List configurationNames; private Instant startTimeTo; private Instant startTimeFrom; private Instant finishTimeTo; private Instant finishTimeFrom; - private Integer scannedEventCount; - private Integer errorCount; + private Integer scannedEventGreaterThanOrEqualTo; + private Integer errorCountGreaterThanOrEqualTo; private Boolean hasErrors; + private Boolean hasScannedEvent; + private Integer pageSize; private Integer page; - StatusLogSearchCommandBuilder() { + private StatusLogSearchCommandBuilder() { } public StatusLogSearchCommandBuilder configurationNames(List configurationNames) { @@ -128,33 +130,34 @@ public StatusLogSearchCommandBuilder finishTimeFrom(Instant finishTimeFrom) { return this; } - public StatusLogSearchCommandBuilder scannedEventCount(Integer scannedEventCount) { - this.scannedEventCount = scannedEventCount; + public StatusLogSearchCommandBuilder scannedEventGreaterThanOrEqualTo(int scannedEventCount) { + this.scannedEventGreaterThanOrEqualTo = scannedEventCount; return this; } - public StatusLogSearchCommandBuilder errorCount(Integer errorCount) { - this.errorCount = errorCount; + public StatusLogSearchCommandBuilder errorCountGreaterThanOrEqualTo(int errorCount) { + this.errorCountGreaterThanOrEqualTo = errorCount; return this; } - public StatusLogSearchCommandBuilder hasErrors(Boolean hasErrors) { - this.hasErrors = hasErrors; + public StatusLogSearchCommandBuilder hasScannedEvent(Boolean hasScannedEvent) { + this.hasScannedEvent = hasScannedEvent; return this; } - public StatusLogSearchCommandBuilder page(Integer page) { - this.page = page; + public StatusLogSearchCommandBuilder hasErrors(Boolean hasErrors) { + this.hasErrors = hasErrors; return this; } - public StatusLogSearchCommandBuilder pageSize(Integer pageSize) { + public StatusLogSearchCommandBuilder page(int page, int pageSize) { + this.page = page; this.pageSize = pageSize; return this; } public StatusLogSearchQuery build() { - return new StatusLogSearchQuery(configurationNames, startTimeTo, startTimeFrom, finishTimeTo, finishTimeFrom, scannedEventCount, errorCount, hasErrors, hasScannedEvent, page, pageSize); + return new StatusLogSearchQuery(configurationNames, startTimeTo, startTimeFrom, finishTimeTo, finishTimeFrom, scannedEventGreaterThanOrEqualTo, errorCountGreaterThanOrEqualTo, hasErrors, hasScannedEvent, page, pageSize); } } diff --git a/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java b/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java index b615ecf..58f1ab0 100644 --- a/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java +++ b/src/main/java/hub/event/scrapers/core/scraper/ScraperConfig.java @@ -2,5 +2,5 @@ import java.time.ZoneId; -public record ScraperConfig(String configurationName, ZoneId timeZone, boolean isActive) { +public record ScraperConfig(Integer scraperId, String configurationName, ZoneId timeZone, boolean isActive) { } diff --git a/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java b/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java index f09b470..c79dd94 100644 --- a/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java +++ b/src/main/java/hub/event/scrapers/ebilet/EbiletScraper.java @@ -2,10 +2,12 @@ import hub.event.scrapers.core.PageScraperPort; import hub.event.scrapers.core.ScrapedEvent; +import org.springframework.stereotype.Component; import java.util.Collection; import java.util.Collections; +@Component class EbiletScraper extends PageScraperPort { @Override diff --git a/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java b/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java index 44c502d..908ef34 100644 --- a/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java +++ b/src/main/java/hub/event/scrapers/empikbilet/EmpikBiletScraper.java @@ -2,10 +2,12 @@ import hub.event.scrapers.core.PageScraperPort; import hub.event.scrapers.core.ScrapedEvent; +import org.springframework.stereotype.Component; import java.util.Collection; import java.util.Collections; +@Component class EmpikBiletScraper extends PageScraperPort { diff --git a/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java b/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java index 76971a2..9d7c360 100644 --- a/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java +++ b/src/main/java/hub/event/scrapers/goingapp/GoingAppScraper.java @@ -2,11 +2,13 @@ import hub.event.scrapers.core.PageScraperPort; import hub.event.scrapers.core.ScrapedEvent; +import org.springframework.stereotype.Component; import java.util.Collection; import java.util.Collections; +@Component class GoingAppScraper extends PageScraperPort { @Override diff --git a/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java b/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java index 6924298..0ea5575 100644 --- a/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java +++ b/src/main/java/hub/event/scrapers/kupbilecik/KupBilecikScraper.java @@ -2,11 +2,12 @@ import hub.event.scrapers.core.PageScraperPort; import hub.event.scrapers.core.ScrapedEvent; +import org.springframework.stereotype.Component; import java.util.Collection; import java.util.Collections; - +@Component class KupBilecikScraper extends PageScraperPort { diff --git a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java index 6d25c72..34af017 100644 --- a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java +++ b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java @@ -2,13 +2,18 @@ import hub.event.scrapers.core.PageScraperPort; import hub.event.scrapers.core.ScrapedEvent; +import org.apache.commons.lang3.NotImplementedException; +import org.springframework.stereotype.Component; +import java.time.ZonedDateTime; import java.util.Collection; -import java.util.Collections; +@Component class ProanimaScraper extends PageScraperPort { @Override protected Collection scrap() { - return Collections.emptyList(); + logError(ZonedDateTime.now().toInstant(), "PRO_ERR_0", "Scraper not implemented yet"); + saveLastScrapedEventMarker(ZonedDateTime.now().toInstant(), null, "Absolutely nothing, i'm just drunk"); + throw new NotImplementedException(); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 3edc9c0..0eff880 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,12 +2,13 @@ spring.profiles.active=prod spring.liquibase.change-log=classpath:database/changelog-root.xml spring.jpa.hibernate.show-sql=true -scrapers.run.cron.expression=0 0 0 * * * +#scrapers.run.cron.expression=0 0 0 * * * +scrapers.run.cron.expression=0 * * * * * #--- spring.config.activate.on-profile=prod spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect -spring.jpa.hibernate.ddl-auto=update +spring.jpa.hibernate.ddl-auto=none spring.jpa.hibernate.show-sql=true spring.datasource.url=jdbc:postgresql://localhost:5432/EventScraper spring.datasource.username=admin diff --git a/src/main/resources/database/change/change_2.yaml b/src/main/resources/database/change/change_2.yaml new file mode 100644 index 0000000..626165f --- /dev/null +++ b/src/main/resources/database/change/change_2.yaml @@ -0,0 +1,197 @@ +databaseChangeLog: + - changeSet: + id: scraper_config_table_2022_01_17 + author: batonikleonardo + comment: Configuration for all avilible scrapers + changes: + - createTable: + preConditions: + not: + tableExists: + tableName: scraper_config + tableName: scraper_config + remarks: Create scraper for configuration for all avilible scrapers + columns: + - column: + name: scraper_id + type: int + autoIncrement: true + constraints: + primaryKey: true + - column: + name: configuration_name + type: varchar + constraints: + nullable: false + unique: true + - column: + name: is_active + type: boolean + defaultValueBoolean: false + constraints: + nullable: false + - column: + name: time_zone + type: varchar + constraints: + nullable: false + + - changeSet: + id: scraper_status_log_table_2022_01_17 + author: batonikleonardo + comment : Create teble for scrapers status logs + changes: + - createTable: + preConditions: + not: + tableExists: + tableName: scraper_status_log + tableName: scraper_status_log + remarks: Scrapers status logs + columns: + - column: + name: log_id + type: int + autoIncrement: true + constraints: + primaryKey: true + - column: + name: error_count + type: int + - column: + name: start_time + type: timestamp with time zone + constraints: + nullable: false + - column: + name: finish_time + type: timestamp with time zone + constraints: + nullable: false + - column: + name: scanned_event_count + type: int + - column: + name: scraper_id + type: int + constraints: + nullable: false + foreignKeyName: status_log_scraper_fk + referencedTableName: scraper_config + referencedColumnNames: scraper_id + + - changeSet: + id: scraper_error_log_table_2022_01_17 + author: batonikleonardo + comment: Create table for scrapres error logs + changes: + - createTable: + preConditions: + not: + tableExists: + tableName: scraper_error_log + tableName: scraper_error_log + remarks: Scrapres error logs + columns: + - column: + name: log_id + type: int + autoIncrement: true + constraints: + primaryKey: true + - column: + name: description + type: varchar + - column: + name: error_code + type: varchar + - column: + name: error_time + type: timestamp with time zone + constraints: + nullable: false + - column: + name: scraper_id + type: int + constraints: + nullable: false + foreignKeyName: error_log_scraper_fk + referencedTableName: scraper_config + referencedColumnNames: scraper_id + + - changeSet: + id: scraper_scraped_event_maker_table_2022_01_17 + author: batonikleonardo + comment: Create table for for last sraped event marker + changes: + - createTable: + preConditions: + not: + tableExists: + tableName: scraper_scraped_event_maker + tableName: scraper_scraped_event_maker + remarks: Marker for last sraped event + columns: + - column: + name: marker_id + type: int + autoIncrement: true + constraints: + primaryKey: true + - column: + name: event_title + type: varchar + - column: + name: is_complete + type: boolean + defaultValueBoolean: false + constraints: + nullable: false + - column: + name: marker + type: varchar + - column: + name: run_time + type: timestamp with time zone + constraints: + nullable: false + - column: + name: scraper_id + type: int + constraints: + nullable: false + foreignKeyName: maker_scraper_fk + referencedTableName: scraper_config + referencedColumnNames: scraper_id + + - changeSet: + id: scraper_status_log_index_2022_01_17 + author: batonikleonardo + comment : scrapers status logs scraper id index + changes: + - createIndex: + preConditions: + not: + indexExists: + indexName: status_log_scraper_id_idx + tableName: scraper_status_log + columns: + - column: + name: scraper_id + indexName: status_log_scraper_id_idx + + - changeSet: + id: scraper_error_log_index_2022_01_17 + author: batonikleonardo + comment: scrapers error logs scraper id index + changes: + - createIndex: + preConditions: + not: + indexExists: + indexName: error_log_scraper_id_idx + tableName: scraper_error_log + columns: + - column: + name: scraper_id + indexName: error_log_scraper_id_idx diff --git a/src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java b/src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java index 06741fd..eef7b61 100644 --- a/src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java +++ b/src/test/java/hub/event/scrapers/core/LastScrapedEventMarkerRepositoryTest.java @@ -1,6 +1,7 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.scraper.LastScrapedEventMarker; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -22,12 +23,14 @@ class LastScrapedEventMarkerRepositoryTest { @Mock - private LastScrapedEventMarkerJpaRepository lastScrapedEventMarkerEntityRepository; + private JpaLastScrapedEventMarkerRepository lastScrapedEventMarkerEntityRepository; + @Mock + private ScraperIdNameCache scraperIdNameCache; @InjectMocks - private LastScrapedEventMarkerEntityRepository lastScrapedEventMarkerRepository; + private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; @Captor - ArgumentCaptor lastScrapedEventMarkerEntityCaptor; + ArgumentCaptor lastScrapedEventMarkerEntityCaptor; @Test void saveTest() { @@ -35,6 +38,9 @@ void saveTest() { LastScrapedEventMarker eventMarker1 = new LastScrapedEventMarker("config1", LocalDateTime.now().toInstant(ZoneOffset.UTC), "title1", "marker1"); LastScrapedEventMarker eventMarker2 = new LastScrapedEventMarker("config2", LocalDateTime.now().toInstant(ZoneOffset.UTC), "title2", "marker2"); + //when + when(scraperIdNameCache.getIdByScraperName("config1")).thenReturn(20); + when(scraperIdNameCache.getIdByScraperName("config2")).thenReturn(40); //then lastScrapedEventMarkerRepository.store(eventMarker1); @@ -44,63 +50,69 @@ void saveTest() { assertThat(lastScrapedEventMarkerEntityCaptor.getAllValues()) .extracting( - LastScrapedEventMarkerEntity::getScraperConfigurationName, - LastScrapedEventMarkerEntity::getRunDateTime, - LastScrapedEventMarkerEntity::getEventTitle, - LastScrapedEventMarkerEntity::getMarker + EntityLastScrapedEventMarker::getScraperId, + EntityLastScrapedEventMarker::getRunTime, + EntityLastScrapedEventMarker::getEventTitle, + EntityLastScrapedEventMarker::getMarker ).contains( - tuple(eventMarker1.scraperConfigurationName(), eventMarker1.runDateTime(), eventMarker1.eventTitle(), eventMarker1.marker()), - tuple(eventMarker2.scraperConfigurationName(), eventMarker2.runDateTime(), eventMarker2.eventTitle(), eventMarker2.marker()) + tuple(20, eventMarker1.runDateTime(), eventMarker1.eventTitle(), eventMarker1.marker()), + tuple(40, eventMarker2.runDateTime(), eventMarker2.eventTitle(), eventMarker2.marker()) ); } - @Test - void whenFindByNotExistsScraperConfigurationNameThenReturnEmptyMarker() { - //given - final String scraperConfigurationName = "not_exists_scraper"; - - //when - when(lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(scraperConfigurationName)) - .thenReturn(Optional.empty()); - - //then - final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, true); - - assertThat(lastScrapedEventMarker).isEmpty(); - verify(lastScrapedEventMarkerEntityRepository).findByScraperConfigurationName(scraperConfigurationName); - } - - @Test - void whenFindByExistsScraperConfigurationNameThenReturnMarker() { - //given - final String scraperConfigurationName = "exists_scraper"; - final Instant localDateTime = LocalDateTime.now().toInstant(ZoneOffset.UTC); - - final LastScrapedEventMarkerEntity lastScrapedEventMarkerEntity = new LastScrapedEventMarkerEntity(); - lastScrapedEventMarkerEntity.setMarker("maker1"); - lastScrapedEventMarkerEntity.setRunDateTime(localDateTime); - lastScrapedEventMarkerEntity.setEventTitle("title1"); - lastScrapedEventMarkerEntity.setScraperConfigurationName("exists_scraper"); - lastScrapedEventMarkerEntity.setComplete(true); - - //when - when(lastScrapedEventMarkerEntityRepository.findByScraperConfigurationName(scraperConfigurationName)) - .thenReturn(Optional.of(lastScrapedEventMarkerEntity)); - - //then - final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, true); - - assertThat(lastScrapedEventMarker).isNotEmpty() - .get() - .extracting( - LastScrapedEventMarker::scraperConfigurationName, - LastScrapedEventMarker::runDateTime, - LastScrapedEventMarker::eventTitle, - LastScrapedEventMarker::marker - ) - .contains(scraperConfigurationName, localDateTime, "title1", "maker1"); - - verify(lastScrapedEventMarkerEntityRepository).findByScraperConfigurationName(scraperConfigurationName); + @Nested + class FindByScraperConfigurationNameTest { + + @Test + void whenNotExistsThenReturnEmptyMarker() { + //given + final int scraperId = 4560; + + //when + when(lastScrapedEventMarkerEntityRepository.findByScraperId(scraperId)) + .thenReturn(Optional.empty()); + + //then + final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findLastCompletedByScraperConfigurationId(scraperId); + + assertThat(lastScrapedEventMarker).isEmpty(); + verify(lastScrapedEventMarkerEntityRepository).findByScraperId(scraperId); + } + + @Test + void whenExistsThenReturnMarker() { + //given + final String scraperConfigurationName = "exists_scraper"; + final Instant localDateTime = LocalDateTime.now().toInstant(ZoneOffset.UTC); + final int scraperId = 4570; + + final EntityLastScrapedEventMarker entityLastScrapedEventMarker = new EntityLastScrapedEventMarker(); + entityLastScrapedEventMarker.setMarker("maker1"); + entityLastScrapedEventMarker.setRunTime(localDateTime); + entityLastScrapedEventMarker.setEventTitle("title1"); + entityLastScrapedEventMarker.setScraperId(scraperId); + entityLastScrapedEventMarker.setComplete(true); + + //when + when(scraperIdNameCache.getScraperNameById(scraperId)).thenReturn(scraperConfigurationName); + when(lastScrapedEventMarkerEntityRepository.findByScraperId(scraperId)) + .thenReturn(Optional.of(entityLastScrapedEventMarker)); + + //then + final Optional lastScrapedEventMarker = lastScrapedEventMarkerRepository.findLastCompletedByScraperConfigurationId(scraperId); + + assertThat(lastScrapedEventMarker).isNotEmpty() + .get() + .extracting( + LastScrapedEventMarker::scraperConfigurationName, + LastScrapedEventMarker::runDateTime, + LastScrapedEventMarker::eventTitle, + LastScrapedEventMarker::marker + ) + .contains(scraperConfigurationName, localDateTime, "title1", "maker1"); + + verify(lastScrapedEventMarkerEntityRepository).findByScraperId(scraperId); + } } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java b/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java index a81a560..4358547 100644 --- a/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java +++ b/src/test/java/hub/event/scrapers/core/PageScraperPortTest.java @@ -15,6 +15,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; import java.util.Optional; @@ -27,74 +28,80 @@ @ExtendWith(MockitoExtension.class) class PageScraperPortTest { @Mock - private ScraperRunLogRepository scraperRunLogRepository; + private ScraperLogRepository scraperLogRepository; @Mock private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; - - @Captor - ArgumentCaptor lastScrapedEventMarkerArgumentCaptor; + @Mock + private ScraperIdNameCache scraperIdNameCache; @InjectMocks - private PageScraperPort pageScraperPort = new PageScraperPort() { + private final PageScraperPort pageScraperPort = new PageScraperPort() { @Override protected Collection scrap() { return Collections.emptyList(); } }; + @Captor + ArgumentCaptor lastScrapedEventMarkerArgumentCaptor; @Captor private ArgumentCaptor scraperRunStatusLogArgumentCaptor; @Captor private ArgumentCaptor scraperRunErrorLogArgumentCaptor; - @Test - void logError() { - //given - final String configurationName = pageScraperPort.configurationName(); - final Instant time = Instant.now(); - final String errorCode = "ER1"; - final String description = "Error 1 server is down"; - //then - pageScraperPort.logError(time, errorCode, description); + @Nested + class LogSaveTest { - verify(scraperRunLogRepository).save(scraperRunErrorLogArgumentCaptor.capture()); + @Test + void logError() { + //given + final String configurationName = pageScraperPort.configurationName(); + final Instant time = Instant.now(); + final String errorCode = "ER1"; + final String description = "Error 1 server is down"; - final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.getValue(); + //then + pageScraperPort.logError(time, errorCode, description); - assertNotNull(scraperRunErrorLog); - assertEquals(configurationName, scraperRunErrorLog.configurationName()); - assertEquals(time, scraperRunErrorLog.time()); - assertEquals(errorCode, scraperRunErrorLog.errorCode()); - assertEquals(description, scraperRunErrorLog.description()); + verify(scraperLogRepository).save(scraperRunErrorLogArgumentCaptor.capture()); - } + final ScraperRunErrorLog scraperRunErrorLog = scraperRunErrorLogArgumentCaptor.getValue(); - @Test - void logStatus() { - //given - final String configurationName = pageScraperPort.configurationName(); - final Instant startTime = Instant.now(); - final Instant finishTime = LocalDateTime.now().plusMinutes(10).toInstant(ZoneOffset.UTC); - final Integer scannedEventCount = 23; - final Integer errorCount = 1; + assertNotNull(scraperRunErrorLog); + assertEquals(configurationName, scraperRunErrorLog.configurationName()); + assertEquals(time, scraperRunErrorLog.time()); + assertEquals(errorCode, scraperRunErrorLog.errorCode()); + assertEquals(description, scraperRunErrorLog.description()); + } - //then - pageScraperPort.logStatus(startTime, finishTime, scannedEventCount, errorCount); + @Test + void logStatus() { + //given + final String configurationName = pageScraperPort.configurationName(); + final Instant startTime = Instant.now(); + final Instant finishTime = LocalDateTime.now().plusMinutes(10).toInstant(ZoneOffset.UTC); + final Integer scannedEventCount = 23; + final Integer errorCount = 1; - verify(scraperRunLogRepository).save(scraperRunStatusLogArgumentCaptor.capture()); - final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.getValue(); + //then + pageScraperPort.logStatus(startTime, finishTime, scannedEventCount, errorCount); + + verify(scraperLogRepository).save(scraperRunStatusLogArgumentCaptor.capture()); + + final ScraperRunStatusLog scraperRunStatusLog = scraperRunStatusLogArgumentCaptor.getValue(); - assertNotNull(scraperRunStatusLog); - assertEquals(configurationName, scraperRunStatusLog.configurationName()); - assertEquals(startTime, scraperRunStatusLog.startTime()); - assertEquals(finishTime, scraperRunStatusLog.finishTime()); - assertEquals(scannedEventCount, scraperRunStatusLog.scannedEventCount()); - assertEquals(errorCount, scraperRunStatusLog.errorCount()); + assertNotNull(scraperRunStatusLog); + assertEquals(configurationName, scraperRunStatusLog.configurationName()); + assertEquals(startTime, scraperRunStatusLog.startTime()); + assertEquals(finishTime, scraperRunStatusLog.finishTime()); + assertEquals(scannedEventCount, scraperRunStatusLog.scannedEventCount()); + assertEquals(errorCount, scraperRunStatusLog.errorCount()); + } } @Nested @@ -103,9 +110,12 @@ class LastScrapedEventMarkerTest { void whenFoundLastScrapedEventMarkerThenReturnNotEmptyOptional() { //given final String scraperName = pageScraperPort.configurationName(); + final int scraperId = 102; final LastScrapedEventMarker lastScrapedEventMarker = new LastScrapedEventMarker(scraperName, LocalDateTime.now().minusDays(2).toInstant(ZoneOffset.UTC), "Event Title", "Marker", false); //when - when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) + + when(scraperIdNameCache.getIdByScraperName(pageScraperPort.configurationName())).thenReturn(scraperId); + when(lastScrapedEventMarkerRepository.findLastCompletedByScraperConfigurationId(scraperId)) .thenReturn(Optional.of(lastScrapedEventMarker)); //then @@ -117,15 +127,16 @@ void whenFoundLastScrapedEventMarkerThenReturnNotEmptyOptional() { .isEqualTo(lastScrapedEventMarker); }); - verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); + verify(lastScrapedEventMarkerRepository).findLastCompletedByScraperConfigurationId(scraperId); } @Test void whenNotFoundLastScrapedEventMarkerThenReturnOptionalEmpty() { //given - final String scraperName = pageScraperPort.configurationName(); + final int scraperId = 103; //when - when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperName, true)) + when(scraperIdNameCache.getIdByScraperName(pageScraperPort.configurationName())).thenReturn(scraperId); + when(lastScrapedEventMarkerRepository.findLastCompletedByScraperConfigurationId(scraperId)) .thenReturn(Optional.empty()); //then @@ -135,34 +146,25 @@ void whenNotFoundLastScrapedEventMarkerThenReturnOptionalEmpty() { .isEmpty(); }); - verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperName, true); + verify(lastScrapedEventMarkerRepository).findLastCompletedByScraperConfigurationId(scraperId); } + } - @Test - void whenCreateLastScrapedEventMarkerThenReplaceNotCompleteOne() { - //given - final String scraperConfigurationName = pageScraperPort.configurationName(); - final LocalDateTime runTime = LocalDateTime.now(); - - final LastScrapedEventMarker previousLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime.minusDays(1).toInstant(ZoneOffset.UTC), "Title 1", "Marker", false); - final LastScrapedEventMarker newLastScrapedEventMarker = new LastScrapedEventMarker(scraperConfigurationName, runTime.toInstant(ZoneOffset.UTC), "Title 2", "Marker 2", false); - - //when - when(lastScrapedEventMarkerRepository.findByScraperConfigurationName(scraperConfigurationName, false)) - .thenReturn(Optional.of(previousLastScrapedEventMarker)); - - //then - pageScraperPort.saveLastScrapedEventMarker(runTime.toInstant(ZoneOffset.UTC), newLastScrapedEventMarker.eventTitle(), newLastScrapedEventMarker.marker()); - - verify(lastScrapedEventMarkerRepository).findByScraperConfigurationName(scraperConfigurationName, false); - verify(lastScrapedEventMarkerRepository).drop(previousLastScrapedEventMarker); + @Test + void saveLastScrapedEventMarker() { + //given + final Instant runDateTime = ZonedDateTime.of(LocalDateTime.now(), pageScraperPort.timeZone()).toInstant(); + final String eventTitle = "Test Event Title 1"; + final String marker = "Example marker value for test"; - verify(lastScrapedEventMarkerRepository).store(lastScrapedEventMarkerArgumentCaptor.capture()); + //then + assertDoesNotThrow(() -> pageScraperPort.saveLastScrapedEventMarker(runDateTime, eventTitle, marker)); - assertThat(lastScrapedEventMarkerArgumentCaptor.getValue()) - .isEqualTo(newLastScrapedEventMarker); + verify(lastScrapedEventMarkerRepository).store(lastScrapedEventMarkerArgumentCaptor.capture()); + final LastScrapedEventMarker capturedLastScrapedEventMarker = lastScrapedEventMarkerArgumentCaptor.getValue(); - } + assertEquals(runDateTime, capturedLastScrapedEventMarker.runDateTime()); + assertEquals(eventTitle, capturedLastScrapedEventMarker.eventTitle()); + assertEquals(marker, capturedLastScrapedEventMarker.marker()); } - -} \ No newline at end of file +} diff --git a/src/test/java/hub/event/scrapers/core/ScraperConfigRepositoryTest.java b/src/test/java/hub/event/scrapers/core/ScraperConfigRepositoryTest.java new file mode 100644 index 0000000..8a62d1b --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/ScraperConfigRepositoryTest.java @@ -0,0 +1,78 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.scraper.ScraperConfig; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +import java.time.ZoneId; + +import static org.assertj.core.api.Assertions.*; + +@SpringBootTest +@ActiveProfiles(profiles = "dev") +class ScraperConfigRepositoryTest { + private final ScraperConfigRepository scraperConfigRepository; + + @Autowired + ScraperConfigRepositoryTest(ScraperConfigRepository scraperConfigRepository) { + this.scraperConfigRepository = scraperConfigRepository; + } + + @Test + @Order(1) + void repositoryTest() { + //given + final String scraper1Name = "SC1"; + final String scraper2Name = "SC2"; + final String scraper3Name = "SC3"; + final String scraper4Name = "SC4"; + final ZoneId zoneId = ZoneId.systemDefault(); + + assertThat(scraperConfigRepository).isNotNull(); + + final ScraperConfig createdScraperConfig1 = scraperConfigRepository.create(scraper1Name, zoneId, false); + final ScraperConfig createdScraperConfig2 = scraperConfigRepository.create(scraper2Name, zoneId, true); + final ScraperConfig createdScraperConfig3 = scraperConfigRepository.create(scraper3Name, zoneId, false); + final ScraperConfig createdScraperConfig4 = scraperConfigRepository.create(scraper4Name, zoneId, true); + + assertThatException().isThrownBy(() -> scraperConfigRepository.create(scraper4Name, zoneId, true)); + + assertThat(createdScraperConfig1.scraperId()).isPositive(); + assertThat(createdScraperConfig2.scraperId()).isPositive(); + assertThat(createdScraperConfig3.scraperId()).isPositive(); + assertThat(createdScraperConfig4.scraperId()).isPositive(); + + assertThat(scraperConfigRepository.exists(createdScraperConfig2.scraperId())).isTrue(); + assertThat(scraperConfigRepository.exists(createdScraperConfig4.scraperId())).isTrue(); + assertThat(scraperConfigRepository.exists(createdScraperConfig4.scraperId() + 1000)).isFalse(); + + assertThat(scraperConfigRepository.allScraperConfigs()) + .extracting(ScraperConfig::configurationName, ScraperConfig::timeZone, ScraperConfig::isActive) + .contains( + tuple(scraper1Name, zoneId, false), + tuple(scraper2Name, zoneId, true), + tuple(scraper3Name, zoneId, false), + tuple(scraper4Name, zoneId, true) + ); + + scraperConfigRepository.activate(createdScraperConfig3.scraperId()); + scraperConfigRepository.deactivate(createdScraperConfig2.scraperId()); + + assertThat(scraperConfigRepository.allScraperConfigs()) + .extracting(ScraperConfig::configurationName, ScraperConfig::timeZone, ScraperConfig::isActive) + .contains( + tuple(scraper1Name, zoneId, false), + tuple(scraper2Name, zoneId, false), + tuple(scraper3Name, zoneId, true), + tuple(scraper4Name, zoneId, true) + ).doesNotContain( + tuple(scraper2Name, zoneId, true), + tuple(scraper3Name, zoneId, false) + ); + + } + +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java index 6a4efd3..f77a815 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperFacadeTest.java @@ -1,12 +1,10 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.exceptions.ScraperConfigurationByNameNotExists; -import hub.event.scrapers.core.scraper.LastScrapedEventMarker; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -19,44 +17,46 @@ @ExtendWith(MockitoExtension.class) class ScraperFacadeTest { - @Mock - private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; @Mock private ScraperConfigRepository scraperConfigRepository; - - @Captor - ArgumentCaptor lastScrapedEventMarkerArgumentCaptor; + @Mock + private ScraperIdNameCache scraperIdNameCache; @InjectMocks private ScraperFacade scraperFacade; @Nested class ScraperConfigTest { @Test - void whenActivateScraperThatNotExistThenCreateNewRecord() { + void whenActivateScraperThatNotExistThenThrowException() { //given final String scraperName = "sc1"; //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(false); + int scraperId = 100; + when(scraperIdNameCache.getIdByScraperName(scraperName)).thenReturn(scraperId); + when(scraperConfigRepository.exists(scraperId)).thenReturn(false); //then - scraperFacade.activateScraperByConfigurationName(scraperName); + Assertions.assertThatExceptionOfType(ScraperConfigurationByNameNotExists.class) + .isThrownBy(() -> scraperFacade.activateScraperByConfigurationName(scraperName)); - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository).create(scraperName, ZoneId.systemDefault(), true); - verify(scraperConfigRepository, never()).activate(scraperName); + verify(scraperConfigRepository).exists(scraperId); + verify(scraperConfigRepository, never()).activate(scraperId); } @Test - void whenActivateScraperThatExistThenFinishSuccessfully() { + void whenActivateScraperThatExistThenCallActive() { //given final String scraperName = "sc4"; //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(true); + int scraperId = 4; + when(scraperIdNameCache.getIdByScraperName(scraperName)).thenReturn(scraperId); + when(scraperConfigRepository.exists(scraperId)).thenReturn(true); //then - scraperFacade.activateScraperByConfigurationName(scraperName); + Assertions.assertThatCode(() -> scraperFacade.activateScraperByConfigurationName(scraperName)) + .doesNotThrowAnyException(); - verify(scraperConfigRepository).exists(scraperName); + verify(scraperConfigRepository).exists(scraperId); verify(scraperConfigRepository, never()).create(scraperName, ZoneId.systemDefault(), true); - verify(scraperConfigRepository).activate(scraperName); + verify(scraperConfigRepository).activate(scraperId); } @Test @@ -64,27 +64,29 @@ void whenDeactivateNotExistsScraperThanThrowConfigNotExistsException() { //given final String scraperName = "sc2"; //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(false); + int scraperId = 2; + when(scraperIdNameCache.getIdByScraperName(scraperName)).thenReturn(scraperId); + when(scraperConfigRepository.exists(scraperId)).thenReturn(false); //then assertThrows(ScraperConfigurationByNameNotExists.class, () -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName, ZoneId.systemDefault(), true); - verify(scraperConfigRepository, never()).deactivate(scraperName); + verify(scraperConfigRepository).exists(scraperId); + verify(scraperConfigRepository, never()).deactivate(scraperId); } @Test - void whenDeactivateExistsScraperThanFinishSuccessfully() { + void whenDeactivateExistsScraperThenCallDeactivate() { //given final String scraperName = "sc3"; //when - when(scraperConfigRepository.exists(scraperName)).thenReturn(true); + int scraperId = 3; + when(scraperIdNameCache.getIdByScraperName(scraperName)).thenReturn(scraperId); + when(scraperConfigRepository.exists(scraperId)).thenReturn(true); //then assertDoesNotThrow(() -> scraperFacade.deactivateScraperByConfigurationName(scraperName)); - verify(scraperConfigRepository).exists(scraperName); - verify(scraperConfigRepository, never()).create(scraperName, ZoneId.systemDefault(), true); - verify(scraperConfigRepository).deactivate(scraperName); + verify(scraperConfigRepository).exists(scraperId); + verify(scraperConfigRepository).deactivate(scraperId); } } diff --git a/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java b/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java new file mode 100644 index 0000000..5d25aa4 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java @@ -0,0 +1,473 @@ +package hub.event.scrapers.core; + +import hub.event.scrapers.core.runlog.ErrorLogSearchQuery; +import hub.event.scrapers.core.runlog.ScraperRunErrorLog; +import hub.event.scrapers.core.runlog.ScraperRunStatusLog; +import hub.event.scrapers.core.runlog.StatusLogSearchQuery; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.jdbc.Sql; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.groups.Tuple.tuple; +import static org.mockito.Mockito.when; + +@SpringBootTest +@ActiveProfiles(profiles = "dev") +@Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"/database/scrapers/core/test_data_init.sql"}) +class ScraperLogQueryFacadeTest { + private static final String ZONE_ID = "Europe/Warsaw"; + @Autowired + private ScraperLogQueryFacade scraperLogQueryFacade; + @Autowired + private ScraperLogRepository scraperLogRepository; + + @MockBean + private ScraperIdNameCache scraperIdNameCache; + + private final String configurationName1 = "Scraper1"; + private final String configurationName2 = "Scraper2"; + private final String configurationName3 = "Scraper3"; + + @BeforeEach + void mockCache() { + when(scraperIdNameCache.getIdByScraperName(configurationName1)).thenReturn(100000); + when(scraperIdNameCache.getIdByScraperName(configurationName2)).thenReturn(200000); + when(scraperIdNameCache.getIdByScraperName(configurationName3)).thenReturn(300000); + + when(scraperIdNameCache.getScraperNameById(100000)).thenReturn(configurationName1); + when(scraperIdNameCache.getScraperNameById(200000)).thenReturn(configurationName2); + when(scraperIdNameCache.getScraperNameById(300000)).thenReturn(configurationName3); + } + + @Nested + class ErrorLogQueryTest { + + @Test + void testSearchByEmptyQuery() { + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder().build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(9) + .extracting( + ScraperRunErrorLog::configurationName, + ScraperRunErrorLog::description, + ScraperRunErrorLog::errorCode, + scraperRunErrorLog -> scraperRunErrorLog.time().toString()) + .contains( + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-19T21:15:00.015Z"), + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-20T21:16:00.011Z"), + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-21T21:17:00.013Z"), + tuple(configurationName1, "Error 1", "ERR_1", "2022-10-23T21:18:00.022Z"), + tuple(configurationName1, "Error 1", "ERR_1", "2022-10-24T21:19:00.020Z"), + tuple(configurationName2, "Error 1", "ERR_1", "2022-10-19T21:20:00.019Z"), + tuple(configurationName2, "Error 10", "ERR_10", "2022-10-30T22:21:00.011Z"), + tuple(configurationName3, "Error 1", "ERR_1", "2022-10-19T21:22:00.015Z"), + tuple(configurationName3, "Error 20", "ERR_20", "2022-10-19T21:23:00.020Z") + ); + } + + @Test + void testSearchByDescription() { + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() + .description("Error 0") + .build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(3) + .extracting( + ScraperRunErrorLog::configurationName, ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode) + .contains( + tuple(configurationName1, "Error 0", "ERR_0"), + tuple(configurationName1, "Error 0", "ERR_0"), + tuple(configurationName1, "Error 0", "ERR_0") + ); + } + + @Test + void testSearchByErrorsCodes() { + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() + .errorCodes(List.of("ERR_10", "ERR_20")) + .build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(2) + .extracting( + ScraperRunErrorLog::configurationName, ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode) + .contains( + tuple(configurationName2, "Error 10", "ERR_10"), + tuple(configurationName3, "Error 20", "ERR_20") + ); + } + + @Test + void testSearchByDateRange() { + final ZonedDateTime fromDate = ZonedDateTime.of(LocalDateTime.of(2022, 10, 20, 12, 0), ZoneId.of(ZONE_ID)); + final ZonedDateTime toDate = ZonedDateTime.of(LocalDateTime.of(2022, 10, 24, 23, 50), ZoneId.of(ZONE_ID)); + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() + .fromDate(fromDate.toInstant()) + .toDate(toDate.toInstant()) + .build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(4) + .extracting( + ScraperRunErrorLog::configurationName, + ScraperRunErrorLog::description, + ScraperRunErrorLog::errorCode, + scraperRunErrorLog -> scraperRunErrorLog.time().toString()) + .contains( + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-20T21:16:00.011Z"), + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-21T21:17:00.013Z"), + tuple(configurationName1, "Error 1", "ERR_1", "2022-10-23T21:18:00.022Z"), + tuple(configurationName1, "Error 1", "ERR_1", "2022-10-24T21:19:00.020Z") + ); + } + + @Test + void testSearchBycConfigurationNames() { + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() + .configurationNames(List.of(configurationName2, configurationName3)) + .build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(4) + .extracting( + ScraperRunErrorLog::configurationName, ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode) + .contains( + tuple(configurationName2, "Error 1", "ERR_1"), + tuple(configurationName2, "Error 10", "ERR_10"), + tuple(configurationName3, "Error 1", "ERR_1"), + tuple(configurationName3, "Error 20", "ERR_20") + ); + } + + @Test + void testSearchWithPagination() { + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() + .page(2, 2) + .build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(2) + .extracting( + ScraperRunErrorLog::configurationName, ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode) + .contains( + tuple(configurationName1, "Error 1", "ERR_1"), + tuple(configurationName2, "Error 1", "ERR_1") + ); + } + + @Test + void testSearchByMultipleConditionQuery() { + ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() + .errorCodes(List.of("ERR_0", "ERR_1")) + .configurationNames(List.of(configurationName3, configurationName2, configurationName1)) + .fromDate(ZonedDateTime.of(LocalDateTime.of(2022, 1, 1, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .toDate(ZonedDateTime.of(LocalDateTime.of(2022, 12, 30, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .build(); + assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) + .hasSize(7) + .extracting( + ScraperRunErrorLog::configurationName, ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode) + .contains( + tuple(configurationName1, "Error 0", "ERR_0"), + tuple(configurationName1, "Error 0", "ERR_0"), + tuple(configurationName1, "Error 0", "ERR_0"), + tuple(configurationName1, "Error 1", "ERR_1"), + tuple(configurationName1, "Error 1", "ERR_1"), + tuple(configurationName2, "Error 1", "ERR_1"), + tuple(configurationName3, "Error 1", "ERR_1") + ); + } + } + + @Nested + class StatusLogQueryTest { + @Test + void testSearchByEmptyQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(12) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByStartDateQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .startTimeFrom(ZonedDateTime.of(LocalDateTime.of(2022, 10, 1, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .startTimeTo(ZonedDateTime.of(LocalDateTime.of(2022, 10, 5, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(1) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByFinishDateQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .finishTimeFrom(ZonedDateTime.of(LocalDateTime.of(2022, 10, 1, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .finishTimeTo(ZonedDateTime.of(LocalDateTime.of(2022, 10, 5, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(1) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByNamesQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .configurationNames(List.of(configurationName1, configurationName3)) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(8) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + + @Test + void testSearchWithPaginationQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .page(2, 3) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(3) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100) + ); + } + } + + @Nested + class StatusLogErrorQueryTest { + @Test + void testSearchByErrorCountQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .errorCountGreaterThanOrEqualTo(75) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(7) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByHasErrorQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .hasErrors(true) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(7) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByNotHasErrorsQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .hasErrors(false) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(5) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0) + ); + } + } + + @Nested + class StatusLogScannedEventQueryTest { + @Test + void testSearchByScannedEventCountQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .scannedEventGreaterThanOrEqualTo(30) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(8) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByHasScannedEventQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .hasScannedEvent(true) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(10) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), + tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByNotHasScannedEventQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .hasScannedEvent(false) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(2) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80) + ); + } + + } +} \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java index 3578fbf..0298c73 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java @@ -3,10 +3,9 @@ import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; import hub.event.scrapers.core.exceptions.EventDateInPastException; import hub.event.scrapers.core.scraper.ScraperConfig; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -17,6 +16,7 @@ import java.util.Arrays; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) @@ -28,136 +28,216 @@ class ScraperRunServiceTest { private EventFacadeAdapter eventFacadeAdapter; @Mock private LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; - @Captor - private ArgumentCaptor> scrapedEventListCaptor; - - @Test - void whenListContainsInactiveScraperThenSkipRunIt() { - //given - final PageScraperPort activeScraper1 = mock(PageScraperPort.class); - final PageScraperPort activeScraper2 = mock(PageScraperPort.class); - final PageScraperPort inactiveScraper = mock(PageScraperPort.class); - - final String activeScraperName1 = "active1"; - final String activeScraperName2 = "active2"; - final String inactiveScraperName = "inactive2"; - - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, ZoneId.systemDefault(), true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, ZoneId.systemDefault(), true); - final ScraperConfig inactiveScraperConfig = new ScraperConfig(inactiveScraperName, ZoneId.systemDefault(), false); - - final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2, inactiveScraper); - final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config, inactiveScraperConfig)); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers); - - //when - when(activeScraper1.configurationName()).thenReturn(activeScraperName1); - when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); - - when(activeScraper1.configurationName()).thenReturn(activeScraperName1); - when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); - - when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - - //then - scraperRunService.start(); - - verify(scraperConfigRepository, never()).create(anyString(), any(ZoneId.class), anyBoolean()); - verify(activeScraper1).scrap(); - verify(activeScraper2).scrap(); - verify(inactiveScraper, never()).scrap(); - } - @Test - void whenScrapedEventWithoutDuplicatesThenSaveAsEvent() throws EventDateInPastException { - //given - final PageScraperPort activeScraper1 = mock(PageScraperPort.class); - final PageScraperPort activeScraper2 = mock(PageScraperPort.class); - - final String activeScraperName1 = "active1"; - final String activeScraperName2 = "active2"; - - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, ZoneId.systemDefault(), true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, ZoneId.systemDefault(), true); - final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); - - final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Palaven", "Addres 1", "location 2")) - .title("Title1") - .description("Description1") - .build(); - final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Thessia", "Addres 2", "location 22")) - .title("Title2") - .description("Description2") - .build(); - final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Eden Prime", "Addres 134", "location 166")) - .title("Title1") - .description("Description1") - .build(); - final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); - - final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers); - - //when - when(activeScraper1.configurationName()).thenReturn(activeScraperName1); - when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - - when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); - - when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - - //then - scraperRunService.start(); - - verify(activeScraper1).scrap(); - verify(activeScraper2).scrap(); - verify(eventFacadeAdapter).saveAll(scrapedEventList); + @Nested + class ScrapersStartTest { + + @Test + void whenListContainsInactiveScraperThenSkipRunIt() { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + final PageScraperPort inactiveScraper = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + final String inactiveScraperName = "inactive2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(1, activeScraperName1, ZoneId.systemDefault(), true); + final ScraperConfig activeScraper2Config = new ScraperConfig(2, activeScraperName2, ZoneId.systemDefault(), true); + final ScraperConfig inactiveScraperConfig = new ScraperConfig(3, inactiveScraperName, ZoneId.systemDefault(), false); + + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2, inactiveScraper); + final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config, inactiveScraperConfig)); + + final ScraperIdNameCache scraperIdNameCache = new ScraperIdNameCache(); + scraperIdNameCache.add(scraperConfigs); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers, scraperIdNameCache); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); + + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + when(inactiveScraper.configurationName()).thenReturn(inactiveScraperName); + + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + //then + scraperRunService.start(); + + verify(scraperConfigRepository, never()).create(anyString(), any(ZoneId.class), anyBoolean()); + verify(activeScraper1).scrap(); + verify(activeScraper2).scrap(); + verify(inactiveScraper, never()).scrap(); + } + + @Test + void whenScrapedEventExistsThenSaveAsEvent() throws EventDateInPastException { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(10, activeScraperName1, ZoneId.systemDefault(), true); + final ScraperConfig activeScraper2Config = new ScraperConfig(11, activeScraperName2, ZoneId.systemDefault(), true); + final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Palaven", "Addres 1", "location 2")) + .title("Title1") + .description("Description1") + .build(); + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Thessia", "Addres 2", "location 22")) + .title("Title2") + .description("Description2") + .build(); + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Eden Prime", "Addres 134", "location 166")) + .title("Title1") + .description("Description1") + .build(); + final List scrapedEventList = Arrays.asList(scrapedEvent1, scrapedEvent2, scrapedEvent3); + + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); + final ScraperIdNameCache scraperIdNameCache = new ScraperIdNameCache(); + scraperIdNameCache.add(scraperConfigs); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers, scraperIdNameCache); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + + when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); + + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + //then + scraperRunService.start(); + + verify(activeScraper1).scrap(); + verify(activeScraper2).scrap(); + verify(eventFacadeAdapter).saveAll(scrapedEventList); + } + + @Test + void whenScrapedEventSavedThenMakeLastScrapedEventMarkerDraftActive() throws EventDateInPastException { + //given + final PageScraperPort activeScraper1 = mock(PageScraperPort.class); + final PageScraperPort activeScraper2 = mock(PageScraperPort.class); + + final String activeScraperName1 = "active1"; + final String activeScraperName2 = "active2"; + + final ScraperConfig activeScraper1Config = new ScraperConfig(20, activeScraperName1, ZoneId.systemDefault(), true); + final ScraperConfig activeScraper2Config = new ScraperConfig(21, activeScraperName2, ZoneId.systemDefault(), true); + final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Palaven", "Addres 1", "location 2")) + .title("Title1") + .description("Description1") + .build(); + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Thessia", "Addres 2", "location 22")) + .title("Title2") + .description("Description2") + .build(); + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Eden Prime", "Addres 134", "location 166")) + .title("Title1") + .description("Description1") + .build(); + + final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); + final ScraperIdNameCache scraperIdNameCache = new ScraperIdNameCache(); + scraperIdNameCache.add(scraperConfigs); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers, scraperIdNameCache); + + //when + when(activeScraper1.configurationName()).thenReturn(activeScraperName1); + when(activeScraper2.configurationName()).thenReturn(activeScraperName2); + + when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); + when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); + + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); + + //then + scraperRunService.start(); + + verify(lastScrapedEventMarkerRepository).markAllMarkersByIdsAsActive(List.of(activeScraper1Config.scraperId(), activeScraper2Config.scraperId())); + } } - @Test - void whenScrapedEventSavedThenMakeLastScrapedEventMarkerDraftActive() throws EventDateInPastException { - //given - final PageScraperPort activeScraper1 = mock(PageScraperPort.class); - final PageScraperPort activeScraper2 = mock(PageScraperPort.class); - - final String activeScraperName1 = "active1"; - final String activeScraperName2 = "active2"; - - final ScraperConfig activeScraper1Config = new ScraperConfig(activeScraperName1, ZoneId.systemDefault(), true); - final ScraperConfig activeScraper2Config = new ScraperConfig(activeScraperName2, ZoneId.systemDefault(), true); - final List scraperConfigs = new ArrayList<>(Arrays.asList(activeScraper1Config, activeScraper2Config)); - - final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Palaven", "Addres 1", "location 2")) - .title("Title1") - .description("Description1") - .build(); - final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Thessia", "Addres 2", "location 22")) - .title("Title2") - .description("Description2") - .build(); - final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(SingleEventDateWithLocation.single(LocalDate.now().plusDays(1), LocalTime.now().plusHours(2), ZoneId.systemDefault(), "Eden Prime", "Addres 134", "location 166")) - .title("Title1") - .description("Description1") - .build(); - - final List pageScrapers = Arrays.asList(activeScraper1, activeScraper2); - final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers); - - //when - when(activeScraper1.configurationName()).thenReturn(activeScraperName1); - when(activeScraper2.configurationName()).thenReturn(activeScraperName2); - - when(activeScraper1.scrap()).thenReturn(Arrays.asList(scrapedEvent1)); - when(activeScraper2.scrap()).thenReturn(Arrays.asList(scrapedEvent2, scrapedEvent3)); - - when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigs); - - //then - scraperRunService.start(); - - verify(lastScrapedEventMarkerRepository).makeDraftActive(List.of(activeScraperName1, activeScraperName2)); + @Nested + class CreateConfigsIfMissingAndCacheBuildTest { + @Test + void whenFoundScrapersWithoutConfigThenCreateDefault() { + //given + final PageScraperPort scraper1 = mock(PageScraperPort.class); + final PageScraperPort scraper2 = mock(PageScraperPort.class); + final List pageScrapers = Arrays.asList(scraper1, scraper2); + + final ZoneId timeZone = ZoneId.systemDefault(); + + final ScraperConfig scraperConfig1 = new ScraperConfig(1, "Scraper1", timeZone, true); + final ScraperConfig scraperConfig2 = new ScraperConfig(2, "Scraper2", timeZone, true); + + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers, new ScraperIdNameCache()); + //when + when(scraper2.timeZone()).thenReturn(timeZone); + when(scraper1.configurationName()).thenReturn("Scraper1"); + when(scraper2.configurationName()).thenReturn("Scraper2"); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(List.of(scraperConfig1)); + when(scraperConfigRepository.create("Scraper2", timeZone, true)).thenReturn(scraperConfig2); + + //then + scraperRunService.createScrapersConfigsIfMissingAndFillIdNameCache(); + + verify(scraperConfigRepository).create("Scraper2", timeZone, true); + } + + @Test + void cacheContainsFoundedAndCreatedScrapersConfig() { + //given + final PageScraperPort scraper1 = mock(PageScraperPort.class); + final PageScraperPort scraper2 = mock(PageScraperPort.class); + final List pageScrapers = Arrays.asList(scraper1, scraper2); + + final ZoneId timeZone = ZoneId.systemDefault(); + + final ScraperConfig scraperConfig1 = new ScraperConfig(1, "Scraper1", timeZone, true); + final ScraperConfig scraperConfig2 = new ScraperConfig(2, "Scraper2", timeZone, true); + final ScraperConfig scraperConfig3 = new ScraperConfig(3, "Scraper3", ZoneId.systemDefault(), true); + final ScraperConfig scraperConfig4 = new ScraperConfig(4, "Scraper4", ZoneId.systemDefault(), false); + final ScraperConfig scraperConfig5 = new ScraperConfig(5, "Scraper5", ZoneId.systemDefault(), true); + final List scraperConfigList = List.of(scraperConfig1, scraperConfig3, scraperConfig4, scraperConfig5); + + final ScraperIdNameCache scraperIdNameCache = new ScraperIdNameCache(); + final ScraperRunService scraperRunService = new ScraperRunService(scraperConfigRepository, eventFacadeAdapter, lastScrapedEventMarkerRepository, pageScrapers, scraperIdNameCache); + + //when + when(scraper2.timeZone()).thenReturn(timeZone); + when(scraper1.configurationName()).thenReturn("Scraper1"); + when(scraper2.configurationName()).thenReturn("Scraper2"); + when(scraperConfigRepository.allScraperConfigs()).thenReturn(scraperConfigList); + when(scraperConfigRepository.create("Scraper2", timeZone, true)).thenReturn(scraperConfig2); + + //then + scraperRunService.createScrapersConfigsIfMissingAndFillIdNameCache(); + + assertEquals(scraperConfig1.scraperId(), scraperIdNameCache.getIdByScraperName(scraperConfig1.configurationName())); + assertEquals(scraperConfig2.scraperId(), scraperIdNameCache.getIdByScraperName(scraperConfig2.configurationName())); + assertEquals(scraperConfig3.scraperId(), scraperIdNameCache.getIdByScraperName(scraperConfig3.configurationName())); + assertEquals(scraperConfig4.scraperId(), scraperIdNameCache.getIdByScraperName(scraperConfig4.configurationName())); + assertEquals(scraperConfig5.scraperId(), scraperIdNameCache.getIdByScraperName(scraperConfig5.configurationName())); + + assertEquals(scraperConfig1.configurationName(), scraperIdNameCache.getScraperNameById(scraperConfig1.scraperId())); + assertEquals(scraperConfig2.configurationName(), scraperIdNameCache.getScraperNameById(scraperConfig2.scraperId())); + assertEquals(scraperConfig3.configurationName(), scraperIdNameCache.getScraperNameById(scraperConfig3.scraperId())); + assertEquals(scraperConfig4.configurationName(), scraperIdNameCache.getScraperNameById(scraperConfig4.scraperId())); + assertEquals(scraperConfig5.configurationName(), scraperIdNameCache.getScraperNameById(scraperConfig5.scraperId())); + } } } \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java index 73001ff..887b31a 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocationsTest.java @@ -18,7 +18,7 @@ void whenBuildWithIncorrectInputThenThrows() throws EventDateInPastException { final LocalDate dateInPast = LocalDate.of(2022, 7, 12); final LocalTime time = LocalTime.of(14, 20); final String city = "Thessia"; - final String address= "Nightmare Street 102/34"; + final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; final ZoneId timeZone = ZoneId.systemDefault(); final MultipleEventDateWithLocations multipleEventDateWithLocations = MultipleEventDateWithLocations.create(date, time, timeZone, city, address, locationName); @@ -41,22 +41,23 @@ void whenCorrectInputThenBuildCorrectly() { final LocalDate date3 = LocalDate.now().plusDays(2).plusDays(8); final LocalTime time3 = LocalTime.of(18, 30); final String city3 = "Rannoch"; - final String address= "Nightmare Street 102/34"; + final String address = "Nightmare Street 102/34"; final String locationName = "Black hole mirror club"; + final ZoneId timeZone = ZoneId.systemDefault(); Assertions.assertThatNoException().isThrownBy(() -> { - MultipleEventDateWithLocations multipleDate = MultipleEventDateWithLocations.create(date1, time1, ZoneId.systemDefault(), city1, address, locationName) - .add(date2, time2, ZoneId.systemDefault(), city2, address, locationName) - .add(date3, time3, ZoneId.systemDefault(), city3, address, locationName); + MultipleEventDateWithLocations multipleDate = MultipleEventDateWithLocations.create(date1, time1, timeZone, city1, address, locationName) + .add(date2, time2, timeZone, city2, address, locationName) + .add(date3, time3, timeZone, city3, address, locationName); assertThat(multipleDate).isNotNull(); assertThat(multipleDate.eventDateWithLocations()).isNotNull() .hasSize(3) - .extracting(EventDateWithLocation::startDate, EventDateWithLocation::startTime, EventDateWithLocation::city, EventDateWithLocation::address, EventDateWithLocation::locationName) - .contains(tuple(date1, time1, city1, address, locationName), - tuple(date2, time2, city2, address, locationName), - tuple(date3, time3, city3, address, locationName) + .extracting(EventDateWithLocation::startDate, EventDateWithLocation::startTime, EventDateWithLocation::city, EventDateWithLocation::address, EventDateWithLocation::locationName, EventDateWithLocation::timeZone) + .contains(tuple(date1, time1, city1, address, locationName, timeZone), + tuple(date2, time2, city2, address, locationName, timeZone), + tuple(date3, time3, city3, address, locationName, timeZone) ); }); diff --git a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java index 6cd2340..6055903 100644 --- a/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java +++ b/src/test/java/hub/event/scrapers/core/datewithlocation/SingleEventDateWithLocationTest.java @@ -59,6 +59,7 @@ void whenCorrectInputThenBuildCorrectly() { assertEquals(city, singleDateContainsStartDateAndTime.city()); assertEquals(address, singleDateContainsStartDateAndTime.address()); assertEquals(locationName, singleDateContainsStartDateAndTime.locationName()); + assertEquals(timeZone, singleDateContainsStartDateAndTime.timeZone()); assertNull(singleDateContainsStartDateAndTime.endDate()); assertNull(singleDateContainsStartDateAndTime.endTime()); assertTrue(singleDateContainsStartDateAndTime.isSingleDate()); @@ -77,6 +78,7 @@ void whenCorrectInputThenBuildCorrectly() { assertEquals(endTime, fullEventDate.endTime()); assertEquals(address, fullEventDate.address()); assertEquals(locationName, fullEventDate.locationName()); + assertEquals(timeZone, fullEventDate.timeZone()); assertTrue(fullEventDate.isSingleDate()); assertFalse(fullEventDate.isPeriodDate()); }); @@ -132,6 +134,7 @@ void whenCorrectInputThenBuildCorrectly() { assertEquals(endDate, periodDateContainsStartDateAndTime.endDate()); assertEquals(address, periodDateContainsStartDateAndTime.address()); assertEquals(locationName, periodDateContainsStartDateAndTime.locationName()); + assertEquals(timeZone, periodDateContainsStartDateAndTime.timeZone()); assertNull(periodDateContainsStartDateAndTime.endTime()); assertTrue(periodDateContainsStartDateAndTime.isPeriodDate()); assertFalse(periodDateContainsStartDateAndTime.isSingleDate()); @@ -149,6 +152,7 @@ void whenCorrectInputThenBuildCorrectly() { assertEquals(endTime, fullEventDate.endTime()); assertEquals(address, fullEventDate.address()); assertEquals(locationName, fullEventDate.locationName()); + assertEquals(timeZone, fullEventDate.timeZone()); assertTrue(fullEventDate.isPeriodDate()); assertFalse(fullEventDate.isSingleDate()); }); diff --git a/src/test/resources/database/scrapers/core/test_data_init.sql b/src/test/resources/database/scrapers/core/test_data_init.sql new file mode 100644 index 0000000..4b3aafe --- /dev/null +++ b/src/test/resources/database/scrapers/core/test_data_init.sql @@ -0,0 +1,32 @@ +DELETE FROM scraper_error_log; +DELETE FROM scraper_status_log; +DELETE FROM scraper_config; + +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(100000, 'Scraper1', true, 'Europe/Warsaw'); +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(200000, 'Scraper2', true, 'Europe/Warsaw'); +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(300000, 'Scraper3', true, 'Europe/Warsaw'); +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(400000, 'Scraper4', true, 'Europe/Warsaw'); +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(500000, 'Scraper5', true, 'Europe/Warsaw'); + +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(101, 'Error 0', 'ERR_0', '2022-10-19 23:15:00.015', 100000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(102, 'Error 0', 'ERR_0', '2022-10-20 23:16:00.011', 100000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(103, 'Error 0', 'ERR_0', '2022-10-21 23:17:00.013', 100000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(104, 'Error 1', 'ERR_1', '2022-10-23 23:18:00.022', 100000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(105, 'Error 1', 'ERR_1', '2022-10-24 23:19:00.020', 100000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(106, 'Error 1', 'ERR_1', '2022-10-19 23:20:00.019', 200000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(107, 'Error 10', 'ERR_10', '2022-10-30 23:21:00.011', 200000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(108, 'Error 1', 'ERR_1', '2022-10-19 23:22:00.015', 300000); +INSERT INTO scraper_error_log (log_id, description, error_code, error_time, scraper_id) VALUES(109, 'Error 20', 'ERR_20', '2022-10-19 23:23:00.020', 300000); + +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(201, 100, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 230, 100000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(202, 0, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 30, 100000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(203, 0, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 0, 100000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(204, 0, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 2, 200000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(205, null, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 3, 200000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(206, 0, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 310, 200000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(207, 80, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 0, 200000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(208, 100, '2022-10-04 20:43:44.735', '2022-10-04 20:43:58.794', 230, 300000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(209, 100, '2022-10-05 20:43:44.735', '2022-10-05 20:43:58.794', 230, 300000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(210, 100, '2022-11-01 20:43:44.735', '2022-11-01 20:43:58.794', 230, 300000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(211, 100, '2022-11-02 20:43:44.735', '2022-11-02 20:43:58.794', 230, 300000); +INSERT INTO scraper_status_log (log_id, error_count, start_time, finish_time, scanned_event_count, scraper_id) VALUES(212, 100, '2022-11-03 20:43:44.735', '2022-11-03 20:43:58.794', 230, 300000); From f4b6e3027077ab8619d07f4f679799ccb0ceb413 Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Mon, 7 Nov 2022 19:43:57 +0100 Subject: [PATCH 07/10] feat: 17 added some logs and properties class. Moved ScraperLogQueryFacade to subpackage runlog --- .gitignore | 1 + pom.xml | 11 ++- .../JpaLastScrapedEventMarkerRepository.java | 2 +- .../LastScrapedEventMarkerRepository.java | 4 +- .../scrapers/core/ScraperIdNameCache.java | 5 ++ .../scrapers/core/ScraperLogRepository.java | 85 ++++++++++++------- .../scrapers/core/ScraperRunService.java | 63 ++++++++++---- .../{ => runlog}/ScraperLogQueryFacade.java | 6 +- .../runlog/ScraperRunErrorLogProperties.java | 11 +++ .../runlog/ScraperRunStatusLogProperties.java | 12 +++ src/main/resources/log4j2.xml | 48 +++++++++++ .../core/ScraperLogQueryFacadeTest.java | 5 +- .../scrapers/core/ScraperRunServiceTest.java | 2 +- 13 files changed, 197 insertions(+), 58 deletions(-) rename src/main/java/hub/event/scrapers/core/{ => runlog}/ScraperLogQueryFacade.java (87%) create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java create mode 100644 src/main/resources/log4j2.xml diff --git a/.gitignore b/.gitignore index fd6154d..7c2953d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ build/ /data/** .DS_Store +/logs/ diff --git a/pom.xml b/pom.xml index cbb9b71..c376383 100644 --- a/pom.xml +++ b/pom.xml @@ -24,8 +24,17 @@ org.springframework.boot spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-starter-log4j2 - com.h2database h2 diff --git a/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java index 00f1c1d..ce7f76d 100644 --- a/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java +++ b/src/main/java/hub/event/scrapers/core/JpaLastScrapedEventMarkerRepository.java @@ -17,5 +17,5 @@ interface JpaLastScrapedEventMarkerRepository extends JpaRepository= :date AND m.scraperId IN :configurationIds") @Transactional - void updateSetAllActiveById(@Param("configurationIds") List configurationIds, @Param("date")Instant date); + void setAllAsCompleteByConfigurationsIds(@Param("configurationIds") List configurationIds, @Param("date")Instant date); } diff --git a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java index 43fdf15..9eaa9b7 100644 --- a/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java +++ b/src/main/java/hub/event/scrapers/core/LastScrapedEventMarkerRepository.java @@ -23,8 +23,8 @@ void store(LastScrapedEventMarker lastScrapedEventMarker) { jpaLastScrapedEventMarkerRepository.save(entityLastScrapedEventMarker); } - void markAllMarkersByIdsAsActive(List ids) { - jpaLastScrapedEventMarkerRepository.updateSetAllActiveById(ids, ZonedDateTime.now().minusDays(1).toInstant()); + void setAllAsCompleteByConfigurationsIds(List ids) { + jpaLastScrapedEventMarkerRepository.setAllAsCompleteByConfigurationsIds(ids, ZonedDateTime.now().minusDays(1).toInstant()); } Optional findLastCompletedByScraperConfigurationId(Integer scraperId) { diff --git a/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java b/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java index 2124d3c..35591df 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java +++ b/src/main/java/hub/event/scrapers/core/ScraperIdNameCache.java @@ -1,6 +1,8 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.scraper.ScraperConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import java.util.Collection; @@ -8,6 +10,8 @@ @Component class ScraperIdNameCache { + + private final Logger logger = LoggerFactory.getLogger(ScraperIdNameCache.class); private final ConcurrentHashMap nameToIdMap; private final ConcurrentHashMap idToNameMap; @@ -33,5 +37,6 @@ void add(Collection scraperConfigs) { public void add(ScraperConfig scraperConfig) { idToNameMap.put(scraperConfig.scraperId(), scraperConfig.configurationName()); nameToIdMap.put(scraperConfig.configurationName(), scraperConfig.scraperId()); + logger.debug("Added values to cache for config: {}", scraperConfig); } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java b/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java index c3d74cb..c4f605e 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java +++ b/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java @@ -1,9 +1,8 @@ package hub.event.scrapers.core; -import hub.event.scrapers.core.runlog.ErrorLogSearchQuery; -import hub.event.scrapers.core.runlog.ScraperRunErrorLog; -import hub.event.scrapers.core.runlog.ScraperRunStatusLog; -import hub.event.scrapers.core.runlog.StatusLogSearchQuery; +import hub.event.scrapers.core.runlog.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -18,13 +17,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; @Repository -class ScraperLogRepository { - +public class ScraperLogRepository { + private final Logger logger = LoggerFactory.getLogger(ScraperLogRepository.class); private final JpaScraperRunLogRepository jpaScraperRunLogRepository; private final JpaScraperRunErrorRepository jpaScraperRunErrorRepository; - private final ScraperIdNameCache scraperIdNameCache; @Autowired @@ -36,16 +35,20 @@ public ScraperLogRepository(JpaScraperRunLogRepository jpaScraperRunLogRepositor void save(ScraperRunStatusLog scraperRunStatusLog) { EntityScraperRunStatusLog entityScraperRunStatusLog = mapToEntity(scraperRunStatusLog); + logger.debug("Mapped EntityScraperRunStatusLog :{}", entityScraperRunStatusLog); jpaScraperRunLogRepository.save(entityScraperRunStatusLog); } void save(ScraperRunErrorLog scraperRunError) { EntityScraperRunErrorLog entityScraperRunErrorLog = mapToEntity(scraperRunError); + logger.debug("Mapped EntityScraperRunErrorLog :{}", entityScraperRunErrorLog); jpaScraperRunErrorRepository.save(entityScraperRunErrorLog); } - List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { - final Sort sort = Sort.by(Sort.Direction.ASC, "scraperId"); + @Deprecated + //TODO move to runlog subpackage + public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { + final Sort sort = Sort.by(Sort.Direction.ASC, ScraperRunErrorLogProperties.SCRAPER_ID); final Specification findAllSpecification = new FindAllErrorLogSpecification(errorLogSearchQuery, scraperIdNameCache); final Pageable pageable = extractPageSettings(errorLogSearchQuery, sort); @@ -56,8 +59,10 @@ List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery } - List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { - final Sort sort = Sort.by(Sort.Direction.ASC, "scraperId"); + @Deprecated + //TODO move to runlog subpackage + public List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { + final Sort sort = Sort.by(Sort.Direction.ASC, ScraperRunErrorLogProperties.SCRAPER_ID); final Specification findAllSpecification = new FindAllStatusLogSpecification(statusLogSearchQuery, scraperIdNameCache); final Pageable pageable = extractPageSettings(statusLogSearchQuery, sort); @@ -67,6 +72,8 @@ List findAllStatusLog(StatusLogSearchQuery statusLogSearchQ .toList(); } + @Deprecated + //TODO move to runlog subpackage private List findAllStatusLog(Specification findAllSpecification, Pageable pageable, Sort sort) { if (Objects.isNull(pageable)) { return jpaScraperRunLogRepository.findAll(findAllSpecification, sort); @@ -74,6 +81,8 @@ private List findAllStatusLog(Specification findAllErrorLog(Specification findAllSpecification, Pageable pageable, Sort sort) { if (Objects.isNull(pageable)) { return jpaScraperRunErrorRepository.findAll(findAllSpecification, sort); @@ -81,6 +90,8 @@ private List findAllErrorLog(Specification { + private final transient Logger logger = LoggerFactory.getLogger(FindAllStatusLogSpecification.class); private final transient StatusLogSearchQuery searchQuery; private final transient ScraperIdNameCache scraperIdNameCache; @@ -158,52 +172,60 @@ public Predicate toPredicate(Root root, CriteriaQuery .stream() .map(scraperIdNameCache::getIdByScraperName) .toList(); - outputPredicates.add(criteriaBuilder.in(root.get("scraperId")).value(idsLists)); + outputPredicates.add(criteriaBuilder.in(root.get(ScraperRunStatusLogProperties.SCRAPER_ID)).value(idsLists)); } if (Objects.nonNull(searchQuery.startTimeFrom())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("startTime"), searchQuery.startTimeFrom())); + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.START_TIME), searchQuery.startTimeFrom())); } if (Objects.nonNull(searchQuery.startTimeTo())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("startTime"), searchQuery.startTimeTo())); + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(ScraperRunStatusLogProperties.START_TIME), searchQuery.startTimeTo())); } if (Objects.nonNull(searchQuery.finishTimeFrom())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("finishTime"), searchQuery.finishTimeFrom())); + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.FINISH_TIME), searchQuery.finishTimeFrom())); } if (Objects.nonNull(searchQuery.finishTimeTo())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("finishTime"), searchQuery.finishTimeTo())); + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(ScraperRunStatusLogProperties.FINISH_TIME), searchQuery.finishTimeTo())); } if (Objects.nonNull(searchQuery.hasScannedEvent())) { final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasScannedEvent()) - ? criteriaBuilder.greaterThan(root.get("scannedEventCount"), 0) - : criteriaBuilder.or(criteriaBuilder.equal(root.get("scannedEventCount"), 0), criteriaBuilder.isNull(root.get("scannedEventCount"))); + ? criteriaBuilder.greaterThan(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), 0) + : criteriaBuilder.or(criteriaBuilder.equal(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), 0), criteriaBuilder.isNull(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT))); outputPredicates.add(predicate); } else if (Objects.nonNull(searchQuery.scannedEventGreaterThanOrEqualTo())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("scannedEventCount"), searchQuery.scannedEventGreaterThanOrEqualTo())); + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), searchQuery.scannedEventGreaterThanOrEqualTo())); } if (Objects.nonNull(searchQuery.hasErrors())) { final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasErrors()) - ? criteriaBuilder.greaterThan(root.get("errorCount"), 0) - : criteriaBuilder.or(criteriaBuilder.equal(root.get("errorCount"), 0), criteriaBuilder.isNull(root.get("errorCount"))); + ? criteriaBuilder.greaterThan(root.get(ScraperRunStatusLogProperties.ERROR_COUNT), 0) + : criteriaBuilder.or(criteriaBuilder.equal(root.get(ScraperRunStatusLogProperties.ERROR_COUNT), 0), criteriaBuilder.isNull(root.get(ScraperRunStatusLogProperties.ERROR_COUNT))); outputPredicates.add(predicate); } else if (Objects.nonNull(searchQuery.errorCountGreaterThanOrEqualTo())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("errorCount"), searchQuery.errorCountGreaterThanOrEqualTo())); + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.ERROR_COUNT), searchQuery.errorCountGreaterThanOrEqualTo())); } - return outputPredicates.stream() + final Predicate outputPredicate = outputPredicates.stream() .reduce(criteriaBuilder::and) .orElse(null); + + Optional.ofNullable(outputPredicate) + .ifPresent(predicate -> logger.debug("FindAllStatusLogSpecification output predicate: {}", predicate)); + + return outputPredicate; } + } private static class FindAllErrorLogSpecification implements Specification { + + private final transient Logger logger = LoggerFactory.getLogger(FindAllErrorLogSpecification.class); private final transient ErrorLogSearchQuery searchQuery; private final transient ScraperIdNameCache scraperIdNameCache; @@ -221,28 +243,33 @@ public Predicate toPredicate(Root root, CriteriaQuery< .stream() .map(scraperIdNameCache::getIdByScraperName) .toList(); - outputPredicates.add(criteriaBuilder.in(root.get("scraperId")).value(idsLists)); + outputPredicates.add(criteriaBuilder.in(root.get(ScraperRunErrorLogProperties.SCRAPER_ID)).value(idsLists)); } if (Objects.nonNull(searchQuery.fromDate())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get("time"), searchQuery.fromDate())); + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunErrorLogProperties.TIME), searchQuery.fromDate())); } if (Objects.nonNull(searchQuery.toDate())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get("time"), searchQuery.toDate())); + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(ScraperRunErrorLogProperties.TIME), searchQuery.toDate())); } if (Objects.nonNull(searchQuery.description())) { - outputPredicates.add(criteriaBuilder.equal(root.get("description"), searchQuery.description())); + outputPredicates.add(criteriaBuilder.equal(root.get(ScraperRunErrorLogProperties.DESCRIPTION), searchQuery.description())); } if (searchQuery.hasErrorCodes()) { - outputPredicates.add(criteriaBuilder.in(root.get("errorCode")).value(searchQuery.errorCodes())); + outputPredicates.add(criteriaBuilder.in(root.get(ScraperRunErrorLogProperties.ERROR_CODE)).value(searchQuery.errorCodes())); } - return outputPredicates.stream() + final Predicate outputPredicate = outputPredicates.stream() .reduce(criteriaBuilder::and) .orElse(null); + + Optional.ofNullable(outputPredicate) + .ifPresent(predicate -> logger.debug("FindAllErrorLogSpecification output predicate: {}", predicate)); + + return outputPredicate; } } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperRunService.java b/src/main/java/hub/event/scrapers/core/ScraperRunService.java index f22dd9a..7a350cb 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperRunService.java +++ b/src/main/java/hub/event/scrapers/core/ScraperRunService.java @@ -1,12 +1,16 @@ package hub.event.scrapers.core; import hub.event.scrapers.core.scraper.ScraperConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -15,7 +19,9 @@ @Service @EnableScheduling -class ScraperRunService { +class ScraperRunService implements ApplicationRunner { + + private final Logger logger = LoggerFactory.getLogger(ScraperRunService.class); private final ScraperConfigRepository scraperConfigRepository; private final EventFacadeAdapter eventFacadeAdapter; private final LastScrapedEventMarkerRepository lastScrapedEventMarkerRepository; @@ -31,32 +37,26 @@ class ScraperRunService { this.scraperIdNameCache = scraperIdNameCache; } - @PostConstruct - void createScrapersConfigsIfMissingAndFillIdNameCache() { - final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); - scraperIdNameCache.add(scraperConfigs); - - final List scrapersWithoutConfig = getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); - - for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { - ScraperConfig scraperConfig = scraperConfigRepository.create(pageScraperPort.configurationName(), pageScraperPort.timeZone(), true); - scraperIdNameCache.add(scraperConfig); - } - } - @Scheduled(cron = "${scrapers.run.cron.expression}") // @Scheduled(cron = "0 0 * * *") void start() { + logger.info("Run scraper schedule"); final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); final List pageScrapersToRun = getActiveScrapersThatShouldBeRun(scraperConfigs); + logger.debug("Scrapers to run:"); + pageScrapersToRun.forEach(pageScraperPort -> logger.debug(pageScraperPort.configurationName())); final List scrapedEventList = runScrapersForEvents(pageScrapersToRun); + logger.info("Scan done by all active scrapers, founded events = {}", scrapedEventList.size()); + logger.debug("Scanned events:"); + scrapedEventList.forEach(scrapedEvent -> logger.debug(scrapedEvent.toString())); eventFacadeAdapter.saveAll(scrapedEventList); final List runScraperConfigurationsIds = getRunScraperConfigurationsIds(pageScrapersToRun); - lastScrapedEventMarkerRepository.markAllMarkersByIdsAsActive(runScraperConfigurationsIds); - + lastScrapedEventMarkerRepository.setAllAsCompleteByConfigurationsIds(runScraperConfigurationsIds); + logger.info("Events saved, markers completed "); + logger.info("Run scraper schedule done"); } private List getScraperThatConfigNotFoundByConfigurationName(Collection scraperConfigs) { @@ -80,15 +80,42 @@ private List getActiveScrapersThatShouldBeRun(Collection runScrapersForEvents(List pageScrapersToRun) { return pageScrapersToRun.parallelStream() - .map(PageScraperPort::scrap) + .map(this::tryRunScraperTask) .flatMap(Collection::stream) .toList(); } + private Collection tryRunScraperTask(PageScraperPort pageScraperPort) { + try { + return pageScraperPort.scrap(); + } catch (Exception runScraperTaskException) { + logger.error(String.format("Run scraper %s failed", pageScraperPort.configurationName()), runScraperTaskException); + return new ArrayList<>(); + } + } + private List getRunScraperConfigurationsIds(List pageScrapersToRun) { return pageScrapersToRun.stream() .map(PageScraperPort::configurationName) .map(scraperIdNameCache::getIdByScraperName) .toList(); } + + @Override + public void run(ApplicationArguments args) { + createScrapersConfigsIfMissingAndFillIdNameCache(); + } + + void createScrapersConfigsIfMissingAndFillIdNameCache() { + final Collection scraperConfigs = scraperConfigRepository.allScraperConfigs(); + scraperIdNameCache.add(scraperConfigs); + + final List scrapersWithoutConfig = getScraperThatConfigNotFoundByConfigurationName(scraperConfigs); + + for (PageScraperPort pageScraperPort : scrapersWithoutConfig) { + ScraperConfig scraperConfig = scraperConfigRepository.create(pageScraperPort.configurationName(), pageScraperPort.timeZone(), true); + scraperIdNameCache.add(scraperConfig); + } + logger.info("Scrapers config validated and added to cache"); + } } diff --git a/src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java similarity index 87% rename from src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java rename to src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java index 4cd298d..d0e0912 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperLogQueryFacade.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java @@ -1,6 +1,6 @@ -package hub.event.scrapers.core; +package hub.event.scrapers.core.runlog; -import hub.event.scrapers.core.runlog.*; +import hub.event.scrapers.core.ScraperLogRepository; import org.springframework.stereotype.Service; import java.util.List; @@ -12,9 +12,11 @@ public class ScraperLogQueryFacade { public ScraperLogQueryFacade(ScraperLogRepository scraperLogRepository) { this.scraperLogRepository = scraperLogRepository; } + public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { return scraperLogRepository.findAllErrorLog(errorLogSearchQuery); } + public List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { return scraperLogRepository.findAllStatusLog(statusLogSearchQuery); } diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java new file mode 100644 index 0000000..c890fab --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java @@ -0,0 +1,11 @@ +package hub.event.scrapers.core.runlog; + +public class ScraperRunErrorLogProperties { + public static final String SCRAPER_ID = "scraperId"; + public static final String TIME = "time"; + public static final String DESCRIPTION = "description"; + public static final String ERROR_CODE = "errorCode"; + + private ScraperRunErrorLogProperties() { + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java new file mode 100644 index 0000000..0b95e22 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java @@ -0,0 +1,12 @@ +package hub.event.scrapers.core.runlog; + +public class ScraperRunStatusLogProperties { + public static final String SCRAPER_ID = "scraperId"; + public static final String START_TIME = "startTime"; + public static final String FINISH_TIME = "finishTime"; + public static final String SCANNED_EVENT_COUNT = "scannedEventCount"; + public static final String ERROR_COUNT = "errorCount"; + + private ScraperRunStatusLogProperties() { + } +} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 0000000..b131f92 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,48 @@ + + + + + + + + + + %d %p %C{1.} [%t] %m%n + + + + + + + + + + + %d %p %C{1.} [%t] %m%n + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java b/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java index 5d25aa4..7318139 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java @@ -1,9 +1,6 @@ package hub.event.scrapers.core; -import hub.event.scrapers.core.runlog.ErrorLogSearchQuery; -import hub.event.scrapers.core.runlog.ScraperRunErrorLog; -import hub.event.scrapers.core.runlog.ScraperRunStatusLog; -import hub.event.scrapers.core.runlog.StatusLogSearchQuery; +import hub.event.scrapers.core.runlog.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; diff --git a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java index 0298c73..b5be512 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java +++ b/src/test/java/hub/event/scrapers/core/ScraperRunServiceTest.java @@ -166,7 +166,7 @@ void whenScrapedEventSavedThenMakeLastScrapedEventMarkerDraftActive() throws Eve //then scraperRunService.start(); - verify(lastScrapedEventMarkerRepository).markAllMarkersByIdsAsActive(List.of(activeScraper1Config.scraperId(), activeScraper2Config.scraperId())); + verify(lastScrapedEventMarkerRepository).setAllAsCompleteByConfigurationsIds(List.of(activeScraper1Config.scraperId(), activeScraper2Config.scraperId())); } } From 62e848f626369bab912808d2cef0bdf7eda4489d Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Thu, 10 Nov 2022 15:08:00 +0100 Subject: [PATCH 08/10] feat: 17 fix sort in ScraperLogQueryFacade --- .../core/EntityScraperRunErrorLog.java | 2 +- .../core/EntityScraperRunStatusLog.java | 2 +- .../scrapers/core/ScraperLogRepository.java | 219 +----------------- .../EntityFindAllErrorLogSpecification.java | 60 +++++ .../EntityFindAllStatusLogSpecification.java | 83 +++++++ .../core/runlog/EntityScraperConfig.java | 55 +++++ .../core/runlog/EntityScraperRunErrorLog.java | 99 ++++++++ .../EntityScraperRunErrorLogProperties.java | 13 ++ .../runlog/EntityScraperRunStatusLog.java | 111 +++++++++ .../EntityScraperRunStatusLogProperties.java | 14 ++ .../JpaScraperRunErrorQueryRepository.java | 7 + .../JpaScraperRunLogQueryRepository.java | 7 + .../core/runlog/ScraperLogQueryFacade.java | 7 +- .../runlog/ScraperLogQueryRepository.java | 100 ++++++++ .../runlog/ScraperRunErrorLogProperties.java | 11 - .../runlog/ScraperRunStatusLogProperties.java | 12 - .../resources/database/change/change_2.yaml | 32 ++- .../ScraperLogQueryFacadeTest.java | 188 ++++++++------- .../database/scrapers/core/test_data_init.sql | 4 +- 19 files changed, 676 insertions(+), 350 deletions(-) create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityFindAllErrorLogSpecification.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityFindAllStatusLogSpecification.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityScraperConfig.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLogProperties.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLogProperties.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunErrorQueryRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunLogQueryRepository.java create mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryRepository.java delete mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java delete mode 100644 src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java rename src/test/java/hub/event/scrapers/core/{ => runlog}/ScraperLogQueryFacadeTest.java (81%) diff --git a/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java index 33dff8e..db6aa05 100644 --- a/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java +++ b/src/main/java/hub/event/scrapers/core/EntityScraperRunErrorLog.java @@ -5,7 +5,7 @@ import java.time.Instant; import java.util.Objects; -@Entity(name = "Scraper_error_log") +@Entity(name = "scraperErrorLog") class EntityScraperRunErrorLog implements Serializable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java b/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java index 89bc013..1d95e6c 100644 --- a/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java +++ b/src/main/java/hub/event/scrapers/core/EntityScraperRunStatusLog.java @@ -5,7 +5,7 @@ import java.time.Instant; import java.util.Objects; -@Entity(name = "Scraper_status_log") +@Entity(name = "scraperStatusLog") class EntityScraperRunStatusLog implements Serializable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java b/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java index c4f605e..f62f6ce 100644 --- a/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java +++ b/src/main/java/hub/event/scrapers/core/ScraperLogRepository.java @@ -1,24 +1,12 @@ package hub.event.scrapers.core; -import hub.event.scrapers.core.runlog.*; +import hub.event.scrapers.core.runlog.ScraperRunErrorLog; +import hub.event.scrapers.core.runlog.ScraperRunStatusLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; -import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Repository; -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Predicate; -import javax.persistence.criteria.Root; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - @Repository public class ScraperLogRepository { private final Logger logger = LoggerFactory.getLogger(ScraperLogRepository.class); @@ -45,75 +33,6 @@ void save(ScraperRunErrorLog scraperRunError) { jpaScraperRunErrorRepository.save(entityScraperRunErrorLog); } - @Deprecated - //TODO move to runlog subpackage - public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { - final Sort sort = Sort.by(Sort.Direction.ASC, ScraperRunErrorLogProperties.SCRAPER_ID); - final Specification findAllSpecification = new FindAllErrorLogSpecification(errorLogSearchQuery, scraperIdNameCache); - final Pageable pageable = extractPageSettings(errorLogSearchQuery, sort); - - return findAllErrorLog(findAllSpecification, pageable, sort) - .stream() - .map(this::mapToLog) - .toList(); - - } - - @Deprecated - //TODO move to runlog subpackage - public List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { - final Sort sort = Sort.by(Sort.Direction.ASC, ScraperRunErrorLogProperties.SCRAPER_ID); - final Specification findAllSpecification = new FindAllStatusLogSpecification(statusLogSearchQuery, scraperIdNameCache); - final Pageable pageable = extractPageSettings(statusLogSearchQuery, sort); - - return findAllStatusLog(findAllSpecification, pageable, sort) - .stream() - .map(this::mapToLog) - .toList(); - } - - @Deprecated - //TODO move to runlog subpackage - private List findAllStatusLog(Specification findAllSpecification, Pageable pageable, Sort sort) { - if (Objects.isNull(pageable)) { - return jpaScraperRunLogRepository.findAll(findAllSpecification, sort); - } - return jpaScraperRunLogRepository.findAll(findAllSpecification, pageable).toList(); - } - - @Deprecated - //TODO move to runlog subpackage - private List findAllErrorLog(Specification findAllSpecification, Pageable pageable, Sort sort) { - if (Objects.isNull(pageable)) { - return jpaScraperRunErrorRepository.findAll(findAllSpecification, sort); - } - return jpaScraperRunErrorRepository.findAll(findAllSpecification, pageable).toList(); - } - - @Deprecated - //TODO move to runlog subpackage - private ScraperRunStatusLog mapToLog(EntityScraperRunStatusLog entityScraperRunStatusLog) { - final String scraperName = scraperIdNameCache.getScraperNameById(entityScraperRunStatusLog.getScraperId()); - - return new ScraperRunStatusLog( - scraperName, - entityScraperRunStatusLog.getStartTime(), - entityScraperRunStatusLog.getFinishTime(), - entityScraperRunStatusLog.getScannedEventCount(), - entityScraperRunStatusLog.getErrorCount() - ); - } - - @Deprecated - //TODO move to runlog subpackage - private ScraperRunErrorLog mapToLog(EntityScraperRunErrorLog entityScraperRunErrorLog) { - final String scraperName = scraperIdNameCache.getScraperNameById(entityScraperRunErrorLog.getScraperId()); - - return new ScraperRunErrorLog(scraperName, - entityScraperRunErrorLog.getTime(), - entityScraperRunErrorLog.getErrorCode(), - entityScraperRunErrorLog.getDescription()); - } private EntityScraperRunStatusLog mapToEntity(ScraperRunStatusLog scraperRunStatusLog) { final Integer scraperId = scraperIdNameCache.getIdByScraperName(scraperRunStatusLog.configurationName()); @@ -138,138 +57,4 @@ private EntityScraperRunErrorLog mapToEntity(ScraperRunErrorLog scraperRunError) ); } - private Pageable extractPageSettings(StatusLogSearchQuery statusLogSearchQuery, Sort sort) { - return statusLogSearchQuery.hasPageSetting() - ? PageRequest.of(statusLogSearchQuery.page(), statusLogSearchQuery.pageSize(), sort) - : null; - } - - private Pageable extractPageSettings(ErrorLogSearchQuery errorLogSearchQuery, Sort sort) { - return errorLogSearchQuery.hasPageSetting() - ? PageRequest.of(errorLogSearchQuery.page(), errorLogSearchQuery.pageSize(), sort) - : null; - } - - private static class FindAllStatusLogSpecification implements Specification { - - private final transient Logger logger = LoggerFactory.getLogger(FindAllStatusLogSpecification.class); - private final transient StatusLogSearchQuery searchQuery; - private final transient ScraperIdNameCache scraperIdNameCache; - - public FindAllStatusLogSpecification(StatusLogSearchQuery statusLogSearchQuery, ScraperIdNameCache scraperIdNameCache) { - - this.searchQuery = statusLogSearchQuery; - this.scraperIdNameCache = scraperIdNameCache; - } - - - @Override - public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { - List outputPredicates = new ArrayList<>(); - - if (searchQuery.hasConfigurationNames()) { - final List idsLists = searchQuery.configurationNames() - .stream() - .map(scraperIdNameCache::getIdByScraperName) - .toList(); - outputPredicates.add(criteriaBuilder.in(root.get(ScraperRunStatusLogProperties.SCRAPER_ID)).value(idsLists)); - } - - if (Objects.nonNull(searchQuery.startTimeFrom())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.START_TIME), searchQuery.startTimeFrom())); - } - - if (Objects.nonNull(searchQuery.startTimeTo())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(ScraperRunStatusLogProperties.START_TIME), searchQuery.startTimeTo())); - } - - if (Objects.nonNull(searchQuery.finishTimeFrom())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.FINISH_TIME), searchQuery.finishTimeFrom())); - } - - if (Objects.nonNull(searchQuery.finishTimeTo())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(ScraperRunStatusLogProperties.FINISH_TIME), searchQuery.finishTimeTo())); - } - - if (Objects.nonNull(searchQuery.hasScannedEvent())) { - final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasScannedEvent()) - ? criteriaBuilder.greaterThan(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), 0) - : criteriaBuilder.or(criteriaBuilder.equal(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), 0), criteriaBuilder.isNull(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT))); - - outputPredicates.add(predicate); - } else if (Objects.nonNull(searchQuery.scannedEventGreaterThanOrEqualTo())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), searchQuery.scannedEventGreaterThanOrEqualTo())); - } - - if (Objects.nonNull(searchQuery.hasErrors())) { - final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasErrors()) - ? criteriaBuilder.greaterThan(root.get(ScraperRunStatusLogProperties.ERROR_COUNT), 0) - : criteriaBuilder.or(criteriaBuilder.equal(root.get(ScraperRunStatusLogProperties.ERROR_COUNT), 0), criteriaBuilder.isNull(root.get(ScraperRunStatusLogProperties.ERROR_COUNT))); - - outputPredicates.add(predicate); - } else if (Objects.nonNull(searchQuery.errorCountGreaterThanOrEqualTo())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunStatusLogProperties.ERROR_COUNT), searchQuery.errorCountGreaterThanOrEqualTo())); - } - - final Predicate outputPredicate = outputPredicates.stream() - .reduce(criteriaBuilder::and) - .orElse(null); - - Optional.ofNullable(outputPredicate) - .ifPresent(predicate -> logger.debug("FindAllStatusLogSpecification output predicate: {}", predicate)); - - return outputPredicate; - } - - } - - private static class FindAllErrorLogSpecification implements Specification { - - private final transient Logger logger = LoggerFactory.getLogger(FindAllErrorLogSpecification.class); - private final transient ErrorLogSearchQuery searchQuery; - private final transient ScraperIdNameCache scraperIdNameCache; - - public FindAllErrorLogSpecification(ErrorLogSearchQuery errorLogSearchQuery, ScraperIdNameCache scraperIdNameCache) { - this.searchQuery = errorLogSearchQuery; - this.scraperIdNameCache = scraperIdNameCache; - } - - @Override - public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { - List outputPredicates = new ArrayList<>(); - - if (searchQuery.hasConfigurationNames()) { - final List idsLists = searchQuery.configurationNames() - .stream() - .map(scraperIdNameCache::getIdByScraperName) - .toList(); - outputPredicates.add(criteriaBuilder.in(root.get(ScraperRunErrorLogProperties.SCRAPER_ID)).value(idsLists)); - } - - if (Objects.nonNull(searchQuery.fromDate())) { - outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(ScraperRunErrorLogProperties.TIME), searchQuery.fromDate())); - } - - if (Objects.nonNull(searchQuery.toDate())) { - outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(ScraperRunErrorLogProperties.TIME), searchQuery.toDate())); - } - - if (Objects.nonNull(searchQuery.description())) { - outputPredicates.add(criteriaBuilder.equal(root.get(ScraperRunErrorLogProperties.DESCRIPTION), searchQuery.description())); - } - - if (searchQuery.hasErrorCodes()) { - outputPredicates.add(criteriaBuilder.in(root.get(ScraperRunErrorLogProperties.ERROR_CODE)).value(searchQuery.errorCodes())); - } - - final Predicate outputPredicate = outputPredicates.stream() - .reduce(criteriaBuilder::and) - .orElse(null); - - Optional.ofNullable(outputPredicate) - .ifPresent(predicate -> logger.debug("FindAllErrorLogSpecification output predicate: {}", predicate)); - - return outputPredicate; - } - } } diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityFindAllErrorLogSpecification.java b/src/main/java/hub/event/scrapers/core/runlog/EntityFindAllErrorLogSpecification.java new file mode 100644 index 0000000..a5c306b --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityFindAllErrorLogSpecification.java @@ -0,0 +1,60 @@ +package hub.event.scrapers.core.runlog; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.jpa.domain.Specification; + +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +class EntityFindAllErrorLogSpecification implements Specification { + + private final transient Logger logger = LoggerFactory.getLogger(EntityFindAllErrorLogSpecification.class); + private final transient ErrorLogSearchQuery searchQuery; + + EntityFindAllErrorLogSpecification(ErrorLogSearchQuery errorLogSearchQuery) { + this.searchQuery = errorLogSearchQuery; + } + + @Override + public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { + final var scraperConfig = root.join("scraperConfig"); + + List outputPredicates = new ArrayList<>(); + + if (searchQuery.hasConfigurationNames()) { + outputPredicates.add(criteriaBuilder.in(scraperConfig.get(EntityScraperRunErrorLogProperties.SCRAPER_CONFIGURATION_NAME)).value(searchQuery.configurationNames())); + } + + if (Objects.nonNull(searchQuery.fromDate())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(EntityScraperRunErrorLogProperties.TIME), searchQuery.fromDate())); + } + + if (Objects.nonNull(searchQuery.toDate())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(EntityScraperRunErrorLogProperties.TIME), searchQuery.toDate())); + } + + if (Objects.nonNull(searchQuery.description())) { + outputPredicates.add(criteriaBuilder.equal(root.get(EntityScraperRunErrorLogProperties.DESCRIPTION), searchQuery.description())); + } + + if (searchQuery.hasErrorCodes()) { + outputPredicates.add(criteriaBuilder.in(root.get(EntityScraperRunErrorLogProperties.ERROR_CODE)).value(searchQuery.errorCodes())); + } + + final Predicate outputPredicate = outputPredicates.stream() + .reduce(criteriaBuilder::and) + .orElse(null); + + Optional.ofNullable(outputPredicate) + .ifPresent(predicate -> logger.debug("FindAllErrorLogSpecification output predicate: {}", predicate)); + + return outputPredicate; + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityFindAllStatusLogSpecification.java b/src/main/java/hub/event/scrapers/core/runlog/EntityFindAllStatusLogSpecification.java new file mode 100644 index 0000000..b0b8db1 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityFindAllStatusLogSpecification.java @@ -0,0 +1,83 @@ +package hub.event.scrapers.core.runlog; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.jpa.domain.Specification; + +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +class EntityFindAllStatusLogSpecification implements Specification { + + private final transient Logger logger = LoggerFactory.getLogger(EntityFindAllStatusLogSpecification.class); + private final transient StatusLogSearchQuery searchQuery; + + + EntityFindAllStatusLogSpecification(StatusLogSearchQuery statusLogSearchQuery) { + + this.searchQuery = statusLogSearchQuery; + } + + @Override + public Predicate toPredicate(Root root, CriteriaQuery query, CriteriaBuilder criteriaBuilder) { + final var scraperConfig = root.join("scraperConfig"); + + List outputPredicates = new ArrayList<>(); + + if (searchQuery.hasConfigurationNames()) { + outputPredicates.add(criteriaBuilder.in(scraperConfig.get(EntityScraperRunStatusLogProperties.SCRAPER_CONFIGURATION_NAME)).value(searchQuery.configurationNames())); + } + + if (Objects.nonNull(searchQuery.startTimeFrom())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(EntityScraperRunStatusLogProperties.START_TIME), searchQuery.startTimeFrom())); + } + + if (Objects.nonNull(searchQuery.startTimeTo())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(EntityScraperRunStatusLogProperties.START_TIME), searchQuery.startTimeTo())); + } + + if (Objects.nonNull(searchQuery.finishTimeFrom())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(EntityScraperRunStatusLogProperties.FINISH_TIME), searchQuery.finishTimeFrom())); + } + + if (Objects.nonNull(searchQuery.finishTimeTo())) { + outputPredicates.add(criteriaBuilder.lessThanOrEqualTo(root.get(EntityScraperRunStatusLogProperties.FINISH_TIME), searchQuery.finishTimeTo())); + } + + if (Objects.nonNull(searchQuery.hasScannedEvent())) { + final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasScannedEvent()) + ? criteriaBuilder.greaterThan(root.get(EntityScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), 0) + : criteriaBuilder.or(criteriaBuilder.equal(root.get(EntityScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), 0), criteriaBuilder.isNull(root.get(EntityScraperRunStatusLogProperties.SCANNED_EVENT_COUNT))); + + outputPredicates.add(predicate); + } else if (Objects.nonNull(searchQuery.scannedEventGreaterThanOrEqualTo())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(EntityScraperRunStatusLogProperties.SCANNED_EVENT_COUNT), searchQuery.scannedEventGreaterThanOrEqualTo())); + } + + if (Objects.nonNull(searchQuery.hasErrors())) { + final Predicate predicate = Boolean.TRUE.equals(searchQuery.hasErrors()) + ? criteriaBuilder.greaterThan(root.get(EntityScraperRunStatusLogProperties.ERROR_COUNT), 0) + : criteriaBuilder.or(criteriaBuilder.equal(root.get(EntityScraperRunStatusLogProperties.ERROR_COUNT), 0), criteriaBuilder.isNull(root.get(EntityScraperRunStatusLogProperties.ERROR_COUNT))); + + outputPredicates.add(predicate); + } else if (Objects.nonNull(searchQuery.errorCountGreaterThanOrEqualTo())) { + outputPredicates.add(criteriaBuilder.greaterThanOrEqualTo(root.get(EntityScraperRunStatusLogProperties.ERROR_COUNT), searchQuery.errorCountGreaterThanOrEqualTo())); + } + + final Predicate outputPredicate = outputPredicates.stream() + .reduce(criteriaBuilder::and) + .orElse(null); + + Optional.ofNullable(outputPredicate) + .ifPresent(predicate -> logger.debug("FindAllStatusLogSpecification output predicate: {}", predicate)); + + return outputPredicate; + } + +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperConfig.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperConfig.java new file mode 100644 index 0000000..61b2c61 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperConfig.java @@ -0,0 +1,55 @@ +package hub.event.scrapers.core.runlog; + +import javax.persistence.*; +import java.io.Serializable; +import java.util.Objects; + +@Entity(name = "queryScraperConfig") +@Table(name = "scraper_config") +class EntityScraperConfig implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer scraperId; + @Column(unique = true) + private String configurationName; + + EntityScraperConfig() { + } + + Integer getScraperId() { + return scraperId; + } + + void setScraperId(Integer scraperId) { + this.scraperId = scraperId; + } + + String getConfigurationName() { + return configurationName; + } + + void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityScraperConfig that = (EntityScraperConfig) o; + return Objects.equals(scraperId, that.scraperId) && Objects.equals(configurationName, that.configurationName); + } + + @Override + public int hashCode() { + return Objects.hash(scraperId, configurationName); + } + + @Override + public String toString() { + return "EntityScraperConfig{" + + "scraperId=" + scraperId + + ", configurationName='" + configurationName + '\'' + + '}'; + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java new file mode 100644 index 0000000..5582875 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java @@ -0,0 +1,99 @@ +package hub.event.scrapers.core.runlog; + +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + +import javax.persistence.*; +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +@Entity(name = "queryScraperErrorLog") +@Table(name = "scraper_error_log") +class EntityScraperRunErrorLog implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer logId; + + @Column(nullable = false, name = "error_time") + private Instant time; + + @Column(nullable = false) + private String errorCode; + private String description; + @ManyToOne + @JoinColumn(name = "scraperId", nullable = false, insertable = false, updatable = false) + @Fetch(FetchMode.JOIN) + private EntityScraperConfig scraperConfig; + + EntityScraperRunErrorLog() { + } + + Integer getLogId() { + return logId; + } + + void setLogId(Integer logId) { + this.logId = logId; + } + + Instant getTime() { + return time; + } + + void setTime(Instant time) { + this.time = time; + } + + String getErrorCode() { + return errorCode; + } + + void setErrorCode(String errorCode) { + this.errorCode = errorCode; + } + + String getDescription() { + return description; + } + + void setDescription(String description) { + this.description = description; + } + + void setScraperConfig(EntityScraperConfig scraperConfig) { + this.scraperConfig = scraperConfig; + } + + EntityScraperConfig getScraperConfig() { + return scraperConfig; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityScraperRunErrorLog that = (EntityScraperRunErrorLog) o; + return Objects.equals(logId, that.logId) && Objects.equals(time, that.time) && Objects.equals(errorCode, that.errorCode) && Objects.equals(description, that.description) && Objects.equals(scraperConfig, that.scraperConfig); + } + + @Override + public int hashCode() { + return Objects.hash(logId, time, errorCode, description, scraperConfig); + } + + @Override + public String toString() { + return "EntityScraperRunErrorLog{" + + "logId=" + logId + + ", time=" + time + + ", errorCode='" + errorCode + '\'' + + ", description='" + description + '\'' + + ", scraperConfig=" + scraperConfig + + '}'; + } + + String getScraperConfigurationName() { + return scraperConfig.getConfigurationName(); + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLogProperties.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLogProperties.java new file mode 100644 index 0000000..b44428c --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLogProperties.java @@ -0,0 +1,13 @@ +package hub.event.scrapers.core.runlog; + +class EntityScraperRunErrorLogProperties { + static final String ID = "logId"; + static final String SCRAPER_CONFIGURATION_NAME_PATH = "scraperConfig.configurationName" ; + static final String SCRAPER_CONFIGURATION_NAME = "configurationName"; + static final String TIME = "time"; + static final String DESCRIPTION = "description"; + static final String ERROR_CODE = "errorCode"; + + private EntityScraperRunErrorLogProperties() { + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java new file mode 100644 index 0000000..ed7b472 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java @@ -0,0 +1,111 @@ +package hub.event.scrapers.core.runlog; + +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; + +import javax.persistence.*; +import java.io.Serializable; +import java.time.Instant; +import java.util.Objects; + +@Entity(name = "queryScraperStatusLog") +@Table(name = "scraper_status_log") +class EntityScraperRunStatusLog implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Integer logId; + @Column(nullable = false) + private Instant startTime; + @Column(nullable = false) + private Instant finishTime; + @Column(nullable = false) + private Integer scannedEventCount; + + @Column(nullable = false) + private Integer errorCount; + + @ManyToOne + @JoinColumn(name = "scraperId", nullable = false, insertable = false, updatable = false) + @Fetch(FetchMode.JOIN) + private EntityScraperConfig scraperConfig; + + EntityScraperRunStatusLog() { + } + + Integer getLogId() { + return logId; + } + + void setLogId(Integer logId) { + this.logId = logId; + } + + Instant getStartTime() { + return startTime; + } + + void setStartTime(Instant startTime) { + this.startTime = startTime; + } + + Instant getFinishTime() { + return finishTime; + } + + void setFinishTime(Instant finishTime) { + this.finishTime = finishTime; + } + + Integer getScannedEventCount() { + return scannedEventCount; + } + + void setScannedEventCount(Integer scannedEventCount) { + this.scannedEventCount = scannedEventCount; + } + + Integer getErrorCount() { + return errorCount; + } + + void setErrorCount(Integer errorCount) { + this.errorCount = errorCount; + } + + void setScraperConfig(EntityScraperConfig scraperConfig) { + this.scraperConfig = scraperConfig; + } + + EntityScraperConfig getScraperConfig() { + return scraperConfig; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EntityScraperRunStatusLog that = (EntityScraperRunStatusLog) o; + return Objects.equals(logId, that.logId) && Objects.equals(startTime, that.startTime) && Objects.equals(finishTime, that.finishTime) && Objects.equals(scannedEventCount, that.scannedEventCount) && Objects.equals(errorCount, that.errorCount) && Objects.equals(scraperConfig, that.scraperConfig); + } + + @Override + public int hashCode() { + return Objects.hash(logId, startTime, finishTime, scannedEventCount, errorCount, scraperConfig); + } + + @Override + public String toString() { + return "EntityScraperRunStatusLog{" + + "logId=" + logId + + ", startTime=" + startTime + + ", finishTime=" + finishTime + + ", scannedEventCount=" + scannedEventCount + + ", errorCount=" + errorCount + + ", scraperConfig=" + scraperConfig + + '}'; + } + + String getScraperConfigurationName() { + return scraperConfig.getConfigurationName(); + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLogProperties.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLogProperties.java new file mode 100644 index 0000000..29a9e71 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLogProperties.java @@ -0,0 +1,14 @@ +package hub.event.scrapers.core.runlog; + +class EntityScraperRunStatusLogProperties { + static final String ID = "logId"; + static final String SCRAPER_CONFIGURATION_NAME_PATH = "scraperConfig.configurationName" ; + static final String SCRAPER_CONFIGURATION_NAME = "configurationName"; + static final String START_TIME = "startTime"; + static final String FINISH_TIME = "finishTime"; + static final String SCANNED_EVENT_COUNT = "scannedEventCount"; + static final String ERROR_COUNT = "errorCount"; + + private EntityScraperRunStatusLogProperties() { + } +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunErrorQueryRepository.java b/src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunErrorQueryRepository.java new file mode 100644 index 0000000..0cad8a0 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunErrorQueryRepository.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core.runlog; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; + +interface JpaScraperRunErrorQueryRepository extends JpaRepository, JpaSpecificationExecutor { +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunLogQueryRepository.java b/src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunLogQueryRepository.java new file mode 100644 index 0000000..a5c4be1 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/JpaScraperRunLogQueryRepository.java @@ -0,0 +1,7 @@ +package hub.event.scrapers.core.runlog; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; + +interface JpaScraperRunLogQueryRepository extends JpaRepository, JpaSpecificationExecutor { +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java index d0e0912..44e9500 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacade.java @@ -1,16 +1,15 @@ package hub.event.scrapers.core.runlog; -import hub.event.scrapers.core.ScraperLogRepository; import org.springframework.stereotype.Service; import java.util.List; @Service public class ScraperLogQueryFacade { - private final ScraperLogRepository scraperLogRepository; + private final ScraperLogQueryRepository scraperLogRepository; - public ScraperLogQueryFacade(ScraperLogRepository scraperLogRepository) { - this.scraperLogRepository = scraperLogRepository; + public ScraperLogQueryFacade(ScraperLogQueryRepository scraperLogQueryRepository) { + this.scraperLogRepository = scraperLogQueryRepository; } public List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryRepository.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryRepository.java new file mode 100644 index 0000000..5500d67 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/runlog/ScraperLogQueryRepository.java @@ -0,0 +1,100 @@ +package hub.event.scrapers.core.runlog; + +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Objects; + +@Repository +class ScraperLogQueryRepository { + private final JpaScraperRunLogQueryRepository runLogQueryRepository; + private final JpaScraperRunErrorQueryRepository runErrorQueryRepository; + + ScraperLogQueryRepository(JpaScraperRunLogQueryRepository runLogQueryRepository, JpaScraperRunErrorQueryRepository runErrorQueryRepository) { + this.runLogQueryRepository = runLogQueryRepository; + this.runErrorQueryRepository = runErrorQueryRepository; + } + + List findAllErrorLog(ErrorLogSearchQuery errorLogSearchQuery) { + final Sort sort = Sort.by( + Sort.Order.asc(EntityScraperRunErrorLogProperties.SCRAPER_CONFIGURATION_NAME_PATH), + Sort.Order.desc(EntityScraperRunErrorLogProperties.TIME), + Sort.Order.desc(EntityScraperRunErrorLogProperties.ID) + ); + + final Specification findAllSpecification = new EntityFindAllErrorLogSpecification(errorLogSearchQuery); + final Pageable pageable = extractPageSettings(errorLogSearchQuery, sort); + + return findAllErrorLog(findAllSpecification, pageable , sort) + .stream() + .map(this::mapToLog) + .toList(); + + } + + List findAllStatusLog(StatusLogSearchQuery statusLogSearchQuery) { + final Sort sort = Sort.by( + Sort.Order.asc(EntityScraperRunStatusLogProperties.SCRAPER_CONFIGURATION_NAME_PATH), + Sort.Order.desc(EntityScraperRunStatusLogProperties.START_TIME), + Sort.Order.desc(EntityScraperRunStatusLogProperties.ID) + ); + final Specification findAllSpecification = new EntityFindAllStatusLogSpecification(statusLogSearchQuery); + final Pageable pageable = extractPageSettings(statusLogSearchQuery, sort); + + return findAllStatusLog(findAllSpecification, pageable, sort) + .stream() + .map(this::mapToLog) + .toList(); + } + + private List findAllStatusLog(Specification findAllSpecification, Pageable pageable, Sort sort) { + if (Objects.isNull(pageable)) { + return runLogQueryRepository.findAll(findAllSpecification, sort); + } + return runLogQueryRepository.findAll(findAllSpecification, pageable).toList(); + } + + private List findAllErrorLog(Specification findAllSpecification, Pageable pageable, Sort sort) { + if (Objects.isNull(pageable)) { + return runErrorQueryRepository.findAll(findAllSpecification, sort); + } + return runErrorQueryRepository.findAll(findAllSpecification, pageable).toList(); + } + + private ScraperRunStatusLog mapToLog(EntityScraperRunStatusLog entityScraperRunStatusLog) { + + return new ScraperRunStatusLog( + entityScraperRunStatusLog.getScraperConfigurationName(), + entityScraperRunStatusLog.getStartTime(), + entityScraperRunStatusLog.getFinishTime(), + entityScraperRunStatusLog.getScannedEventCount(), + entityScraperRunStatusLog.getErrorCount() + ); + } + + private ScraperRunErrorLog mapToLog(EntityScraperRunErrorLog entityScraperRunErrorLog) { + + return new ScraperRunErrorLog( + entityScraperRunErrorLog.getScraperConfigurationName(), + entityScraperRunErrorLog.getTime(), + entityScraperRunErrorLog.getErrorCode(), + entityScraperRunErrorLog.getDescription()); + } + + private Pageable extractPageSettings(StatusLogSearchQuery statusLogSearchQuery, Sort sort) { + return statusLogSearchQuery.hasPageSetting() + ? PageRequest.of(statusLogSearchQuery.page(), statusLogSearchQuery.pageSize(),sort ) + : null; + } + + private Pageable extractPageSettings(ErrorLogSearchQuery errorLogSearchQuery, Sort sort) { + return errorLogSearchQuery.hasPageSetting() + ? PageRequest.of(errorLogSearchQuery.page(), errorLogSearchQuery.pageSize(), sort) + : null; + } + +} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java deleted file mode 100644 index c890fab..0000000 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunErrorLogProperties.java +++ /dev/null @@ -1,11 +0,0 @@ -package hub.event.scrapers.core.runlog; - -public class ScraperRunErrorLogProperties { - public static final String SCRAPER_ID = "scraperId"; - public static final String TIME = "time"; - public static final String DESCRIPTION = "description"; - public static final String ERROR_CODE = "errorCode"; - - private ScraperRunErrorLogProperties() { - } -} diff --git a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java b/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java deleted file mode 100644 index 0b95e22..0000000 --- a/src/main/java/hub/event/scrapers/core/runlog/ScraperRunStatusLogProperties.java +++ /dev/null @@ -1,12 +0,0 @@ -package hub.event.scrapers.core.runlog; - -public class ScraperRunStatusLogProperties { - public static final String SCRAPER_ID = "scraperId"; - public static final String START_TIME = "startTime"; - public static final String FINISH_TIME = "finishTime"; - public static final String SCANNED_EVENT_COUNT = "scannedEventCount"; - public static final String ERROR_COUNT = "errorCount"; - - private ScraperRunStatusLogProperties() { - } -} diff --git a/src/main/resources/database/change/change_2.yaml b/src/main/resources/database/change/change_2.yaml index 626165f..4320b4c 100644 --- a/src/main/resources/database/change/change_2.yaml +++ b/src/main/resources/database/change/change_2.yaml @@ -1,6 +1,6 @@ databaseChangeLog: - changeSet: - id: scraper_config_table_2022_01_17 + id: scraper_config_table_2022_10_17 author: batonikleonardo comment: Configuration for all avilible scrapers changes: @@ -37,7 +37,7 @@ databaseChangeLog: nullable: false - changeSet: - id: scraper_status_log_table_2022_01_17 + id: scraper_status_log_table_2022_10_17 author: batonikleonardo comment : Create teble for scrapers status logs changes: @@ -81,7 +81,7 @@ databaseChangeLog: referencedColumnNames: scraper_id - changeSet: - id: scraper_error_log_table_2022_01_17 + id: scraper_error_log_table_2022_10_17 author: batonikleonardo comment: Create table for scrapres error logs changes: @@ -120,7 +120,7 @@ databaseChangeLog: referencedColumnNames: scraper_id - changeSet: - id: scraper_scraped_event_maker_table_2022_01_17 + id: scraper_scraped_event_maker_table_2022_10_17 author: batonikleonardo comment: Create table for for last sraped event marker changes: @@ -165,7 +165,7 @@ databaseChangeLog: referencedColumnNames: scraper_id - changeSet: - id: scraper_status_log_index_2022_01_17 + id: scraper_status_log_index_2022_10_17 author: batonikleonardo comment : scrapers status logs scraper id index changes: @@ -173,15 +173,19 @@ databaseChangeLog: preConditions: not: indexExists: - indexName: status_log_scraper_id_idx + indexName: status_log_serach_idx tableName: scraper_status_log + indexName: status_log_scraper_id_idx columns: - column: - name: scraper_id - indexName: status_log_scraper_id_idx + name: start_time + descending: true + - column: + name: log_id + descending: true - changeSet: - id: scraper_error_log_index_2022_01_17 + id: scraper_error_log_index_2022_10_17 author: batonikleonardo comment: scrapers error logs scraper id index changes: @@ -189,9 +193,13 @@ databaseChangeLog: preConditions: not: indexExists: - indexName: error_log_scraper_id_idx + indexName: error_log_serach_idx tableName: scraper_error_log + indexName: error_log_scraper_id_idx columns: - column: - name: scraper_id - indexName: error_log_scraper_id_idx + name: error_time + descending: true + - column: + name: log_id + descending: true \ No newline at end of file diff --git a/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java b/src/test/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacadeTest.java similarity index 81% rename from src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java rename to src/test/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacadeTest.java index 7318139..2c964b5 100644 --- a/src/test/java/hub/event/scrapers/core/ScraperLogQueryFacadeTest.java +++ b/src/test/java/hub/event/scrapers/core/runlog/ScraperLogQueryFacadeTest.java @@ -1,12 +1,9 @@ -package hub.event.scrapers.core; +package hub.event.scrapers.core.runlog; -import hub.event.scrapers.core.runlog.*; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.jdbc.Sql; @@ -17,7 +14,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.groups.Tuple.tuple; -import static org.mockito.Mockito.when; @SpringBootTest @ActiveProfiles(profiles = "dev") @@ -27,25 +23,12 @@ class ScraperLogQueryFacadeTest { @Autowired private ScraperLogQueryFacade scraperLogQueryFacade; @Autowired - private ScraperLogRepository scraperLogRepository; + private ScraperLogQueryRepository scraperLogQueryRepository; - @MockBean - private ScraperIdNameCache scraperIdNameCache; - - private final String configurationName1 = "Scraper1"; + private final String configurationName1 = "Scraper3"; private final String configurationName2 = "Scraper2"; - private final String configurationName3 = "Scraper3"; - - @BeforeEach - void mockCache() { - when(scraperIdNameCache.getIdByScraperName(configurationName1)).thenReturn(100000); - when(scraperIdNameCache.getIdByScraperName(configurationName2)).thenReturn(200000); - when(scraperIdNameCache.getIdByScraperName(configurationName3)).thenReturn(300000); + private final String configurationName3 = "Scraper1"; - when(scraperIdNameCache.getScraperNameById(100000)).thenReturn(configurationName1); - when(scraperIdNameCache.getScraperNameById(200000)).thenReturn(configurationName2); - when(scraperIdNameCache.getScraperNameById(300000)).thenReturn(configurationName3); - } @Nested class ErrorLogQueryTest { @@ -60,17 +43,17 @@ void testSearchByEmptyQuery() { ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode, scraperRunErrorLog -> scraperRunErrorLog.time().toString()) - .contains( - tuple(configurationName1, "Error 0", "ERR_0", "2022-10-19T21:15:00.015Z"), - tuple(configurationName1, "Error 0", "ERR_0", "2022-10-20T21:16:00.011Z"), - tuple(configurationName1, "Error 0", "ERR_0", "2022-10-21T21:17:00.013Z"), - tuple(configurationName1, "Error 1", "ERR_1", "2022-10-23T21:18:00.022Z"), - tuple(configurationName1, "Error 1", "ERR_1", "2022-10-24T21:19:00.020Z"), - tuple(configurationName2, "Error 1", "ERR_1", "2022-10-19T21:20:00.019Z"), - tuple(configurationName2, "Error 10", "ERR_10", "2022-10-30T22:21:00.011Z"), + .containsExactly( + tuple(configurationName3, "Error 20", "ERR_20", "2022-10-19T21:23:00.020Z"), tuple(configurationName3, "Error 1", "ERR_1", "2022-10-19T21:22:00.015Z"), - tuple(configurationName3, "Error 20", "ERR_20", "2022-10-19T21:23:00.020Z") - ); + tuple(configurationName2, "Error 10", "ERR_10", "2022-10-30T22:21:00.011Z"), + tuple(configurationName2, "Error 1", "ERR_1", "2022-10-19T21:20:00.019Z"), + tuple(configurationName1, "Error 1", "ERR_1", "2022-10-24T21:19:00.020Z"), + tuple(configurationName1, "Error 1", "ERR_1", "2022-10-23T21:18:00.022Z"), + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-21T21:17:00.013Z"), + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-20T21:16:00.011Z"), + tuple(configurationName1, "Error 0", "ERR_0", "2022-10-19T21:15:00.015Z") + ); } @Test @@ -128,7 +111,7 @@ void testSearchByDateRange() { } @Test - void testSearchBycConfigurationNames() { + void testSearchByConfigurationNames() { ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() .configurationNames(List.of(configurationName2, configurationName3)) .build(); @@ -147,15 +130,18 @@ void testSearchBycConfigurationNames() { @Test void testSearchWithPagination() { ErrorLogSearchQuery errorLogSearchQuery = ErrorLogSearchQuery.builder() - .page(2, 2) + .page(1, 2) .build(); assertThat(scraperLogQueryFacade.findAllErrorLog(errorLogSearchQuery)) .hasSize(2) .extracting( - ScraperRunErrorLog::configurationName, ScraperRunErrorLog::description, ScraperRunErrorLog::errorCode) - .contains( - tuple(configurationName1, "Error 1", "ERR_1"), - tuple(configurationName2, "Error 1", "ERR_1") + ScraperRunErrorLog::configurationName, + ScraperRunErrorLog::description, + ScraperRunErrorLog::errorCode, + scraperRunErrorLog -> scraperRunErrorLog.time().toString()) + .containsExactly( + tuple(configurationName2, "Error 10", "ERR_10", "2022-10-30T22:21:00.011Z"), + tuple(configurationName2, "Error 1", "ERR_1", "2022-10-19T21:20:00.019Z") ); } @@ -199,20 +185,20 @@ void testSearchByEmptyQuery() { ScraperRunStatusLog::scannedEventCount, ScraperRunStatusLog::errorCount ) - .contains( + .containsExactly( tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), - tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), - tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), - tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), + tuple("Scraper1", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) - ); + ); } @Test @@ -232,7 +218,7 @@ void testSearchByStartDateQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) ); } @@ -253,7 +239,7 @@ void testSearchByFinishDateQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) ); } @@ -273,14 +259,36 @@ void testSearchByNamesQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + ); + } + + @Test + void testSearchByMultipleConditionQuery() { + StatusLogSearchQuery statusLogSearchQuery = StatusLogSearchQuery.builder() + .startTimeFrom(ZonedDateTime.of(LocalDateTime.of(2022, 10, 1, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .startTimeTo(ZonedDateTime.of(LocalDateTime.of(2022, 10, 5, 0, 0), ZoneId.of(ZONE_ID)).toInstant()) + .configurationNames(List.of(configurationName1, configurationName3)) + .build(); + + assertThat(scraperLogQueryFacade.findAllStatusLog(statusLogSearchQuery)) + .hasSize(1) + .extracting( + ScraperRunStatusLog::configurationName, + scraperRunStatusLog -> scraperRunStatusLog.startTime().toString(), + scraperRunStatusLog -> scraperRunStatusLog.finishTime().toString(), + ScraperRunStatusLog::scannedEventCount, + ScraperRunStatusLog::errorCount + ) + .contains( + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100) ); } @@ -301,9 +309,9 @@ void testSearchWithPaginationQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100) + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), + tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0) ); } } @@ -326,13 +334,13 @@ void testSearchByErrorCountQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) ); } @@ -352,13 +360,13 @@ void testSearchByHasErrorQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) ); } @@ -378,8 +386,8 @@ void testSearchByNotHasErrorsQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0) @@ -405,14 +413,14 @@ void testSearchByScannedEventCountQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) ); } @@ -432,16 +440,16 @@ void testSearchByHasScannedEventQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 30, 0), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 2, 0), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 3, null), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 310, 0), - tuple("Scraper3", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), - tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) + tuple("Scraper1", "2022-10-04T18:43:44.735Z", "2022-10-04T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-10-05T18:43:44.735Z", "2022-10-05T18:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-01T19:43:44.735Z", "2022-11-01T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-02T19:43:44.735Z", "2022-11-02T19:43:58.794Z", 230, 100), + tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 230, 100) ); } @@ -461,7 +469,7 @@ void testSearchByNotHasScannedEventQuery() { ScraperRunStatusLog::errorCount ) .contains( - tuple("Scraper1", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), + tuple("Scraper3", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 0), tuple("Scraper2", "2022-11-03T19:43:44.735Z", "2022-11-03T19:43:58.794Z", 0, 80) ); } diff --git a/src/test/resources/database/scrapers/core/test_data_init.sql b/src/test/resources/database/scrapers/core/test_data_init.sql index 4b3aafe..964d2d1 100644 --- a/src/test/resources/database/scrapers/core/test_data_init.sql +++ b/src/test/resources/database/scrapers/core/test_data_init.sql @@ -2,9 +2,9 @@ DELETE FROM scraper_error_log; DELETE FROM scraper_status_log; DELETE FROM scraper_config; -INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(100000, 'Scraper1', true, 'Europe/Warsaw'); +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(100000, 'Scraper3', true, 'Europe/Warsaw'); INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(200000, 'Scraper2', true, 'Europe/Warsaw'); -INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(300000, 'Scraper3', true, 'Europe/Warsaw'); +INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(300000, 'Scraper1', true, 'Europe/Warsaw'); INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(400000, 'Scraper4', true, 'Europe/Warsaw'); INSERT INTO scraper_config (scraper_id, configuration_name, is_active, time_zone) VALUES(500000, 'Scraper5', true, 'Europe/Warsaw'); From 16fcba82c757055c5745113a0062f1d3b013f57e Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Thu, 10 Nov 2022 21:39:24 +0100 Subject: [PATCH 09/10] feat: 17 EventFacadeAdapter integration with event module by facade --- .../scrapers/core/EventFacadeAdapter.java | 65 ++++++++- .../MultipleDateMappedToEvent.java | 24 ++++ .../MultipleEventDateWithLocations.java | 2 +- .../MultipleMappedEvents.java | 22 +++ .../SingleDateMappedToEvent.java | 33 +++++ .../scrapers/core/EventFacadeAdapterTest.java | 128 ++++++++++++++++++ 6 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/MultipleDateMappedToEvent.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/MultipleMappedEvents.java create mode 100644 src/main/java/hub/event/scrapers/core/datewithlocation/SingleDateMappedToEvent.java create mode 100644 src/test/java/hub/event/scrapers/core/EventFacadeAdapterTest.java diff --git a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java index 331d8d0..826a417 100644 --- a/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java +++ b/src/main/java/hub/event/scrapers/core/EventFacadeAdapter.java @@ -1,12 +1,71 @@ package hub.event.scrapers.core; -import org.springframework.stereotype.Repository; +import hub.event.events.EventFacade; +import hub.event.events.event.Event; +import hub.event.events.type.Type; +import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; +import hub.event.scrapers.core.datewithlocation.MultipleMappedEvents; +import hub.event.scrapers.core.datewithlocation.SingleDateMappedToEvent; +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import org.springframework.stereotype.Service; +import java.util.Collection; import java.util.List; -@Repository +@Service class EventFacadeAdapter { + private final EventFacade eventFacade; + + EventFacadeAdapter(EventFacade eventFacade) { + this.eventFacade = eventFacade; + } + public void saveAll(List scrapedEventList) { - //TODO integration with EventFacade from event module - mapping to event module API dto + List eventList = mapScrapedEventListToEventList(scrapedEventList); + eventFacade.saveEvents(eventList); + } + + private List mapScrapedEventListToEventList(List scrapedEventList) { + return scrapedEventList.stream() + .map(this::mapEvent) + .flatMap(Collection::stream) + .toList(); + } + + private List mapEvent(ScrapedEvent scrapedEvent) { + return scrapedEvent.hasMultipleDateAndLocations() + ? mapMultipleDateToEvent(scrapedEvent) + : mapSingleDateToEvent(scrapedEvent); + } + + private List mapMultipleDateToEvent(ScrapedEvent scrapedEvent) { + final MultipleEventDateWithLocations multipleEventDateWithLocations = scrapedEvent.multipleEventDateWithLocations(); + + List eventList = new MultipleMappedEvents(multipleEventDateWithLocations).events(); + + for (Event event : eventList) { + event.setTitle(scrapedEvent.title()); + event.setDescription(scrapedEvent.description()); + event.setTypes(mapTypes(scrapedEvent.types())); + } + + return eventList; + } + + private List mapSingleDateToEvent(ScrapedEvent scrapedEvent) { + final SingleEventDateWithLocation eventDateWithLocation = scrapedEvent.singleEventDateWithLocation(); + final Event event = new SingleDateMappedToEvent(eventDateWithLocation); + + event.setTitle(scrapedEvent.title()); + event.setDescription(scrapedEvent.description()); + event.setTypes(mapTypes(scrapedEvent.types())); + + return List.of(event); + } + + private List mapTypes(List types) { + return types.stream() + .map(type -> new Type(null, type)) + .toList(); } } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleDateMappedToEvent.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleDateMappedToEvent.java new file mode 100644 index 0000000..f79567c --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleDateMappedToEvent.java @@ -0,0 +1,24 @@ +package hub.event.scrapers.core.datewithlocation; + +import hub.event.events.city.City; +import hub.event.events.event.Event; +import hub.event.events.place.Place; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; + +public class MultipleDateMappedToEvent extends Event { + public MultipleDateMappedToEvent(EventDateWithLocation dateWithLocation) { + super(); + + final LocalDateTime startDateTime = LocalDateTime.of(dateWithLocation.startDate(), dateWithLocation.startTime()); + final ZonedDateTime zonedStartDateTime = ZonedDateTime.of(startDateTime, dateWithLocation.timeZone()); + + final City city = new City(null, dateWithLocation.city()); + final Place place = new Place(null, dateWithLocation.address(), null, null); + + setCity(city); + setPlace(place); + setStartDate(zonedStartDateTime); + } +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java index 9ee6f0e..98b2956 100644 --- a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleEventDateWithLocations.java @@ -31,7 +31,7 @@ public MultipleEventDateWithLocations add(LocalDate date, LocalTime time, ZoneId return this; } - Collection eventDateWithLocations() { + public Collection eventDateWithLocations() { return this.eventDateWithLocations; } diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleMappedEvents.java b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleMappedEvents.java new file mode 100644 index 0000000..5e1bbdb --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/MultipleMappedEvents.java @@ -0,0 +1,22 @@ +package hub.event.scrapers.core.datewithlocation; + +import hub.event.events.event.Event; + +import java.util.List; +import java.util.stream.Collectors; + +public class MultipleMappedEvents { + private MultipleEventDateWithLocations multipleEventDateWithLocations; + + public MultipleMappedEvents(MultipleEventDateWithLocations multipleEventDateWithLocations) { + + this.multipleEventDateWithLocations = multipleEventDateWithLocations; + } + + public List events() { + return multipleEventDateWithLocations.eventDateWithLocations() + .stream() + .map(MultipleDateMappedToEvent::new) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/hub/event/scrapers/core/datewithlocation/SingleDateMappedToEvent.java b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleDateMappedToEvent.java new file mode 100644 index 0000000..52fc6d1 --- /dev/null +++ b/src/main/java/hub/event/scrapers/core/datewithlocation/SingleDateMappedToEvent.java @@ -0,0 +1,33 @@ +package hub.event.scrapers.core.datewithlocation; + +import hub.event.events.city.City; +import hub.event.events.event.Event; +import hub.event.events.place.Place; + +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.Objects; +import java.util.Optional; + +public class SingleDateMappedToEvent extends Event { + + public SingleDateMappedToEvent(SingleEventDateWithLocation dateWithLocation) { + final LocalDateTime startDateTime = LocalDateTime.of(dateWithLocation.startDate(), dateWithLocation.startTime()); + final ZonedDateTime zonedStartDateTime = ZonedDateTime.of(startDateTime, dateWithLocation.timeZone()); + + + final City city = new City(null, dateWithLocation.city()); + final Place place = new Place(null, dateWithLocation.address(), null, null); + + setCity(city); + setPlace(place); + setStartDate(zonedStartDateTime); + + if (Objects.nonNull(dateWithLocation.endDate())) { + final LocalDateTime endDateTime = LocalDateTime.of(dateWithLocation.endDate(), Optional.ofNullable(dateWithLocation.endTime()).orElse(LocalTime.of(0, 0))); + final ZonedDateTime endStartDateTime = ZonedDateTime.of(endDateTime, dateWithLocation.timeZone()); + setEndDate(endStartDateTime); + } + } +} diff --git a/src/test/java/hub/event/scrapers/core/EventFacadeAdapterTest.java b/src/test/java/hub/event/scrapers/core/EventFacadeAdapterTest.java new file mode 100644 index 0000000..c5f22a2 --- /dev/null +++ b/src/test/java/hub/event/scrapers/core/EventFacadeAdapterTest.java @@ -0,0 +1,128 @@ +package hub.event.scrapers.core; + +import hub.event.events.EventFacade; +import hub.event.events.event.Event; +import hub.event.events.type.Type; +import hub.event.scrapers.core.datewithlocation.MultipleEventDateWithLocations; +import hub.event.scrapers.core.datewithlocation.SingleEventDateWithLocation; +import hub.event.scrapers.core.exceptions.EventDateEndDateTimeBeforeStartDateTimeException; +import hub.event.scrapers.core.exceptions.EventDateInPastException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.time.*; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class EventFacadeAdapterTest { + + @Mock + private EventFacade eventFacade; + @InjectMocks + private EventFacadeAdapter eventFacadeAdapter; + + @Captor + ArgumentCaptor> eventListCaptor; + + @Test + void saveAllTest() throws EventDateInPastException, EventDateEndDateTimeBeforeStartDateTimeException { + //given + final LocalDate date1 = LocalDate.now().plusDays(2); + final LocalTime time1 = LocalTime.of(10, 20); + final String city1 = "Thessia"; + final LocalDate date2 = LocalDate.now().plusDays(4); + final LocalTime time2 = LocalTime.of(13, 0); + final String city2 = "Eden Prime"; + final LocalDate date3 = LocalDate.now().plusDays(2).plusDays(8); + final LocalTime time3 = LocalTime.of(18, 30); + final String city3 = "Rannoch"; + final String address = "Nightmare Street 102/34"; + final String locationName = "Black hole mirror club"; + final ZoneId timeZone = ZoneId.systemDefault(); + + final LocalDate startDate = LocalDate.now().plusDays(2); + final LocalDate endDate = LocalDate.now().plusDays(2); + final LocalTime startTime = LocalTime.of(10, 20); + final LocalTime endTime = LocalTime.of(20, 40); + final String city = "Thessia"; + final String address1 = "Nightmare Street 102/34"; + final String locationName1 = "Black hole mirror club"; + final ZoneId timeZone1 = ZoneId.systemDefault(); + + final MultipleEventDateWithLocations multipleDate = MultipleEventDateWithLocations.create(date1, time1, timeZone, city1, address, locationName) + .add(date2, time2, timeZone, city2, address, locationName) + .add(date3, time3, timeZone, city3, address, locationName); + final MultipleEventDateWithLocations multipleDate2 = MultipleEventDateWithLocations.create(date2, time3, timeZone, city1, address, locationName); + final SingleEventDateWithLocation singleDate1 = SingleEventDateWithLocation.single(startDate, startTime, timeZone1, city, address1, locationName1); + final SingleEventDateWithLocation singleDate2 = SingleEventDateWithLocation.single(startDate, startTime, endDate, endTime, timeZone1, city, address1, locationName1); + + final ScrapedEvent scrapedEvent1 = ScrapedEvent.builder(multipleDate) + .title("title1") + .description("description1") + .type("type1") + .type("type2") + .type("type3") + .sourceLink("http://eventhub.com/event1") + .build(); + + final ScrapedEvent scrapedEvent2 = ScrapedEvent.builder(multipleDate2) + .title("title2") + .description("description2") + .type("type56") + .sourceLink("http://eventhub.com/event2") + .build(); + + final ScrapedEvent scrapedEvent3 = ScrapedEvent.builder(singleDate1) + .title("title3") + .description("description3") + .type("type56") + .sourceLink("http://eventhub.com/event3") + .build(); + + final ScrapedEvent scrapedEvent4 = ScrapedEvent.builder(singleDate2) + .title("title4") + .description("description4") + .type("type564") + .sourceLink("http://eventhub.com/event4") + .build(); + + final List scrapedEventList = List.of(scrapedEvent1, scrapedEvent2, scrapedEvent3, scrapedEvent4); + //then + eventFacadeAdapter.saveAll(scrapedEventList); + + //verify + verify(eventFacade).saveEvents(eventListCaptor.capture()); + + Iterable facadeInputEventIterableList = eventListCaptor.getValue(); + + assertThat(facadeInputEventIterableList) + .extracting( + Event::getTitle, + Event::getDescription, + event -> event.getCity().getName(), + event -> event.getPlace().getName(), + Event::getStartDate, + Event::getEndDate, + event -> event.getTypes().stream().map(Type::getType).sorted().collect(Collectors.joining(",")) + ) + .contains( + tuple("title1", "description1", city1, address1, ZonedDateTime.of(LocalDateTime.of(date1, time1), timeZone), null, "type1,type2,type3"), + tuple("title1", "description1", city2, address, ZonedDateTime.of(LocalDateTime.of(date2, time2), timeZone), null, "type1,type2,type3"), + tuple("title1", "description1", city3, address, ZonedDateTime.of(LocalDateTime.of(date3, time3), timeZone), null, "type1,type2,type3"), + tuple("title2", "description2", city1, address, ZonedDateTime.of(LocalDateTime.of(date2, time3), timeZone), null, "type56"), + tuple("title3", "description3", city1, address, ZonedDateTime.of(LocalDateTime.of(startDate, startTime), timeZone), null, "type56"), + tuple("title4", "description4", city1, address, ZonedDateTime.of(LocalDateTime.of(startDate, startTime), timeZone), ZonedDateTime.of(LocalDateTime.of(endDate, endTime), timeZone), "type564") + ); + + } +} \ No newline at end of file From ea6536be6322f2767eb7f38180a9361088f17e0d Mon Sep 17 00:00:00 2001 From: batonikleonardo Date: Fri, 11 Nov 2022 22:16:10 +0100 Subject: [PATCH 10/10] feat: 17 fixed some bugs from tests --- .../scrapers/core/runlog/EntityScraperRunErrorLog.java | 4 ---- .../scrapers/core/runlog/EntityScraperRunStatusLog.java | 4 ---- .../java/hub/event/scrapers/proanima/ProanimaScraper.java | 7 ++----- src/test/java/hub/event/ScraperApplicationTests.java | 2 ++ src/test/java/hub/event/arch/ArchTest.java | 3 ++- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java index 5582875..885a202 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunErrorLog.java @@ -1,8 +1,5 @@ package hub.event.scrapers.core.runlog; -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; - import javax.persistence.*; import java.io.Serializable; import java.time.Instant; @@ -23,7 +20,6 @@ class EntityScraperRunErrorLog implements Serializable { private String description; @ManyToOne @JoinColumn(name = "scraperId", nullable = false, insertable = false, updatable = false) - @Fetch(FetchMode.JOIN) private EntityScraperConfig scraperConfig; EntityScraperRunErrorLog() { diff --git a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java index ed7b472..efa748c 100644 --- a/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java +++ b/src/main/java/hub/event/scrapers/core/runlog/EntityScraperRunStatusLog.java @@ -1,8 +1,5 @@ package hub.event.scrapers.core.runlog; -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; - import javax.persistence.*; import java.io.Serializable; import java.time.Instant; @@ -26,7 +23,6 @@ class EntityScraperRunStatusLog implements Serializable { @ManyToOne @JoinColumn(name = "scraperId", nullable = false, insertable = false, updatable = false) - @Fetch(FetchMode.JOIN) private EntityScraperConfig scraperConfig; EntityScraperRunStatusLog() { diff --git a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java index 34af017..4b882f2 100644 --- a/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java +++ b/src/main/java/hub/event/scrapers/proanima/ProanimaScraper.java @@ -2,18 +2,15 @@ import hub.event.scrapers.core.PageScraperPort; import hub.event.scrapers.core.ScrapedEvent; -import org.apache.commons.lang3.NotImplementedException; import org.springframework.stereotype.Component; -import java.time.ZonedDateTime; import java.util.Collection; +import java.util.Collections; @Component class ProanimaScraper extends PageScraperPort { @Override protected Collection scrap() { - logError(ZonedDateTime.now().toInstant(), "PRO_ERR_0", "Scraper not implemented yet"); - saveLastScrapedEventMarker(ZonedDateTime.now().toInstant(), null, "Absolutely nothing, i'm just drunk"); - throw new NotImplementedException(); + return Collections.emptyList(); } } diff --git a/src/test/java/hub/event/ScraperApplicationTests.java b/src/test/java/hub/event/ScraperApplicationTests.java index e95cdd3..821946b 100644 --- a/src/test/java/hub/event/ScraperApplicationTests.java +++ b/src/test/java/hub/event/ScraperApplicationTests.java @@ -2,8 +2,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; @SpringBootTest +@ActiveProfiles(profiles = "dev") class ScraperApplicationTests { @Test diff --git a/src/test/java/hub/event/arch/ArchTest.java b/src/test/java/hub/event/arch/ArchTest.java index ccc71a8..e8af861 100644 --- a/src/test/java/hub/event/arch/ArchTest.java +++ b/src/test/java/hub/event/arch/ArchTest.java @@ -39,6 +39,7 @@ class ArchTest { public static final String JAVAX = "..javax.."; public static final String HTMLUNIT = "..htmlunit.."; public static final String JSOUP = "..jsoup.."; + private static final String SLF4J = "..org.slf4j.."; @Test @@ -53,7 +54,7 @@ void givenScrapersModule_thenCheckDependencyOnlyOnEvents() { SCRAPERS_KUPBILECIK, SCRAPERS_PROANIMA) .should().onlyDependOnClassesThat() .resideInAnyPackage(JAVA, JAVAX, SPRING, - EVENTS, SCRAPERS); + EVENTS, SCRAPERS, SLF4J); archRule.check(javaClasses);