From 14dc092da7d6c845131ae3dcb0f7040aaa35be56 Mon Sep 17 00:00:00 2001 From: James Sharkey Date: Fri, 24 Apr 2026 17:19:52 +0100 Subject: [PATCH] Remove postcodes from Admin user search If postcode filtering was used with no other search terms, every single user was loaded from the database and we then attempted to filter them. This is not sustainable, and indeed did not work. We can better filter users offline if we need location-based subsets. This also allows us to remove all the apparatus for finding the lat-lon of postcodes we did not already have, too. --- .../cl/dtg/isaac/dos/ILocationHistory.java | 20 -- .../cl/dtg/isaac/dos/PgLocationHistory.java | 65 ---- .../ac/cam/cl/dtg/segue/api/AdminFacade.java | 76 +--- .../SegueGuiceConfigurationModule.java | 4 - .../cam/cl/dtg/segue/dao/LocationManager.java | 32 +- .../cam/cl/dtg/util/locations/PostCode.java | 47 --- .../locations/PostCodeIOLocationResolver.java | 327 ------------------ .../locations/PostCodeLocationResolver.java | 48 --- .../cl/dtg/util/locations/PostCodeRadius.java | 49 --- .../util/PostCodeLocationResolverTest.java | 232 ------------- 10 files changed, 2 insertions(+), 898 deletions(-) delete mode 100644 src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCode.java delete mode 100644 src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeIOLocationResolver.java delete mode 100644 src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeLocationResolver.java delete mode 100644 src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeRadius.java delete mode 100644 src/test/java/uk/ac/cam/cl/dtg/segue/util/PostCodeLocationResolverTest.java diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/ILocationHistory.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/ILocationHistory.java index 025140e830..83619b9d2b 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/ILocationHistory.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/ILocationHistory.java @@ -18,9 +18,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import uk.ac.cam.cl.dtg.segue.dao.SegueDatabaseException; import uk.ac.cam.cl.dtg.util.locations.Location; -import uk.ac.cam.cl.dtg.util.locations.PostCode; - -import java.util.List; /** * @@ -67,21 +64,4 @@ LocationHistoryEvent storeLocationEvent(final String ipAddress, final Location l * - if there is a db error. */ void updateLocationEventDate(final Long id, boolean isCurrent) throws SegueDatabaseException; - - /** - * @param postCode - * - a given postcode - * @return - a postcode object - * @throws SegueDatabaseException - * - if something goes wrong with the database. - */ - PostCode getPostCode(final String postCode) throws SegueDatabaseException; - - /** - * @param postCodes - * - a list of given postcodes - * @throws SegueDatabaseException - * - if something goes wrong with the database. - */ - void storePostCodes(List postCodes) throws SegueDatabaseException; } \ No newline at end of file diff --git a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/PgLocationHistory.java b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/PgLocationHistory.java index 28e2b4245c..951e92a57a 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/PgLocationHistory.java +++ b/src/main/java/uk/ac/cam/cl/dtg/isaac/dos/PgLocationHistory.java @@ -26,7 +26,6 @@ import uk.ac.cam.cl.dtg.segue.dao.SegueDatabaseException; import uk.ac.cam.cl.dtg.segue.database.PostgresSqlDb; import uk.ac.cam.cl.dtg.util.locations.Location; -import uk.ac.cam.cl.dtg.util.locations.PostCode; import java.io.IOException; import java.sql.Connection; @@ -35,7 +34,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Date; -import java.util.List; import java.util.Objects; /** @@ -91,33 +89,6 @@ public LocationHistoryEvent getLatestByIPAddress(final String ipAddress) throws } } - @Override - public PostCode getPostCode(final String postCode) throws SegueDatabaseException { - if (null == postCode || postCode.isEmpty()) { - return null; - } - - String query = "SELECT postcode, lat, lon FROM uk_post_codes WHERE postcode = ?"; - try (Connection conn = database.getDatabaseConnection(); - PreparedStatement pst = conn.prepareStatement(query); - ) { - pst.setString(1, postCode); - - try (ResultSet results = pst.executeQuery()) { - - while (results.next()) { - return new PostCode(results.getString("postcode"), results.getDouble("lat"), - results.getDouble("lon")); - } - - // we must not have found anything. - return null; - } - } catch (SQLException e) { - throw new SegueDatabaseException("Postgres exception", e); - } - } - /* * (non-Javadoc) * @@ -227,40 +198,4 @@ private PgLocationEvent buildPgLocationEntry(final ResultSet results) throws SQL return new PgLocationEvent(results.getLong("id"), results.getString("ip_address"), location, results.getTimestamp("created"), results.getTimestamp("last_lookup")); } - - /* - * (non-Javadoc) - * - * @see uk.ac.cam.cl.dtg.isaac.dos.LocationHistory#storePostCodes(java.util.List) - */ - @Override - public void storePostCodes(List foundPostCodes) throws SegueDatabaseException { - String query = "INSERT INTO uk_post_codes(postcode, lat, lon) VALUES (?, ?, ?)"; - try (Connection conn = database.getDatabaseConnection()) { - conn.setAutoCommit(false); - - for (PostCode postCode : foundPostCodes) { - - // Ignore post codes with invalid lat/lon - if (postCode.lat() == null || postCode.lon() == null) { - continue; - } - - try (PreparedStatement pst = conn.prepareStatement(query)) { - pst.setString(1, postCode.postCode()); - pst.setDouble(2, postCode.lat()); - pst.setDouble(3, postCode.lon()); - - if (pst.executeUpdate() == 0) { - throw new SegueDatabaseException("Unable to save location event."); - } - } - } - conn.commit(); - conn.setAutoCommit(true); - - } catch (SQLException e) { - throw new SegueDatabaseException("Postgres exception", e); - } - } } diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java index 9563df2052..f3579742ad 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/AdminFacade.java @@ -15,8 +15,6 @@ */ package uk.ac.cam.cl.dtg.segue.api; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.google.api.client.util.Maps; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; @@ -36,7 +34,6 @@ import uk.ac.cam.cl.dtg.isaac.dos.content.Content; import uk.ac.cam.cl.dtg.isaac.dos.users.EmailVerificationStatus; import uk.ac.cam.cl.dtg.isaac.dos.users.Role; -import uk.ac.cam.cl.dtg.isaac.dos.users.School; import uk.ac.cam.cl.dtg.isaac.dto.SegueErrorResponse; import uk.ac.cam.cl.dtg.isaac.dto.users.RegisteredUserDTO; import uk.ac.cam.cl.dtg.isaac.dto.users.UserIdMergeDTO; @@ -59,14 +56,10 @@ import uk.ac.cam.cl.dtg.segue.dao.content.ContentManagerException; import uk.ac.cam.cl.dtg.segue.dao.content.GitContentManager; import uk.ac.cam.cl.dtg.segue.dao.schools.SchoolListReader; -import uk.ac.cam.cl.dtg.segue.dao.schools.UnableToIndexSchoolsException; import uk.ac.cam.cl.dtg.segue.etl.GithubPushEventPayload; import uk.ac.cam.cl.dtg.segue.scheduler.SegueJobService; -import uk.ac.cam.cl.dtg.segue.search.SegueSearchException; import uk.ac.cam.cl.dtg.util.AbstractConfigLoader; import uk.ac.cam.cl.dtg.util.RequestIPExtractor; -import uk.ac.cam.cl.dtg.util.locations.LocationServerException; -import uk.ac.cam.cl.dtg.util.locations.PostCodeRadius; import jakarta.annotation.Nullable; import jakarta.servlet.http.HttpServletRequest; @@ -93,14 +86,11 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -804,8 +794,6 @@ public Response getContentProblems(@Context final HttpServletRequest request, * - if searching by role * @param schoolOther * - if searching by school other field. - * @param postcode - * - if searching by postcode. * @param schoolURN * - if searching by school by the URN. * @param emailVerificationStatus @@ -820,8 +808,6 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, @QueryParam("id") final Long userId, @QueryParam("email") @Nullable final String email, @QueryParam("familyName") @Nullable final String familyName, @QueryParam("role") @Nullable final Role role, @QueryParam("schoolOther") @Nullable final String schoolOther, - @QueryParam("postcode") @Nullable final String postcode, - @QueryParam("postcodeRadius") @Nullable final String postcodeRadius, @QueryParam("schoolURN") @Nullable final String schoolURN, @QueryParam("emailVerificationStatus") @Nullable final EmailVerificationStatus emailVerificationStatus) { @@ -839,8 +825,7 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, && (null == familyName || familyName.isEmpty()) && (null == schoolOther || schoolOther.isEmpty()) && (null == email || email.isEmpty()) - && (null == schoolURN || schoolURN.isEmpty()) - && (null == postcode || postcode.isEmpty())) { + && (null == schoolURN || schoolURN.isEmpty())) { return new SegueErrorResponse(Status.FORBIDDEN, "You do not have permission to do wildcard searches.") .toResponse(); @@ -905,65 +890,6 @@ public Response findUsers(@Context final HttpServletRequest httpServletRequest, } else { foundUsers = this.userManager.findUsers(userPrototype); } - Map userMapById = foundUsers.parallelStream().collect(Collectors.toMap(RegisteredUserDTO::getId, Function.identity())); - - // if postcode is set, filter found users - if (null != postcode) { - try { - Map> postCodeAndUserIds = Maps.newHashMap(); - for (RegisteredUserDTO userDTO : foundUsers) { - if (userDTO.getSchoolId() != null) { - School school = this.schoolReader.findSchoolById(userDTO.getSchoolId()); - if (school != null) { - String schoolPostCode = school.getPostcode(); - if (null == schoolPostCode || schoolPostCode.isEmpty()) { - continue; - } - List ids; - if (postCodeAndUserIds.containsKey(schoolPostCode)) { - ids = postCodeAndUserIds.get(schoolPostCode); - } else { - ids = Lists.newArrayList(); - } - ids.add(userDTO.getId()); - postCodeAndUserIds.put(schoolPostCode, ids); - } - } - } - - PostCodeRadius radius = PostCodeRadius.valueOf(postcodeRadius); - - List userIdsWithinRadius = locationManager.getUsersWithinPostCodeDistanceOf( - postCodeAndUserIds, postcode, radius); - - // Make sure the list returned is users who have schools in our postcode radius - List nearbyUsers = new ArrayList<>(); - for (Long id : userIdsWithinRadius) { - RegisteredUserDTO user = userMapById.get(id); //this.userManager.getUserDTOById(id); - if (user != null) { - nearbyUsers.add(user); - } - } - foundUsers = nearbyUsers; - - } catch (LocationServerException e) { - log.error("Location service unavailable. ", e); - return new SegueErrorResponse(Status.SERVICE_UNAVAILABLE, - "Unable to process request using 3rd party location provider").toResponse(); - } catch (UnableToIndexSchoolsException | SegueSearchException e) { - log.error("Unable to get school statistics", e); - return new SegueErrorResponse(Status.INTERNAL_SERVER_ERROR, - "Unable to process schools information").toResponse(); - } catch (JsonParseException | JsonMappingException e) { - log.error("Problem parsing school", e); - return new SegueErrorResponse(Status.INTERNAL_SERVER_ERROR, "Unable to read school") - .toResponse(); - } catch (IOException e) { - log.error("Problem parsing school", e); - return new SegueErrorResponse(Status.INTERNAL_SERVER_ERROR, - "IOException while trying to communicate with the school service.").toResponse(); - } - } // Calculate the ETag EntityTag etag = new EntityTag(foundUsers.size() + foundUsers.toString().hashCode() diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/configuration/SegueGuiceConfigurationModule.java b/src/main/java/uk/ac/cam/cl/dtg/segue/configuration/SegueGuiceConfigurationModule.java index 6d09c799cd..9106b3bdb1 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/configuration/SegueGuiceConfigurationModule.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/configuration/SegueGuiceConfigurationModule.java @@ -144,8 +144,6 @@ import uk.ac.cam.cl.dtg.util.email.MailJetApiClientWrapper; import uk.ac.cam.cl.dtg.util.locations.IPLocationResolver; import uk.ac.cam.cl.dtg.util.locations.MaxMindIPLocationResolver; -import uk.ac.cam.cl.dtg.util.locations.PostCodeIOLocationResolver; -import uk.ac.cam.cl.dtg.util.locations.PostCodeLocationResolver; import uk.ac.cam.cl.dtg.util.mappers.AssignmentMapper; import uk.ac.cam.cl.dtg.util.mappers.ContentMapper; import uk.ac.cam.cl.dtg.util.mappers.EventBookingMapper; @@ -422,8 +420,6 @@ private void configureAuthenticationProviders() { private void configureApplicationManagers() { bind(ILocationHistory.class).to(PgLocationHistory.class); - bind(PostCodeLocationResolver.class).to(PostCodeIOLocationResolver.class); - bind(IUserDataManager.class).to(PgUsers.class); bind(IAnonymousUserDataManager.class).to(PgAnonymousUsers.class); diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/LocationManager.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/LocationManager.java index 7b5a1973d5..51c52ed11d 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/LocationManager.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/LocationManager.java @@ -25,14 +25,10 @@ import uk.ac.cam.cl.dtg.util.locations.IPLocationResolver; import uk.ac.cam.cl.dtg.util.locations.Location; import uk.ac.cam.cl.dtg.util.locations.LocationServerException; -import uk.ac.cam.cl.dtg.util.locations.PostCodeLocationResolver; -import uk.ac.cam.cl.dtg.util.locations.PostCodeRadius; import java.io.IOException; import java.util.Calendar; import java.util.Date; -import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; /** @@ -46,7 +42,6 @@ public class LocationManager { private final ILocationHistory dao; private final IPLocationResolver ipLocationResolver; - private final PostCodeLocationResolver postCodeLocationResolver; private final Cache locationUpdatedRecentlyCache; /** @@ -54,15 +49,11 @@ public class LocationManager { * - the location history data access object. * @param ipLocationResolver * - the external ip location resolver. - * @param postCodeLocationResolver - * - the external postCode location resolver. */ @Inject - public LocationManager(final ILocationHistory dao, final IPLocationResolver ipLocationResolver, - final PostCodeLocationResolver postCodeLocationResolver) { + public LocationManager(final ILocationHistory dao, final IPLocationResolver ipLocationResolver) { this.dao = dao; this.ipLocationResolver = ipLocationResolver; - this.postCodeLocationResolver = postCodeLocationResolver; // This cache is here to prevent lots of needless look-ups to the database. locationUpdatedRecentlyCache = CacheBuilder.newBuilder().expireAfterWrite(NON_PERSISTENT_CACHE_TIME_IN_HOURS, TimeUnit.HOURS).build(); @@ -135,25 +126,4 @@ public void refreshLocation(final String ipAddress) throws SegueDatabaseExceptio this.locationUpdatedRecentlyCache.put(ipAddress, false); } } - - /** - * @param postCodeAndUserIds - * - A map of postcodes to userids - * @param targetPostCode - * - The post code we want to find users near to - * @param radius - * - radius to search - * @return - a list of userids who have schools in that radius - * @throws LocationServerException - * - anm exception when the location service fails - * @throws SegueDatabaseException - * - anm exception when the database service fails - */ - public List getUsersWithinPostCodeDistanceOf(final Map> postCodeAndUserIds, - final String targetPostCode, final PostCodeRadius radius) throws LocationServerException, - SegueDatabaseException { - return postCodeLocationResolver.filterPostcodesWithinProximityOfPostcode(postCodeAndUserIds, - targetPostCode, radius); - } - } diff --git a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCode.java b/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCode.java deleted file mode 100644 index 7806b00450..0000000000 --- a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCode.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2016 Alistair Stead - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.cam.cl.dtg.util.locations; - -/** - * A class to hold the structure of a postcode. - * - * @author Alistair Stead - * - */ -public record PostCode(String postCode, Double lat, Double lon) { - - /** - * A class to hold the structure of a postcode. - * - * @param postCode - * - the string version of the postcode - * @param lat - * - the latitude - * @param lon - * - the longitude - */ - public PostCode(final String postCode, final Double lat, final Double lon) { - // Strip whitespace to make comparison easier - if (postCode != null) { - this.postCode = postCode.replace(" ", ""); - } else { - this.postCode = null; - } - - this.lat = lat; - this.lon = lon; - } -} diff --git a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeIOLocationResolver.java b/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeIOLocationResolver.java deleted file mode 100644 index 39e8e9a170..0000000000 --- a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeIOLocationResolver.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright 2016 Alistair Stead - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.cam.cl.dtg.util.locations; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.util.Lists; -import com.google.api.client.util.Maps; -import com.google.inject.Inject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import uk.ac.cam.cl.dtg.isaac.dos.ILocationHistory; -import uk.ac.cam.cl.dtg.segue.dao.SegueDatabaseException; - -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * Class to allow postcode-related searches using external service. - * - * @author Alistair Stead - * - */ -public class PostCodeIOLocationResolver implements PostCodeLocationResolver { - private static final Logger log = LoggerFactory.getLogger(PostCodeIOLocationResolver.class); - - private final String postCodeUrl = "https://api.postcodes.io/postcodes"; // For complete postcodes - private final String outCodeUrl = "https://api.postcodes.io/outcodes"; // For partial postcodes (e.g. CB3) - private final int POSTCODEIO_MAX_REQUESTS = 100; - - private final ILocationHistory locationHistory; - private final HttpClient httpClient; - - /** - * PostCode resolver that uses queries postcodes from the local database and external postcodes.io database. - * - * @param locationHistory - * - the location history so we can access the database of existing post codes - */ - @Inject - public PostCodeIOLocationResolver(final ILocationHistory locationHistory) { - this.locationHistory = locationHistory; - this.httpClient = HttpClient.newHttpClient(); - } - - /* - * (non-Javadoc) - * - * @see uk.ac.cam.cl.dtg.util.locations.PostCodeLocationResolver#filterPostcodesWithinProximityOfPostcode( - * java.util.HashMap, java.lang.String, int) - */ - @Override - public List filterPostcodesWithinProximityOfPostcode(final Map> postCodeIDMap, - final String targetPostCode, final PostCodeRadius postCodeRadius) - throws LocationServerException, - SegueDatabaseException { - - if (null == postCodeIDMap) { - throw new LocationServerException("Map of postcodes cannot be null"); - } - - final Map> cleanPostCodeIDMap = Maps.newHashMap(); - for (String key : postCodeIDMap.keySet()) { - List val = postCodeIDMap.get(key); - if (key != null) { - cleanPostCodeIDMap.put(key.replace(" ", ""), val); - } - } - - LinkedList resultingUserIds = new LinkedList<>(); - - // first do a database lookup, then fallback on the service - List knownPostCodes = Lists.newArrayList(); - List unknownPostCodes = Lists.newArrayList(); - for (String postCode : cleanPostCodeIDMap.keySet()) { - PostCode result = this.locationHistory.getPostCode(postCode); - if (null == result) { - unknownPostCodes.add(postCode); - } else { - knownPostCodes.add(result); - } - } - - // add the target postcode, so we can do it in one request - PostCode targetPostCodeObject = this.locationHistory.getPostCode(targetPostCode); - - if (null == targetPostCodeObject) { - List targetPostCodeList = Lists.newArrayList(); - targetPostCodeList.add(targetPostCode); - List results = submitPostCodeRequest(targetPostCodeList); - if (results.size() == 1) { - targetPostCodeObject = results.getFirst(); - } else { - throw new LocationServerException( - "Location service failed to return valid lat/lon for target postcode"); - } - } - - List foundPostCodes = carryOutExternalPostCodeServiceRequest(unknownPostCodes); - - // Store new postcodes back to the database - this.locationHistory.storePostCodes(foundPostCodes); - - knownPostCodes.addAll(foundPostCodes); - - for (PostCode postCode : knownPostCodes) { - - if (null == postCode.lat() || null == postCode.lon()) { - continue; - } - - double distInMiles = getLatLonDistanceInMiles(targetPostCodeObject.lat(), - targetPostCodeObject.lon(), postCode.lat(), - postCode.lon()); - - if (distInMiles <= postCodeRadius.getDistance() - && cleanPostCodeIDMap.containsKey(postCode.postCode())) { - // Add this to a list, with user ids - resultingUserIds.addAll(cleanPostCodeIDMap.get(postCode.postCode())); - } - - } - - return resultingUserIds; - } - - /** - * Method to ensure that only 100 (the max) post codes are queried using the external service at once. - * - * @param unknownPostCodes - * - a list of post codes not exceeding 100 in length - * @return - a list of post code objects - * @throws LocationServerException - * - if there was an issue with the service - */ - private List carryOutExternalPostCodeServiceRequest(final List unknownPostCodes) - throws LocationServerException { - - log.info("Carrying out external postcode service request with {} unknown postcodes", unknownPostCodes.size()); - - if (unknownPostCodes.size() > 100) { - List completeResults = Lists.newArrayList(); - for (int i = 0; i < unknownPostCodes.size(); i += 100) { - List subList = unknownPostCodes.subList(i, Math.min(i + 100, unknownPostCodes.size())); - List results = submitPostCodeRequest(subList); - completeResults.addAll(results); - } - return completeResults; - } else { - return submitPostCodeRequest(unknownPostCodes); - } - - } - - /** - * @param unknownPostCodes - * - a list of postcodes not exceeding maxRequests in length - * @return - the results - * @throws LocationServerException - * - if there was an issue with the service - */ - @SuppressWarnings("unchecked") - private List submitPostCodeRequest(final List unknownPostCodes) - throws LocationServerException { - - List outCodes = Lists.newArrayList(); - List completePostCodes = Lists.newArrayList(); - - // Just filter by length, the regex for this would be too complex - for (String postCode : unknownPostCodes) { - if (postCode.length() > 4) { - completePostCodes.add(postCode); - } - else { - outCodes.add(postCode); - } - } - - if (completePostCodes.size() + outCodes.size() > POSTCODEIO_MAX_REQUESTS) { - throw new IllegalArgumentException(String.format("Number of postcodes cannot be bigger than %d!", - POSTCODEIO_MAX_REQUESTS)); - } - - StringBuilder sb = new StringBuilder(); - sb.append("{ \"postcodes\" : ["); - for (int i = 0; i < completePostCodes.size(); i++) { - sb.append("\""); - sb.append(completePostCodes.get(i)); - sb.append("\""); - if (i < completePostCodes.size() - 1) { - sb.append(", "); - } - } - sb.append("] }"); - - String requestJson = sb.toString(); - - HashMap postCodeResponse; - HashMap outCodeResponse = new HashMap<>(); - - try { - HttpRequest httpRequest; - java.net.http.HttpResponse httpResponse; - ObjectMapper objectMapper = new ObjectMapper(); - - // Complete postcodes can be requested in bulk - httpRequest = HttpRequest.newBuilder() - .uri(URI.create(postCodeUrl)) - .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(requestJson)) - .build(); - httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - postCodeResponse = objectMapper.readValue(httpResponse.body(), HashMap.class); - - // Outcodes can only be requested one at a time - if (!outCodes.isEmpty()) { - String url; - for (String outCode : outCodes) { - url = outCodeUrl + "/" + outCode; - httpRequest = HttpRequest.newBuilder() - .uri(URI.create(url)) - .GET() - .build(); - httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - outCodeResponse.putAll(objectMapper.readValue(httpResponse.body(), HashMap.class)); - } - } - - } catch (UnsupportedEncodingException | JsonParseException | JsonMappingException e) { - String error = "Unable to parse postcode location response " + e.getMessage(); - log.error(error); - throw new LocationServerException(error); - } catch (IOException | InterruptedException e) { - String error = "Unable to read postcode location response " + e.getMessage(); - log.error(error); - throw new LocationServerException(error); - } - - List returnList = Lists.newArrayList(); - int responseCode = (int) postCodeResponse.get("status"); - if (responseCode == Response.Status.OK.getStatusCode()) { - ArrayList> responseResult = (ArrayList>) postCodeResponse - .get("result"); - responseResult.add(outCodeResponse); - - for (Map item : responseResult) { - HashMap postCodeDetails = (HashMap) item.get("result"); - - if (postCodeDetails != null) { - Double sourceLat = (Double) postCodeDetails.get("latitude"); - Double sourceLon = (Double) postCodeDetails.get("longitude"); - String postCodeStr; - if (item.get("query") != null) { - postCodeStr = (String) item.get("query"); - } - else { - postCodeStr = (String) postCodeDetails.get("outcode"); - } - PostCode postCode = new PostCode(postCodeStr, sourceLat, sourceLon); - returnList.add(postCode); - } - } - } - return returnList; - } - - /** - * @param lat1 - * - latitude 1 - * @param lon1 - * - longitude 1 - * @param lat2 - * - latitude 2 - * @param lon2 - * - longitude 2 - * @return - distance in miles - */ - private double getLatLonDistanceInMiles(final double lat1, final double lon1, final double lat2, - final double lon2) { - // borrowed from http://www.movable-type.co.uk/scripts/latlong.html - int R = 6371000; - double phi1 = Math.toRadians(lat1); - double phi2 = Math.toRadians(lat2); - double deltaPhi = Math.toRadians(lat2 - lat1); - double deltaLambda = Math.toRadians(lon2 - lon1); - - double a = Math.sin(deltaPhi / 2) * Math.sin(deltaPhi / 2) + Math.cos(phi1) * Math.cos(phi2) - * Math.sin(deltaLambda / 2) * Math.sin(deltaLambda / 2); - double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); - - double d = R * c; - - // convert from metres to miles - d = (d / 1000) * 0.621371; - - return d; - } - - - - - -} diff --git a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeLocationResolver.java b/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeLocationResolver.java deleted file mode 100644 index 2294096c6f..0000000000 --- a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeLocationResolver.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2016 Alistair Stead - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.cam.cl.dtg.util.locations; - -import java.util.List; -import java.util.Map; - -import uk.ac.cam.cl.dtg.segue.dao.SegueDatabaseException; - -/** - * Interface to allow postcode-related searches using external service. - * - * @author Alistair Stead - * - */ -public interface PostCodeLocationResolver { - - /** - * @param postCodeAndUserIds - * - a map of postcodes to userids - * @param targetPostCode - * - the target post code - * @param distanceInMiles - * - the distance away from the target postcode to filter by - * @return - a list of userids within the specified distance from the target postcode - * @throws LocationServerException - * - an exception when there's an issue with the location service - * @throws SegueDatabaseException - * - something went wrong in the database - */ - List filterPostcodesWithinProximityOfPostcode(final Map> postCodeAndUserIds, - final String targetPostCode, final PostCodeRadius distanceInMiles) - throws LocationServerException, SegueDatabaseException; - -} diff --git a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeRadius.java b/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeRadius.java deleted file mode 100644 index c48e8bd66f..0000000000 --- a/src/main/java/uk/ac/cam/cl/dtg/util/locations/PostCodeRadius.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2016 Alistair Stead - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.cam.cl.dtg.util.locations; - -/** - * Enum to store allowable post code radius search distance. - * - * @author Alistair Stead - * - */ -public enum PostCodeRadius { - FIVE_MILES, TEN_MILES, FIFTEEN_MILES, TWENTY_MILES, TWENTY_FIVE_MILES, FIFTY_MILES; - - /** - * @return distance in miles of radius search - */ - public double getDistance() { - switch (this) { - case FIVE_MILES: - return 5.0; - case TEN_MILES: - return 10.0; - case FIFTEEN_MILES: - return 15.0; - case TWENTY_MILES: - return 20.0; - case TWENTY_FIVE_MILES: - return 25.0; - case FIFTY_MILES: - return 50.0; - default: - return 50.0; - } - } - -} diff --git a/src/test/java/uk/ac/cam/cl/dtg/segue/util/PostCodeLocationResolverTest.java b/src/test/java/uk/ac/cam/cl/dtg/segue/util/PostCodeLocationResolverTest.java deleted file mode 100644 index 30d32f9bee..0000000000 --- a/src/test/java/uk/ac/cam/cl/dtg/segue/util/PostCodeLocationResolverTest.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright 2016 Alistair Stead - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package uk.ac.cam.cl.dtg.segue.util; - -import com.google.api.client.util.Maps; -import org.easymock.EasyMock; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import uk.ac.cam.cl.dtg.isaac.dos.ILocationHistory; -import uk.ac.cam.cl.dtg.isaac.dos.PgLocationHistory; -import uk.ac.cam.cl.dtg.segue.dao.SegueDatabaseException; -import uk.ac.cam.cl.dtg.segue.database.PostgresSqlDb; -import uk.ac.cam.cl.dtg.util.locations.LocationServerException; -import uk.ac.cam.cl.dtg.util.locations.PostCodeIOLocationResolver; -import uk.ac.cam.cl.dtg.util.locations.PostCodeRadius; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * Test suite to check that the use of the 3rd party service postcodes.io - * - * @author Alistair Stead - * - */ -public class PostCodeLocationResolverTest { - - private ILocationHistory locationHistory; - private PostCodeIOLocationResolver resolver; - private PostgresSqlDb mockDatabase; - - @BeforeEach - public final void setUp() throws Exception { - mockDatabase = EasyMock.createMock(PostgresSqlDb.class); - locationHistory = new PgLocationHistory(mockDatabase); - resolver = new PostCodeIOLocationResolver(locationHistory); - - ResultSet mockResultSet = EasyMock.createMock(ResultSet.class); - EasyMock.expect(mockResultSet.next()).andReturn(false).anyTimes(); - mockResultSet.close(); - EasyMock.expectLastCall().atLeastOnce(); // It seems close is called many times? - EasyMock.replay(mockResultSet); - - PreparedStatement mockPst = EasyMock.createNiceMock(PreparedStatement.class); - EasyMock.expect(mockPst.executeQuery()).andReturn(mockResultSet).anyTimes(); - mockPst.setString(EasyMock.anyInt(), EasyMock.anyString()); - EasyMock.expectLastCall().anyTimes(); - mockPst.setDouble(EasyMock.anyInt(), EasyMock.anyDouble()); - EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(mockPst.executeUpdate()).andReturn(1).anyTimes(); - mockPst.close(); - EasyMock.replay(mockPst); - - Connection mockConnection = EasyMock.createNiceMock(Connection.class); - EasyMock.expect(mockConnection.prepareStatement(EasyMock.anyString())).andReturn(mockPst).anyTimes(); - EasyMock.replay(mockConnection); - EasyMock.expect(mockDatabase.getDatabaseConnection()).andReturn(mockConnection).anyTimes(); - EasyMock.replay(mockDatabase); - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingCorrectPostCodesOfKnownDistance_ListOfSizeOneReturned() { - - Map> map = Maps.newHashMap(); - - ArrayList list1 = new ArrayList(); - list1.add(5l); - - ArrayList list2 = new ArrayList(); - list2.add(6l); - - map.put("BD175TP", list1); - - map.put("CB237AN", list2); - - try { - List ids = resolver.filterPostcodesWithinProximityOfPostcode(map, "BD175TT", - PostCodeRadius.TWENTY_FIVE_MILES); - System.out.println(ids.toString()); - assertTrue(ids.contains(5l)); - } catch (LocationServerException e) { - fail(); - } catch (SegueDatabaseException e) { - fail(); - } - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingGoodPostCodesOutOfProximity_ExpectEmptyListReturned() { - - HashMap> map = Maps.newHashMap(); - - ArrayList list1 = new ArrayList(); - list1.add(1l); - - ArrayList list2 = new ArrayList(); - list2.add(2l); - - map.put("BD175TP", list1); - - map.put("IP327JY", list2); - - try { - List ids = resolver.filterPostcodesWithinProximityOfPostcode(map, "CB237AN", - PostCodeRadius.FIFTY_MILES); - System.out.println(ids.toString()); - assertEquals(1, ids.size()); - } catch (LocationServerException | SegueDatabaseException e) { - fail(); - } - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingBadPostCodes_ExpectEmptyListReturned() { - - HashMap> map = Maps.newHashMap(); - - ArrayList list1 = new ArrayList(); - list1.add(1l); - - ArrayList list2 = new ArrayList(); - list2.add(2l); - - map.put("5436643", list1); - - map.put("654653", list2); - - try { - List ids = resolver.filterPostcodesWithinProximityOfPostcode(map, "BD175TT", - PostCodeRadius.TWENTY_FIVE_MILES); - assertTrue(ids.isEmpty()); - System.out.println(ids.toString()); - } catch (LocationServerException | SegueDatabaseException e) { - fail(); - } - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingEmptyMap_ExpectEmptyListReturned() { - HashMap> map = Maps.newHashMap(); - - try { - List ids = resolver.filterPostcodesWithinProximityOfPostcode(map, "BD175TT", - PostCodeRadius.TWENTY_FIVE_MILES); - assertTrue(ids.isEmpty()); - System.out.println(ids.toString()); - } catch (LocationServerException | SegueDatabaseException e) { - fail(); - } - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingBadTargetPostCode_ExpectEmptyListReturned() { - HashMap> map = Maps.newHashMap(); - - ArrayList list1 = new ArrayList(); - list1.add(1l); - - ArrayList list2 = new ArrayList(); - list2.add(2l); - - map.put("BD175TP", list1); - - map.put("654653", list2); - - try { - resolver.filterPostcodesWithinProximityOfPostcode(map, "46346364", - PostCodeRadius.TWENTY_FIVE_MILES); - fail(); - } catch (LocationServerException | SegueDatabaseException e) { - System.out.println(e.getMessage()); - } - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingBadArguments_ExpectEmptyListReturned() { - - try { - resolver.filterPostcodesWithinProximityOfPostcode(null, "", null); - fail(); - } catch (LocationServerException | SegueDatabaseException e) { - System.out.println(e.getMessage()); - } - } - - @Test - public void filterPostcodesWithinProximityOfPostcode_passingPostCodesWithRandomSpaces_ExpectNonEmptyListReturned() { - HashMap> map = Maps.newHashMap(); - - ArrayList list1 = new ArrayList(); - list1.add(1l); - - ArrayList list2 = new ArrayList(); - list2.add(2l); - - map.put("BD17 5TP", list1); - - map.put(" CB23 6FE ", list2); - - try { - List ids = resolver.filterPostcodesWithinProximityOfPostcode(map, "CB237AN", - PostCodeRadius.TWENTY_FIVE_MILES); - assertTrue(ids.contains(2l)); - } catch (LocationServerException | SegueDatabaseException e) { - System.out.println(e.getMessage()); - fail(); - } - } - -}