diff --git a/.clabot b/.clabot index d6f6c4a1777..0342a7afa8a 100644 --- a/.clabot +++ b/.clabot @@ -83,6 +83,7 @@ "ZuitAMB", "marlowp", "sijandh35", - "mcihad" + "mcihad", + "nrjadkry" ] } \ No newline at end of file diff --git a/.devcontainer/.env b/.devcontainer/.env index 56a59c2138e..4d4e75899e3 100644 --- a/.devcontainer/.env +++ b/.devcontainer/.env @@ -1,9 +1,6 @@ COMPOSE_PROJECT_NAME=geonode DOCKER_HOST_IP= DOCKER_ENV=production -# See https://github.com/geosolutions-it/geonode-generic/issues/28 -# to see why we force API version to 1.24 -DOCKER_API_VERSION="1.24" BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 diff --git a/.env.sample b/.env.sample index 579a8a782fe..fee9293c82d 100644 --- a/.env.sample +++ b/.env.sample @@ -2,9 +2,6 @@ COMPOSE_PROJECT_NAME=geonode # See https://github.com/containers/podman/issues/13889 # DOCKER_BUILDKIT=0 DOCKER_ENV=production -# See https://github.com/geosolutions-it/geonode-generic/issues/28 -# to see why we force API version to 1.24 -DOCKER_API_VERSION="1.24" BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 @@ -47,7 +44,6 @@ SITEURL={siteurl}/ ALLOWED_HOSTS="['django', '{hostname}']" # Data Uploader -DEFAULT_BACKEND_UPLOADER=geonode.importer TIME_ENABLED=True MOSAIC_ENABLED=False @@ -156,7 +152,6 @@ OAUTH2_CLIENT_SECRET={clientsecret} # GeoNode APIs API_LOCKDOWN=False -TASTYPIE_APIKEY= # ################# # Production and @@ -169,7 +164,6 @@ SECRET_KEY='{secret_key}' STATIC_ROOT=/mnt/volumes/statics/static/ MEDIA_ROOT=/mnt/volumes/statics/uploaded/ ASSETS_ROOT=/mnt/volumes/statics/assets/ -GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False @@ -192,7 +186,6 @@ GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore MAPBOX_ACCESS_TOKEN= GOOGLE_API_KEY= - # Other Options/Contribs MODIFY_TOPICCATEGORY=True AVATAR_GRAVATAR_SSL=True @@ -205,15 +198,15 @@ RESOURCE_PUBLISHING=False ADMIN_MODERATE_UPLOADS=False # LDAP -LDAP_ENABLED=False -LDAP_SERVER_URL=ldap:// -LDAP_BIND_DN=uid=ldapinfo,cn=users,dc=ad,dc=example,dc=org -LDAP_BIND_PASSWORD= -LDAP_USER_SEARCH_DN=dc=ad,dc=example,dc=org -LDAP_USER_SEARCH_FILTERSTR=(&(uid=%(user)s)(objectClass=person)) -LDAP_GROUP_SEARCH_DN=cn=groups,dc=ad,dc=example,dc=org -LDAP_GROUP_SEARCH_FILTERSTR=(|(cn=abt1)(cn=abt2)(cn=abt3)(cn=abt4)(cn=abt5)(cn=abt6)) -LDAP_GROUP_PROFILE_MEMBER_ATTR=uniqueMember +#LDAP_ENABLED=False +#LDAP_SERVER_URL=ldap:// +#LDAP_BIND_DN=uid=ldapinfo,cn=users,dc=ad,dc=example,dc=org +#LDAP_BIND_PASSWORD= +#LDAP_USER_SEARCH_DN=dc=ad,dc=example,dc=org +#LDAP_USER_SEARCH_FILTERSTR=(&(uid=%(user)s)(objectClass=person)) +#LDAP_GROUP_SEARCH_DN=cn=groups,dc=ad,dc=example,dc=org +#LDAP_GROUP_SEARCH_FILTERSTR=(|(cn=abt1)(cn=abt2)(cn=abt3)(cn=abt4)(cn=abt5)(cn=abt6)) +#LDAP_GROUP_PROFILE_MEMBER_ATTR=uniqueMember # CELERY @@ -241,7 +234,6 @@ LDAP_GROUP_PROFILE_MEMBER_ATTR=uniqueMember HARVESTING_MONITOR_ENABLED=True HARVESTING_MONITOR_DELAY=60 - # PostgreSQL POSTGRESQL_MAX_CONNECTIONS=200 @@ -253,7 +245,4 @@ RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 -# FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration - -# enable the generation of dynamic models in GeoNode -IMPORTER_ENABLE_DYN_MODELS=False +# FORCE_READ_ONLY_MODE=False Override the read-only value saved in the configuration \ No newline at end of file diff --git a/.env_dev b/.env_dev index 8b012d82e4b..0cfa9dad6c7 100644 --- a/.env_dev +++ b/.env_dev @@ -2,9 +2,6 @@ COMPOSE_PROJECT_NAME=geonode # See https://github.com/containers/podman/issues/13889 # DOCKER_BUILDKIT=0 DOCKER_ENV=production -# See https://github.com/geosolutions-it/geonode-generic/issues/28 -# to see why we force API version to 1.24 -DOCKER_API_VERSION="1.24" BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 @@ -50,7 +47,6 @@ SITEURL=http://localhost:8000/ ALLOWED_HOSTS="['django', '*']" # Data Uploader -DEFAULT_BACKEND_UPLOADER=geonode.importer TIME_ENABLED=True MOSAIC_ENABLED=False @@ -158,7 +154,6 @@ OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZK # GeoNode APIs API_LOCKDOWN=False -TASTYPIE_APIKEY= # ################# # Production and @@ -168,7 +163,6 @@ DEBUG=True SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' - CACHE_BUSTING_STATIC_ENABLED=False MEMCACHED_ENABLED=False @@ -212,6 +206,5 @@ RESTART_POLICY_MAX_ATTEMPTS="3" RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 -IMPORTER_ENABLE_DYN_MODELS=True UPSERT_CHUNK_SIZE= 100 UPSERT_LIMIT_ERROR_LOG=100 \ No newline at end of file diff --git a/.env_local b/.env_local index b3a2fcac0e4..583a9fc32d6 100644 --- a/.env_local +++ b/.env_local @@ -2,9 +2,6 @@ COMPOSE_PROJECT_NAME=geonode # See https://github.com/containers/podman/issues/13889 # DOCKER_BUILDKIT=0 DOCKER_ENV=production -# See https://github.com/geosolutions-it/geonode-generic/issues/28 -# to see why we force API version to 1.24 -DOCKER_API_VERSION="1.24" BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 @@ -50,7 +47,6 @@ SITEURL=http://localhost:8000/ ALLOWED_HOSTS="['django', '*']" # Data Uploader -DEFAULT_BACKEND_UPLOADER=geonode.importer TIME_ENABLED=True MOSAIC_ENABLED=False @@ -158,7 +154,6 @@ OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZK # GeoNode APIs API_LOCKDOWN=False -TASTYPIE_APIKEY= # ################# # Production and @@ -170,7 +165,6 @@ SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' # STATIC_ROOT=/mnt/volumes/statics/static/ # MEDIA_ROOT=/mnt/volumes/statics/uploaded/ -# GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False @@ -215,5 +209,3 @@ RESTART_POLICY_MAX_ATTEMPTS="3" RESTART_POLICY_WINDOW=120s DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 - -IMPORTER_ENABLE_DYN_MODELS=False diff --git a/.env_test b/.env_test index 5e270acd875..a770063d228 100644 --- a/.env_test +++ b/.env_test @@ -2,9 +2,6 @@ COMPOSE_PROJECT_NAME=geonode # See https://github.com/containers/podman/issues/13889 # DOCKER_BUILDKIT=0 DOCKER_ENV=production -# See https://github.com/geosolutions-it/geonode-generic/issues/28 -# to see why we force API version to 1.24 -DOCKER_API_VERSION="1.24" BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 @@ -50,7 +47,6 @@ SITEURL=http://localhost:8000/ ALLOWED_HOSTS="['django', 'localhost', '127.0.0.1']" # Data Uploader -DEFAULT_BACKEND_UPLOADER=geonode.importer TIME_ENABLED=True MOSAIC_ENABLED=False @@ -167,7 +163,6 @@ SOCIALACCOUNT_PROVIDER=google # GeoNode APIs API_LOCKDOWN=False -TASTYPIE_APIKEY= # ################# # Production and @@ -179,7 +174,6 @@ SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' STATIC_ROOT=/mnt/volumes/statics/static/ MEDIA_ROOT=/mnt/volumes/statics/uploaded/ -GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False @@ -230,4 +224,3 @@ MICROSOFT_TENANT_ID= AZURE_CLIENT_ID= AZURE_SECRET_KEY= AZURE_KEY= -IMPORTER_ENABLE_DYN_MODELS=False diff --git a/celery-cmd b/celery-cmd index 674c7079ce5..bb2f04529e5 100644 --- a/celery-cmd +++ b/celery-cmd @@ -8,26 +8,52 @@ CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker@%h.state"} CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"} CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"10,5"} CELERY__MAX_TASKS_PER_CHILD=${CELERY__MAX_TASKS_PER_CHILD:-"10"} -CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -B -E"} -CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"celery.beat:PersistentScheduler"} +CELERY__OPTS=${CELERY__OPTS:-"--without-gossip --without-mingle -Ofair -E"} CELERY__LOG_LEVEL=${CELERY__LOG_LEVEL:-"ERROR"} CELERY__LOG_FILE=${CELERY__LOG_FILE:-"/var/log/celery.log"} CELERY__WORKER_NAME=${CELERY__WORKER_NAME:-"worker1@%h"} CELERY__WORKER_CONCURRENCY=${CELERY__WORKER_CONCURRENCY:-"4"} +# Celery beat settings +CELERY__BEAT_SCHEDULE=${CELERY__BEAT_SCHEDULE:-"celery.beat:PersistentScheduler"} +CELERY__BEAT_LOG=${CELERY__BEAT_LOG:-"/var/log/celery_beat.log"} + # Harvester settings CELERY__HARVESTER_WORKER_NAME=${CELERY__HARVESTER_WORKER_NAME:-"harvesting_worker@%h"} CELERY__HARVESTER_CONCURRENCY=${CELERY__HARVESTER_CONCURRENCY:-"10"} CELERY__HARVESTER_AUTOSCALE_VALUES=${CELERY__HARVESTER_AUTOSCALE_VALUES:-"15,10"} CELERY__HARVESTER_MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"500000"} +# --- FIX: Remove stale Beat pidfile before starting beat --- +BEAT_PIDFILE="/tmp/celerybeat.pid" + +if [ -f "$BEAT_PIDFILE" ]; then + PID=$(cat "$BEAT_PIDFILE" 2>/dev/null) + + # If PID exists and is running → warn but continue (avoid killing) + if kill -0 "$PID" 2>/dev/null; then + echo "WARNING: Celery Beat seems to be running already (PID $PID). Removing stale pidfile anyway." + else + echo "Removing stale Celery Beat pidfile: $BEAT_PIDFILE" + fi + + rm -f "$BEAT_PIDFILE" +fi +# --- END FIX --- + +echo "Starting Celery Beat..." +$CELERY_BIN -A $CELERY_APP beat --scheduler=$CELERY__BEAT_SCHEDULE \ + --loglevel=$CELERY__LOG_LEVEL -f $CELERY__BEAT_LOG --pidfile=/tmp/celerybeat.pid & + +echo "Starting Default Celery Worker..." $CELERY_BIN -A $CELERY_APP worker --autoscale=$CELERY__AUTOSCALE_VALUES \ --max-memory-per-child=$CELERY__MAX_MEMORY_PER_CHILD $CELERY__OPTS \ - --statedb=$CELERY__STATE_DB --scheduler=$CELERY__BEAT_SCHEDULE \ + --statedb=$CELERY__STATE_DB \ --loglevel=$CELERY__LOG_LEVEL -n $CELERY__WORKER_NAME -f $CELERY__LOG_FILE \ --concurrency=$CELERY__WORKER_CONCURRENCY --max-tasks-per-child=$CELERY__MAX_TASKS_PER_CHILD \ -X harvesting & +echo "Starting Harvester Celery Worker..." $CELERY_BIN -A $CELERY_APP worker -Q harvesting \ --autoscale=$CELERY__HARVESTER_AUTOSCALE_VALUES \ --max-memory-per-child=$CELERY__HARVESTER_MAX_MEMORY_PER_CHILD \ @@ -36,5 +62,9 @@ $CELERY_BIN -A $CELERY_APP worker -Q harvesting \ --concurrency=$CELERY__HARVESTER_CONCURRENCY \ -f $CELERY__LOG_FILE & -# Keep the container alive -wait \ No newline at end of file +# Wait for any process to exit +wait -n + +# Exit with the status of the process that exited first +# Docker will restart the container if this is non-zero (i.e., a failure) +exit $? \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index add55fc3364..b19042ecce1 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -79,7 +79,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.24.3-latest + image: geonode/geoserver:2.27.4-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -105,7 +105,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.24.3-latest + image: geonode/geoserver_data:2.27.4-latest container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml index 5a83f79b7c7..3fe37fb2bad 100644 --- a/docker-compose-geoserver-server.yml +++ b/docker-compose-geoserver-server.yml @@ -2,7 +2,7 @@ version: '2.2' services: data-dir-conf: - image: geonode/geoserver_data:latest + image: geonode/geoserver_data:2.27.4-latest restart: on-failure container_name: gsconf4${COMPOSE_PROJECT_NAME} labels: @@ -13,7 +13,7 @@ services: - geoserver-data-dir:/geoserver_data/data geoserver: - image: geonode/geoserver:latest + image: geonode/geoserver:2.27.4-latest restart: unless-stopped container_name: geoserver4${COMPOSE_PROJECT_NAME} stdin_open: true diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 29fa6a23b39..ed933655dfe 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -80,7 +80,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.24.4-latest + image: geonode/geoserver:2.27.4-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -106,7 +106,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.24.4-latest + image: geonode/geoserver_data:2.27.4-latest container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 8c2b1f456cd..e9d3c2c122c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,7 +92,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.27.x-latest + image: geonode/geoserver:2.27.4-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -118,7 +118,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.27.2-latest + image: geonode/geoserver_data:2.27.4-latest container_name: gsconf4${COMPOSE_PROJECT_NAME} entrypoint: sleep infinity volumes: diff --git a/entrypoint.sh b/entrypoint.sh index e149adc3d93..ed9469efac2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -27,7 +27,6 @@ invoke update source $HOME/.bashrc source $HOME/.override_env -echo DOCKER_API_VERSION=$DOCKER_API_VERSION echo POSTGRES_USER=$POSTGRES_USER echo POSTGRES_PASSWORD=$POSTGRES_PASSWORD echo DATABASE_URL=$DATABASE_URL diff --git a/geonode/base/api/filters.py b/geonode/base/api/filters.py index a2428bfdc65..e06f5fea00e 100644 --- a/geonode/base/api/filters.py +++ b/geonode/base/api/filters.py @@ -131,3 +131,33 @@ def filter_queryset(self, request, queryset, _): _filter["id__in"] = [_facet.id for _facet in queryset if not _facet.resourcebase_set.exists()] return queryset.filter(**_filter) + + +class AdvertisedFilter(BaseFilterBackend): + def filter_queryset(self, request, queryset, view): + # advertised + # if superuser, all resources will be visible, otherwise only the advertised one and + # the resource which the user is owner will be returned + + if getattr(view, "action", None) == "retrieve": + return queryset + + user = request.user + try: + _filter = request.query_params.get("advertised", "None") + advertised = strtobool(_filter) if _filter.lower() != "all" else "all" + except Exception: + advertised = None + + if advertised == "all": + pass + elif advertised is not None: + queryset = queryset.filter(advertised=advertised) + else: + is_admin = user.is_superuser if user and user.is_authenticated else False + if not is_admin and user and not user.is_anonymous: + queryset = (queryset.filter(advertised=True) | queryset.filter(owner=user)).distinct() + elif not user or user.is_anonymous: + queryset = queryset.filter(advertised=True) + + return queryset diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 1fe5db49ecd..4169f6dfeb6 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -367,7 +367,7 @@ def get_attribute(self, instance): download_urls.append({"url": obj.download_url, "ajax_safe": obj.is_ajax_safe, "default": False}) if asset: - download_urls.append({"url": asset_url, "ajax_safe": True, "default": False if download_urls else True}) + download_urls.append({"url": asset_url, "ajax_safe": True, "default": False}) return download_urls else: diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 357c9563c32..43d37b5663e 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -2885,9 +2885,7 @@ def test_base_resources_return_download_links_for_datasets(self): Ensure we can access the Resource Base list. """ _dataset = Dataset.objects.first() - expected_payload = [ - {"url": reverse("dataset_download", args=[_dataset.alternate]), "ajax_safe": True, "default": True} - ] + expected_payload = [] # From resource base API json = self._get_for_object(_dataset, "base-resources-detail") diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index a584f24decc..045a5da1b7e 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -65,6 +65,7 @@ FacetVisibleResourceFilter, FavoriteFilter, TKeywordsFilter, + AdvertisedFilter, ) from geonode.groups.models import GroupProfile, Group from geonode.security.permissions import get_compact_perms_list, PermSpec @@ -308,6 +309,7 @@ class ResourceBaseViewSet(ApiPresetsInitializer, DynamicModelViewSet, Advertised permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] filter_backends = [ TKeywordsFilter, + AdvertisedFilter, DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, diff --git a/geonode/base/models.py b/geonode/base/models.py index b51c7b2c053..15172c1bc0f 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -2085,6 +2085,15 @@ class Configuration(SingletonModel): read_only = models.BooleanField(default=False) maintenance = models.BooleanField(default=False) + def save(self, *args, **kwargs): + previous_read_only = Configuration.objects.filter(pk=self.pk).values_list("read_only", flat=True).first() + super().save(*args, **kwargs) + + if previous_read_only != self.read_only: + from geonode.security.registry import permissions_registry + + permissions_registry.clear_permissions_cache() + class Meta: verbose_name_plural = "Configuration" diff --git a/geonode/br/management/commands/settings_docker_sample.ini b/geonode/br/management/commands/settings_docker_sample.ini index f5401a9c8fa..0dd9635729f 100644 --- a/geonode/br/management/commands/settings_docker_sample.ini +++ b/geonode/br/management/commands/settings_docker_sample.ini @@ -13,6 +13,6 @@ dumprasterdata = yes # data_layername_exclude_filter = {comma separated list of layernames, optionally with glob syntax} e.g.: tuscany_*,italy [fixtures] -apps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client -dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client +apps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,metadata,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,geonode_themes,geoapps,favorite,geonode_client +dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,metadata,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,geonode_themes,geoapps,favorite,geonode_client diff --git a/geonode/br/management/commands/settings_sample.ini b/geonode/br/management/commands/settings_sample.ini index 112b6618474..0e85f961e71 100644 --- a/geonode/br/management/commands/settings_sample.ini +++ b/geonode/br/management/commands/settings_sample.ini @@ -13,5 +13,5 @@ dumprasterdata = yes # data_layername_exclude_filter = {comma separated list of layernames, optionally with glob syntax} e.g.: tuscany_*,italy [fixtures] -apps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client -dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,user_messages,geonode_themes,geoapps,favorite,geonode_client +apps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,metadata,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,geonode_themes,geoapps,favorite,geonode_client +dumps = contenttypes,auth,people,groups,account,guardian,admin,actstream,announcements,avatar,assets,base,documents,geoserver,invitations,pinax_notifications,harvesting,services,layers,maps,metadata,oauth2_provider,sites,socialaccount,taggit,tastypie,upload,geonode_themes,geoapps,favorite,geonode_client diff --git a/geonode/catalogue/backends/pycsw_local.py b/geonode/catalogue/backends/pycsw_local.py index 4bfa8819b94..b56b677e777 100644 --- a/geonode/catalogue/backends/pycsw_local.py +++ b/geonode/catalogue/backends/pycsw_local.py @@ -124,10 +124,11 @@ def _csw_local_dispatch(self, keywords=None, start=0, limit=10, bbox=None, ident HTTP-less CSW """ - mdict = dict(settings.PYCSW["CONFIGURATION"], **CONFIGURATION) - if "server" in settings.PYCSW["CONFIGURATION"]: - # override server system defaults with user specified directives - mdict["server"].update(settings.PYCSW["CONFIGURATION"]["server"]) + mdict = dict(CONFIGURATION, **settings.PYCSW["CONFIGURATION"]) + # if "server" in settings.PYCSW["CONFIGURATION"]: + # # override server system defaults with user specified directives + # # When swapping parameters from dict(), this if statement becomes useless + # mdict["server"].update(settings.PYCSW["CONFIGURATION"]["server"]) # fake HTTP environment variable os.environ["QUERY_STRING"] = "" diff --git a/geonode/catalogue/management/commands/regenerate_xml.py b/geonode/catalogue/management/commands/regenerate_xml.py index ce89abdaaa7..43f038a8d2e 100644 --- a/geonode/catalogue/management/commands/regenerate_xml.py +++ b/geonode/catalogue/management/commands/regenerate_xml.py @@ -20,10 +20,12 @@ import logging +from django.conf import settings from django.core.management.base import BaseCommand from geonode.base.management import command_utils from geonode.base.models import ResourceBase +from geonode.catalogue.models import catalogue_post_save from geonode.layers.models import Dataset @@ -39,7 +41,15 @@ def add_arguments(self, parser): '--layer', dest="layers", action='append', - help="Only process specified layers ") + help="Only process layers with specified name") + + parser.add_argument( + '-i', + '--id', + dest="ids", + type=int, + action='append', + help="Only process resources with specified id") parser.add_argument( "--skip-logger-setup", @@ -56,6 +66,7 @@ def add_arguments(self, parser): def handle(self, **options): requested_layers = options.get('layers') + requested_ids = options.get('ids') dry_run = options.get('dry-run') if options.get("setup_logger"): @@ -67,11 +78,16 @@ def handle(self, **options): logger.debug(f"DRY-RUN is {dry_run}") logger.debug(f"LAYERS is {requested_layers}") + logger.debug(f"IDS is {requested_ids}") - try: + uuid_handler_class = None + if hasattr(settings, "LAYER_UUID_HANDLER") and settings.LAYER_UUID_HANDLER: + from geonode.layers.utils import get_uuid_handler + uuid_handler_class = get_uuid_handler() - layers = Dataset.objects.all() - tot = len(layers) + try: + resources = Dataset.objects.all().order_by("id") + tot = resources.count() logger.info(f"Total layers in GeoNode: {tot}") i = 0 cnt_ok = 0 @@ -79,44 +95,48 @@ def handle(self, **options): cnt_skip = 0 instance: ResourceBase - for instance in layers: + for instance in resources: i += 1 - logger.info(f"- {i}/{tot} Processing layer {instance.id} [{instance.typename}] '{instance.title}'") + logger.info(f"- {i}/{tot} Processing resource {instance.id} [{instance.typename}] '{instance.title}'") + + include_by_rl = requested_layers and instance.typename in requested_layers + include_by_id = requested_ids and instance.id in requested_ids + accepted = (not requested_layers and not requested_ids) or include_by_id or include_by_rl - if requested_layers and instance.typename not in requested_layers: - logger.info(" - Layer filtered out by args") + if not accepted: + logger.info(" - Resource filtered out by args") cnt_skip += 1 continue if instance.metadata_uploaded and instance.metadata_uploaded_preserve: - logger.info(" - Layer filtered out since it uses custom XML") + logger.info(" - Resource filtered out since it uses custom XML") cnt_skip += 1 continue - try: - good = None - if not dry_run: - try: - try: - # the save() method triggers the metadata regeneration - instance.save() - good = True - except Exception as e: - logger.error(f"Error saving instance '{instance.title}': {e}") - raise e - - except Exception as e: - logger.exception(f"Error processing '{instance.title}': {e}", e) - - if dry_run or good: - logger.info(f" - Done {instance.name}") - cnt_ok += 1 - else: - logger.warning(f"Metadata couldn't be regenerated for instance '{instance.title}' ") - cnt_bad += 1 - - except Exception as e: - raise e + good = None + if not dry_run: + try: + # regenerate UUID + if uuid_handler_class: + _uuid = uuid_handler_class(instance).create_uuid() + if _uuid != instance.uuid: + logger.info(f"Replacing UUID: {instance.uuid} --> {_uuid}") + instance.uuid = _uuid + ResourceBase.objects.filter(id=instance.id).update(uuid=_uuid) + + # regenerate XML + catalogue_post_save(instance, None) + good = True + except Exception as e: + logger.exception(f"Error processing '{instance.title}': {e}", e) + + if dry_run or good: + logger.info(f" - Done {instance.name}") + cnt_ok += 1 + else: + logger.warning(f"Metadata couldn't be regenerated for instance '{instance.title}' ") + cnt_bad += 1 + except Exception as e: raise e diff --git a/geonode/catalogue/views.py b/geonode/catalogue/views.py index abb18b76e5b..b724e7c9cc0 100644 --- a/geonode/catalogue/views.py +++ b/geonode/catalogue/views.py @@ -46,7 +46,7 @@ def csw_global_dispatch(request, dataset_filter=None, config_updater=None): if settings.CATALOGUE["default"]["ENGINE"] != "geonode.catalogue.backends.pycsw_local": return HttpResponseRedirect(settings.CATALOGUE["default"]["URL"]) - mdict = dict(settings.PYCSW["CONFIGURATION"], **CONFIGURATION) + mdict = dict(CONFIGURATION, **settings.PYCSW["CONFIGURATION"]) mdict = config_updater(mdict) if config_updater else mdict access_token = None diff --git a/geonode/context_processors.py b/geonode/context_processors.py index 3b30e0b93dd..c600fafa42b 100644 --- a/geonode/context_processors.py +++ b/geonode/context_processors.py @@ -54,9 +54,7 @@ def resource_urls(request): SITEURL=settings.SITEURL, INSTALLED_APPS=settings.INSTALLED_APPS, THEME_ACCOUNT_CONTACT_EMAIL=settings.THEME_ACCOUNT_CONTACT_EMAIL, - TINYMCE_DEFAULT_CONFIG=settings.TINYMCE_DEFAULT_CONFIG, PROXY_URL=getattr(settings, "PROXY_URL", "/proxy/?url="), - DISPLAY_SOCIAL=getattr(settings, "DISPLAY_SOCIAL", False), DISPLAY_RATINGS=getattr(settings, "DISPLAY_RATINGS", False), DISPLAY_WMS_LINKS=getattr(settings, "DISPLAY_WMS_LINKS", True), CREATE_LAYER=getattr(settings, "CREATE_LAYER", True), @@ -82,7 +80,6 @@ def resource_urls(request): DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=getattr(settings, "DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION", False), EXIF_ENABLED=getattr(settings, "EXIF_ENABLED", False), FAVORITE_ENABLED=getattr(settings, "FAVORITE_ENABLED", False), - SEARCH_FILTERS=getattr(settings, "SEARCH_FILTERS", False), THESAURI_FILTERS=( [ t["name"] diff --git a/geonode/documents/exif/utils.py b/geonode/documents/exif/utils.py index 2c751e31714..684c9908883 100644 --- a/geonode/documents/exif/utils.py +++ b/geonode/documents/exif/utils.py @@ -69,12 +69,13 @@ def exif_extract_metadata_doc(doc): if not doc.files: return None - _, ext = os.path.splitext(os.path.basename(doc.files[0])) + file_path = doc.files[0] + _, ext = os.path.splitext(os.path.basename(file_path)) - if ext[1:] in {"jpg", "jpeg"}: + if ext[1:] in {"jpg", "jpeg", "png"}: from PIL import Image, ExifTags - img = Image.open(doc.doc_file.path) + img = Image.open(file_path) exif_data = {ExifTags.TAGS[k]: v for k, v in img._getexif().items() if k in ExifTags.TAGS} model = None diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index 32c5743a9bd..b20332a9d20 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -296,14 +296,8 @@ def test_documents_thumbnail(self): with self.settings(THUMBNAIL_SIZE={"width": 400, "height": 200}): self.client.post(reverse("document_upload"), data=data) d = Document.objects.get(title="Remote img File Doc") - self.assertIsNotNone(d.thumbnail_url) - thumb_file = os.path.join( - settings.MEDIA_ROOT, f"thumbs/{os.path.basename(urlparse(d.thumbnail_url).path)}" - ) - file = Image.open(thumb_file) - self.assertEqual(file.size, (400, 200)) - # check thumbnail qualty and extention - self.assertEqual(file.format, "JPEG") + self.assertIsNone(d.thumbnail_url, "Thumbnails are not allowed for remote documents.") + # test pdf doc with open(os.path.join(f"{self.project_root}", "tests/data/pdf_doc.pdf"), "rb") as f: data = { diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 1eacf637b94..30cda95bc5e 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -199,6 +199,7 @@ def form_valid(self, form): except Exception: logger.debug("Exif extraction failed.") + bbox_poly = BBOXHelper.from_xy(bbox).as_polygon() if bbox else None resource_manager.update( self.object.uuid, instance=self.object, @@ -208,11 +209,17 @@ def form_valid(self, form): abstract=abstract, date=date, date_type="Creation", - bbox_polygon=BBOXHelper.from_xy(bbox).as_polygon() if bbox else None, + bbox_polygon=bbox_poly, + ll_bbox_polygon=bbox_poly, ), notify=True, ) - resource_manager.set_thumbnail(self.object.uuid, instance=self.object, overwrite=False) + + # Only trigger thumbnailing for local documents, not for remote URLs + if self.object.is_local: + resource_manager.set_thumbnail(self.object.uuid, instance=self.object, overwrite=False) + else: + logger.info(f"Skipping thumbnail generation for remote document: {self.object.doc_url}") register_event(self.request, enumerations.EventType.EVENT_UPLOAD, self.object) diff --git a/geonode/facets/views.py b/geonode/facets/views.py index 974e79cf636..225e2fa9d4b 100644 --- a/geonode/facets/views.py +++ b/geonode/facets/views.py @@ -27,7 +27,6 @@ from django.conf import settings from geonode.base.api.views import ResourceBaseViewSet -from geonode.base.models import ResourceBase from geonode.facets.models import FacetProvider, DEFAULT_FACET_PAGE_SIZE, facet_registry from geonode.security.utils import get_visible_resources @@ -86,13 +85,16 @@ def _prefilter_topics(cls, request): filters = {k: vlist for k, vlist in request.query_params.lists() if k.startswith("filter{")} logger.warning(f"FILTERING BY {filters}") - if filters: - viewset = ResourceBaseViewSet(request=request, format_kwarg={}, kwargs=filters) - viewset.initial(request) - return get_visible_resources(queryset=viewset.filter_queryset(viewset.get_queryset()), user=request.user) - else: - # return ResourceBase.objects - return get_visible_resources(ResourceBase.objects, request.user) + # kwargs will be {} if no filter applied + viewset = ResourceBaseViewSet( + request=request, + format_kwarg={}, + kwargs=filters, + ) + viewset.initial(request) + + queryset = viewset.filter_queryset(viewset.get_queryset()) + return get_visible_resources(queryset=queryset, user=request.user) @classmethod def _resolve_language(cls, request) -> (str, bool): diff --git a/geonode/geoserver/createlayer/utils.py b/geonode/geoserver/createlayer/utils.py index 20eb5d13558..0f7de44e6ba 100644 --- a/geonode/geoserver/createlayer/utils.py +++ b/geonode/geoserver/createlayer/utils.py @@ -222,5 +222,4 @@ def create_gs_dataset(name, title, geometry_type, attributes=None): logger.error(f"Response was: {req.text}") raise Exception(f"Dataset could not be created in GeoServer {req.text}") - cat.reload() return workspace, datastore diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index 554a3170612..c90fa82c8f1 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -478,8 +478,6 @@ def cascading_delete(dataset_name=None, catalog=None): finally: # Let's reset the connections first cat._cache.clear() - cat.reset() - cat.reload() if resource is None: # If there is no associated resource, diff --git a/geonode/harvesting/tasks.py b/geonode/harvesting/tasks.py index 05d7c2081ce..6b55f3e47ff 100644 --- a/geonode/harvesting/tasks.py +++ b/geonode/harvesting/tasks.py @@ -31,7 +31,7 @@ from django.db.models.functions import Concat from django.utils import timezone from django.conf import settings -from django.db import transaction +from django.db import transaction, IntegrityError from geonode.resource.models import ExecutionRequest from geonode.resource.enumerator import ExecutionRequestAction @@ -741,16 +741,30 @@ def _update_harvestable_resources_batch(self, refresh_session_id: int, page: int else: processed = 0 for remote_resource in found_resources: - resource, created = models.HarvestableResource.objects.get_or_create( - harvester=harvester, - unique_identifier=remote_resource.unique_identifier, - title=remote_resource.title, - defaults={ - "should_be_harvested": harvester.harvest_new_resources_by_default, - "remote_resource_type": remote_resource.resource_type, - "last_refreshed": timezone.now(), - }, - ) + try: + resource, created = models.HarvestableResource.objects.get_or_create( + harvester=harvester, + unique_identifier=remote_resource.unique_identifier, + defaults={ + "title": remote_resource.title, + "should_be_harvested": harvester.harvest_new_resources_by_default, + "remote_resource_type": remote_resource.resource_type, + "last_refreshed": timezone.now(), + }, + ) + except IntegrityError: + # RACE CONDITION: Another worker created this between our SELECT and INSERT. + # We catch the error and simply fetch the one they created. + resource = models.HarvestableResource.objects.get( + harvester=harvester, unique_identifier=remote_resource.unique_identifier + ) + created = False + # If the resource wasn't just created, check if the title changed + # (e.g. from 'copy title' to '28409') and update it. + if not created: + resource.title = remote_resource.title + resource.remote_resource_type = remote_resource.resource_type + processed += 1 # NOTE: make sure to save the resource because we need to have its # `last_updated` property be refreshed - this is done in order to be able diff --git a/geonode/harvesting/tests/test_tasks.py b/geonode/harvesting/tests/test_tasks.py index ffc6f8b13a5..926f161ba26 100644 --- a/geonode/harvesting/tests/test_tasks.py +++ b/geonode/harvesting/tests/test_tasks.py @@ -251,6 +251,45 @@ def test_update_harvestable_resources_sends_batched_requests( # Assert chord was called with expiration mock_chord.return_value.apply_async.assert_called_once_with(args=(), expires=123) + @mock.patch("geonode.harvesting.models.Harvester.get_harvester_worker") + def test_update_batch_corrects_title_mismatch(self, mock_get_worker): + """ + Verify that if a remote resource title changes, the existing + HarvestableResource is updated instead of causing an IntegrityError. + """ + # Setup the session to be "ON_GOING" so the task doesn't skip it + self.harvesting_session.status = models.AsynchronousHarvestingSession.STATUS_ON_GOING + self.harvesting_session.save() + + # Pick one of the resources created in setUpTestData + # Let's say we target 'fake-identifier-0' which currently has 'fake-title-0' + target_uid = "fake-identifier-0" + new_remote_title = "COMPLETELY_NEW_TITLE_FROM_WMS" + + # Mock the worker to return the resource with the new title + mock_remote_resource = mock.MagicMock() + mock_remote_resource.unique_identifier = target_uid + mock_remote_resource.title = new_remote_title + mock_remote_resource.resource_type = "fake-remote-resource-type" + + mock_worker = mock.MagicMock() + mock_worker.list_resources.return_value = [mock_remote_resource] + mock_get_worker.return_value = mock_worker + + # Run the batch task using the session ID from setUpTestData + # We process page 0 with page_size 1 to just handle our mocked resource + tasks._update_harvestable_resources_batch(self.harvesting_session.pk, page=0, page_size=1) + + # ASSERTIONS + # Fetch the resource from the DB to see if it updated + resource = models.HarvestableResource.objects.get(unique_identifier=target_uid) + + # Verify the title was updated correctly + self.assertEqual(resource.title, new_remote_title) + + # Verify no new records were created (count should still be 3 from setUpTestData) + self.assertEqual(models.HarvestableResource.objects.count(), 3) + def test_harvesting_scheduler(self): mock_harvester = mock.MagicMock(spec=models.Harvester).return_value mock_harvester.scheduling_enabled = True diff --git a/geonode/layers/api/tests.py b/geonode/layers/api/tests.py index 35ded0ef724..89027131586 100644 --- a/geonode/layers/api/tests.py +++ b/geonode/layers/api/tests.py @@ -682,11 +682,7 @@ def test_download_api(self): response = self.client.get(url) self.assertTrue(response.status_code == 200) data = response.json()["dataset"] - download_url_data = data["download_urls"][0] - download_url = reverse("dataset_download", args=[dataset.alternate]) - self.assertEqual(download_url_data["default"], True) - self.assertEqual(download_url_data["ajax_safe"], True) - self.assertEqual(download_url_data["url"], download_url) + self.assertEqual(data["download_urls"], []) link = Link(link_type="original", url="https://myoriginal.org", resource=dataset) link.save() @@ -694,7 +690,6 @@ def test_download_api(self): response = self.client.get(url) data = response.json()["dataset"] download_url_data = data["download_urls"][0] - download_url = reverse("dataset_download", args=[dataset.alternate]) self.assertEqual(download_url_data["default"], True) self.assertEqual(download_url_data["ajax_safe"], False) self.assertEqual(download_url_data["url"], "https://myoriginal.org") diff --git a/geonode/layers/download_handler.py b/geonode/layers/download_handler.py index 7f154a6f30d..843258af903 100644 --- a/geonode/layers/download_handler.py +++ b/geonode/layers/download_handler.py @@ -18,18 +18,11 @@ ######################################################################### import logging -import xml.etree.ElementTree as ET -from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse -from django.template.loader import get_template -from django.urls import reverse +from django.http import Http404 from django.utils.translation import gettext_lazy as _ from django.conf import settings -from geonode.base.auth import get_or_create_token -from geonode.geoserver.helpers import wps_format_is_supported from geonode.layers.views import _resolve_dataset -from geonode.proxy.views import fetch_response_headers -from geonode.utils import HttpClient logger = logging.getLogger("geonode.layers.download_handler") @@ -51,11 +44,7 @@ def get_download_response(self): Basic method. Should return the Response object that allow the resource download """ - resource = self.get_resource() - if not resource: - raise Http404("Resource requested is not available") - response = self.process_dowload(resource) - return response + raise Http404("Direct download for the requested resource is not supported") @property def is_link_resource(self): @@ -82,7 +71,7 @@ def download_url(self): if self.is_link_resource: return resource.link_set.filter(resource=resource.get_self_resource(), link_type="original").first().url - return reverse("dataset_download", args=[resource.alternate]) + return None def get_resource(self): """ @@ -99,70 +88,4 @@ def get_resource(self): except Exception as e: logger.debug(e) - return self._resource - - def process_dowload(self, resource=None): - """ - Generate the response object - """ - if not resource: - resource = self.get_resource() - if not settings.USE_GEOSERVER: - # if GeoServer is not used, we redirect to the proxy download - return HttpResponseRedirect(reverse("download", args=[resource.id])) - - download_format = self.request.GET.get("export_format") - - if download_format and not wps_format_is_supported(download_format, resource.subtype): - logger.error("The format provided is not valid for the selected resource") - return JsonResponse({"error": "The format provided is not valid for the selected resource"}, status=500) - - _format = "application/zip" if resource.is_vector() else "image/tiff" - # getting default payload - tpl = get_template("geoserver/dataset_download.xml") - ctx = {"alternate": resource.alternate, "download_format": download_format or _format} - # applying context for the payload - payload = tpl.render(ctx) - - # init of Client - client = HttpClient() - - headers = {"Content-type": "application/xml", "Accept": "application/xml"} - - # defining the URL needed fr the download - url = f"{settings.OGC_SERVER['default']['LOCATION']}ows?service=WPS&version=1.0.0&REQUEST=Execute" - if not self.request.user.is_anonymous: - # define access token for the user - access_token = get_or_create_token(self.request.user) - url += f"&access_token={access_token}" - - # request to geoserver - response, content = client.request(url=url, data=payload, method="post", headers=headers) - - if not response or response.status_code != 200: - logger.error(f"Download dataset exception: error during call with GeoServer: {content}") - return JsonResponse( - {"error": "Download dataset exception: error during call with GeoServer"}, - status=500, - ) - - # error handling - namespaces = {"ows": "http://www.opengis.net/ows/1.1", "wps": "http://www.opengis.net/wps/1.0.0"} - response_type = response.headers.get("Content-Type") - if response_type == "text/xml": - # parsing XML for get exception - content = ET.fromstring(response.text) - exc = content.find("*//ows:Exception", namespaces=namespaces) or content.find( - "ows:Exception", namespaces=namespaces - ) - if exc: - exc_text = exc.find("ows:ExceptionText", namespaces=namespaces) - logger.error(f"{exc.attrib.get('exceptionCode')} {exc_text.text}") - return JsonResponse({"error": f"{exc.attrib.get('exceptionCode')}: {exc_text.text}"}, status=500) - - return_response = fetch_response_headers( - HttpResponse(content=response.content, status=response.status_code, content_type=download_format), - response.headers, - ) - return_response.headers["Content-Type"] = download_format or _format - return return_response + return self._resource \ No newline at end of file diff --git a/geonode/layers/tests.py b/geonode/layers/tests.py index 6a296bff494..9ae0420c921 100644 --- a/geonode/layers/tests.py +++ b/geonode/layers/tests.py @@ -644,35 +644,26 @@ def test_dataset_download_not_found_for_non_existing_dataset(self): self.assertEqual(404, response.status_code) @override_settings(USE_GEOSERVER=False) - def test_dataset_download_redirect_to_proxy_url(self): - # if settings.USE_GEOSERVER is false, the URL must be redirected + def test_dataset_download_returns_404(self): self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() url = reverse("dataset_download", args=[dataset.alternate]) response = self.client.get(url) - self.assertEqual(302, response.status_code) - self.assertEqual(f"/download/{dataset.id}", response.url) + self.assertEqual(404, response.status_code) - def test_dataset_download_invalid_wps_format(self): - # if settings.USE_GEOSERVER is false, the URL must be redirected + def test_dataset_download_invalid_format(self): self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() url = reverse("dataset_download", args=[dataset.alternate]) response = self.client.get(f"{url}?export_format=foo") - self.assertEqual(500, response.status_code) - self.assertDictEqual({"error": "The format provided is not valid for the selected resource"}, response.json()) + self.assertEqual(404, response.status_code) - @patch("geonode.layers.download_handler.HttpClient.request") - def test_dataset_download_call_the_catalog_raise_error_for_no_200(self, mocked_catalog): - _response = MagicMock(status_code=500, content="foo-bar") - mocked_catalog.return_value = _response, "foo-bar" - # if settings.USE_GEOSERVER is false, the URL must be redirected + def test_dataset_download_no_geoserver_call(self): self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() url = reverse("dataset_download", args=[dataset.alternate]) response = self.client.get(url) - self.assertEqual(500, response.status_code) - self.assertDictEqual({"error": "Download dataset exception: error during call with GeoServer"}, response.json()) + self.assertEqual(404, response.status_code) def test_dataset_download_call_the_catalog_raise_error_for_error_content(self): content = """ @@ -686,24 +677,23 @@ def test_dataset_download_call_the_catalog_raise_error_for_error_content(self): # if settings.USE_GEOSERVER is false, the URL must be redirected self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() - with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.utils.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, content url = reverse("dataset_download", args=[dataset.alternate]) response = self.client.get(url) - self.assertEqual(500, response.status_code) - self.assertDictEqual({"error": "InvalidParameterValue: Foo Bar Exception"}, response.json()) + self.assertEqual(404, response.status_code) - def test_dataset_download_call_the_catalog_works(self): + def test_dataset_download_call_the_catalog(self): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa self.client.login(username="admin", password="admin") dataset = Dataset.objects.first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.utils.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) - self.assertTrue(response.status_code == 200) + self.assertTrue(response.status_code == 404) def test_dataset_download_call_the_catalog_not_work_without_download_resurcebase_perm(self): dataset = Dataset.objects.first() @@ -713,55 +703,42 @@ def test_dataset_download_call_the_catalog_not_work_without_download_resurcebase response = self.client.get(url) self.assertEqual(404, response.status_code) - def test_dataset_download_call_the_catalog_work_anonymous(self): - # if settings.USE_GEOSERVER is false, the URL must be redirected + def test_dataset_download_anonymous(self): _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.utils.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) - self.assertTrue(response.status_code == 200) + self.assertTrue(response.status_code == 404) @override_settings(USE_GEOSERVER=True) - @patch("geonode.layers.download_handler.get_template") - def test_dataset_download_call_the_catalog_work_for_raster(self, pathed_template): + @patch("django.template.loader.get_template") + def test_dataset_download_call_the_catalog_for_raster(self, pathed_template): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.filter(subtype="raster").first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") Dataset.objects.filter(alternate=layer.alternate).update(subtype="raster") - with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.utils.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) - self.assertTrue(response.status_code == 200) - """ - Evaluate that the context used by the template contains the right mimetype for the resource - """ - self.assertTupleEqual( - ({"alternate": layer.alternate, "download_format": "image/tiff"},), pathed_template.mock_calls[1].args - ) + self.assertTrue(response.status_code == 404) @override_settings(USE_GEOSERVER=True) - @patch("geonode.layers.download_handler.get_template") - def test_dataset_download_call_the_catalog_work_for_vector(self, pathed_template): + @patch("django.template.loader.get_template") + def test_dataset_download_call_the_catalog_not_work_for_vector(self, pathed_template): # if settings.USE_GEOSERVER is false, the URL must be redirected _response = MagicMock(status_code=200, text="", headers={"Content-Type": ""}) # noqa dataset = Dataset.objects.filter(subtype="vector").first() layer = create_dataset(dataset.title, dataset.title, dataset.owner, "Point") - with patch("geonode.layers.download_handler.HttpClient.request") as mocked_catalog: + with patch("geonode.utils.HttpClient.request") as mocked_catalog: mocked_catalog.return_value = _response, "" url = reverse("dataset_download", args=[layer.alternate]) response = self.client.get(url) - self.assertTrue(response.status_code == 200) - """ - Evaluate that the context used by the template contains the right mimetype for the resource - """ - self.assertTupleEqual( - ({"alternate": layer.alternate, "download_format": "application/zip"},), pathed_template.mock_calls[1].args - ) + self.assertTrue(response.status_code == 404) @patch.object(Dataset, "get_choices", new_callable=PropertyMock) def test_supports_time_with_vector_time_subtype(self, mock_get_choices): @@ -1327,8 +1304,8 @@ def setUp(self): self.sut = DatasetDownloadHandler(request, self.dataset.alternate) def test_download_url_without_original_link(self): - expected_url = reverse("dataset_download", args=[self.dataset.alternate]) - self.assertEqual(expected_url, self.sut.download_url) + + self.assertIsNone(self.sut.download_url) def test_download_url_with_original_link(self): Link.objects.update_or_create( @@ -1347,10 +1324,6 @@ def test_download_url_with_original_link(self): def test_get_resource_exists(self): self.assertIsNotNone(self.sut.get_resource()) - def test_process_dowload(self): - response = self.sut.get_download_response() - self.assertIsNotNone(response) - class DummyDownloadHandler(DatasetDownloadHandler): def get_download_response(self): diff --git a/geonode/local_settings.py.geoserver.sample b/geonode/local_settings.py.geoserver.sample deleted file mode 100644 index 4b07812523b..00000000000 --- a/geonode/local_settings.py.geoserver.sample +++ /dev/null @@ -1,451 +0,0 @@ -# -*- coding: utf-8 -*- -######################################################################### -# -# Copyright (C) 2018 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### - -""" There are 3 ways to override GeoNode settings: - 1. Using environment variables, if your changes to GeoNode are minimal. - 2. Creating a downstream project, if you are doing a lot of customization. - 3. Override settings in a local_settings.py file, legacy. -""" - -import ast -import os - -try: # python2 - from urlparse import urlparse, urlunparse, urlsplit, urljoin -except ImportError: - # Python 3 fallback - from urllib.parse import urlparse, urlunparse, urlsplit, urljoin -from geonode.settings import * - -PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) - -MEDIA_ROOT = os.getenv("MEDIA_ROOT", os.path.join(PROJECT_ROOT, "uploaded")) - -STATIC_ROOT = os.getenv("STATIC_ROOT", os.path.join(PROJECT_ROOT, "static_root")) - -TIME_ZONE = "UTC" - -# Login and logout urls override -LOGIN_URL = os.getenv("LOGIN_URL", "{}account/login/".format(SITEURL)) -LOGOUT_URL = os.getenv("LOGOUT_URL", "{}account/logout/".format(SITEURL)) - -ACCOUNT_LOGIN_REDIRECT_URL = os.getenv("LOGIN_REDIRECT_URL", SITEURL) -ACCOUNT_LOGOUT_REDIRECT_URL = os.getenv("LOGOUT_REDIRECT_URL", SITEURL) - -AVATAR_GRAVATAR_SSL = ast.literal_eval(os.getenv("AVATAR_GRAVATAR_SSL", "True")) - -# Backend -DATABASES = { - "default": { - "ENGINE": "django.contrib.gis.db.backends.postgis", - "NAME": "geonode", - "USER": "geonode", - "PASSWORD": "geonode", - "HOST": "localhost", - "PORT": "5432", - "CONN_MAX_AGE": 0, - "CONN_TOUT": 5, - "OPTIONS": { - "connect_timeout": 5, - }, - }, - # vector datastore for uploads - "datastore": { - "ENGINE": "django.contrib.gis.db.backends.postgis", - # 'ENGINE': '', # Empty ENGINE name disables - "NAME": "geonode_data", - "USER": "geonode", - "PASSWORD": "geonode", - "HOST": "localhost", - "PORT": "5432", - "CONN_MAX_AGE": 0, - "CONN_TOUT": 5, - "OPTIONS": { - "connect_timeout": 5, - }, - }, -} - -GEOSERVER_LOCATION = os.getenv("GEOSERVER_LOCATION", "http://localhost:8080/geoserver/") - -GEOSERVER_PUBLIC_HOST = os.getenv("GEOSERVER_PUBLIC_HOST", SITE_HOST_NAME) - -GEOSERVER_PUBLIC_PORT = os.getenv("GEOSERVER_PUBLIC_PORT", 8080) - -_default_public_location = ( - "http://{}:{}/geoserver/".format(GEOSERVER_PUBLIC_HOST, GEOSERVER_PUBLIC_PORT) - if GEOSERVER_PUBLIC_PORT - else "http://{}/geoserver/".format(GEOSERVER_PUBLIC_HOST) -) - -GEOSERVER_WEB_UI_LOCATION = os.getenv("GEOSERVER_WEB_UI_LOCATION", GEOSERVER_LOCATION) - -GEOSERVER_PUBLIC_LOCATION = os.getenv("GEOSERVER_PUBLIC_LOCATION", _default_public_location) - -GEOSERVER_ADMIN_USER = os.getenv("GEOSERVER_ADMIN_USER", "admin") - -GEOSERVER_ADMIN_PASSWORD = os.getenv("GEOSERVER_ADMIN_PASSWORD", "geoserver") - -# OGC (WMS/WFS/WCS) Server Settings -OGC_SERVER = { - "default": { - "BACKEND": "geonode.geoserver", - "LOCATION": GEOSERVER_LOCATION, - "WEB_UI_LOCATION": GEOSERVER_WEB_UI_LOCATION, - "LOGIN_ENDPOINT": "j_spring_oauth2_geonode_login", - "LOGOUT_ENDPOINT": "j_spring_oauth2_geonode_logout", - # PUBLIC_LOCATION needs to be kept like this because in dev mode - # the proxy won't work and the integration tests will fail - # the entire block has to be overridden in the local_settings - "PUBLIC_LOCATION": GEOSERVER_PUBLIC_LOCATION, - "USER": GEOSERVER_ADMIN_USER, - "PASSWORD": GEOSERVER_ADMIN_PASSWORD, - "MAPFISH_PRINT_ENABLED": True, - "PRINT_NG_ENABLED": True, - "GEONODE_SECURITY_ENABLED": True, - "GEOFENCE_SECURITY_ENABLED": True, - "GEOFENCE_TIMEOUT": int(os.getenv("GEOFENCE_TIMEOUT", os.getenv("OGC_REQUEST_TIMEOUT", "60"))), - "WMST_ENABLED": False, - "BACKEND_WRITE_ENABLED": True, - "WPS_ENABLED": False, - "LOG_FILE": "%s/geoserver/data/logs/geoserver.log" % os.path.abspath(os.path.join(PROJECT_ROOT, os.pardir)), - # Set to dictionary identifier of database containing spatial data in DATABASES dictionary to enable - "DATASTORE": "datastore", - "TIMEOUT": int(os.getenv("OGC_REQUEST_TIMEOUT", "60")), - "MAX_RETRIES": int(os.getenv("OGC_REQUEST_MAX_RETRIES", "5")), - "BACKOFF_FACTOR": float(os.getenv("OGC_REQUEST_BACKOFF_FACTOR", "0.3")), - "POOL_MAXSIZE": int(os.getenv("OGC_REQUEST_POOL_MAXSIZE", "10")), - "POOL_CONNECTIONS": int(os.getenv("OGC_REQUEST_POOL_CONNECTIONS", "10")), - } -} - -# If you want to enable Mosaics use the following configuration -UPLOADER = { - "BACKEND": "geonode.importer", - "OPTIONS": { - "TIME_ENABLED": True, - "MOSAIC_ENABLED": False, - }, - "SUPPORTED_CRS": ["EPSG:4326", "EPSG:3785", "EPSG:3857", "EPSG:32647", "EPSG:32736"], - "SUPPORTED_EXT": [".shp", ".csv", ".kml", ".kmz", ".json", ".geojson", ".tif", ".tiff", ".geotiff", ".gml", ".xml"], -} - -# CSW settings -CATALOGUE = { - "default": { - # The underlying CSW implementation - # default is pycsw in local mode (tied directly to GeoNode Django DB) - "ENGINE": "geonode.catalogue.backends.pycsw_local", - # pycsw in non-local mode - # 'ENGINE': 'geonode.catalogue.backends.pycsw_http', - # deegree and others - # 'ENGINE': 'geonode.catalogue.backends.generic', - # The FULLY QUALIFIED base url to the CSW instance for this GeoNode - "URL": urljoin(SITEURL, "/catalogue/csw"), - # 'URL': 'http://localhost:8080/deegree-csw-demo-3.0.4/services', - # 'ALTERNATES_ONLY': True, - } -} - -# pycsw settings -PYCSW = { - # pycsw configuration - "CONFIGURATION": { - # uncomment / adjust to override server config system defaults - # 'server': { - # 'maxrecords': '10', - # 'pretty_print': 'true', - # 'federatedcatalogues': 'http://catalog.data.gov/csw' - # }, - "server": { - "home": ".", - "url": CATALOGUE["default"]["URL"], - "encoding": "UTF-8", - "language": LANGUAGE_CODE if LANGUAGE_CODE in ("en", "fr", "el") else "en", - "maxrecords": "20", - "pretty_print": "true", - # 'domainquerytype': 'range', - "domaincounts": "true", - "profiles": "apiso,ebrim", - }, - "manager": { - # authentication/authorization is handled by Django - "transactions": "false", - "allowed_ips": "*", - # 'csw_harvest_pagesize': '10', - }, - "metadata": { - "inspire": { - "enabled": True, - "languages_supported": "eng,gre", - "default_language": "eng", - "date": "YYYY-MM-DD", - "gemet_keywords": "Utility and governmental services", - "conformity_service": "notEvaluated", - "contact_name": "Organization Name", - "contact_email": "Email Address", - "temp_extent": { - "begin": "YYYY-MM-DD", - "end": "YYYY-MM-DD" - }, - }, - "identification": { - "title": "GeoNode Catalogue", - "description": "GeoNode is an open source platform" - " that facilitates the creation, sharing, and collaborative use" - " of geospatial data", - "keywords": "sdi, catalogue, discovery, metadata," " GeoNode", - "keywords_type": "theme", - "fees": "None", - "accessconstraints": "None", - }, - "provider": { - "name": "Organization Name", - "url": SITEURL, - }, - "contact": { - "name": "Lastname, Firstname", - "position": "Position Title", - "address": "Mailing Address", - "city": "City", - "stateorprovince": "Administrative Area", - "postalcode": "Zip or Postal Code", - "country": "Country", - "phone": "+xx-xxx-xxx-xxxx", - "fax": "+xx-xxx-xxx-xxxx", - "email": "Email Address", - "url": "Contact URL", - "hours": "Hours of Service", - "instructions": "During hours of service. Off on " "weekends.", - "role": "pointOfContact", - } - } - } -} - -# -- START Client Hooksets Setup - -# GeoNode javascript client configuration - -# default map projection -# Note: If set to EPSG:4326, then only EPSG:4326 basemaps will work. -DEFAULT_MAP_CRS = os.environ.get("DEFAULT_MAP_CRS", "EPSG:3857") - -DEFAULT_LAYER_FORMAT = os.environ.get("DEFAULT_LAYER_FORMAT", "image/png") - -# Where should newly created maps be focused? -DEFAULT_MAP_CENTER = (os.environ.get("DEFAULT_MAP_CENTER_X", 0), os.environ.get("DEFAULT_MAP_CENTER_Y", 0)) - -# How tightly zoomed should newly created maps be? -# 0 = entire world; -# maximum zoom is between 12 and 15 (for Google Maps, coverage varies by area) -DEFAULT_MAP_ZOOM = int(os.environ.get("DEFAULT_MAP_ZOOM", 3)) - -MAPBOX_ACCESS_TOKEN = os.environ.get("MAPBOX_ACCESS_TOKEN", None) -GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", None) - -GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY = os.getenv("GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore") - -MAP_BASELAYERS = [{}] - -""" -MapStore2 REACT based Client parameters -""" -if GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY == "mapstore": - GEONODE_CLIENT_HOOKSET = os.getenv("GEONODE_CLIENT_HOOKSET", "geonode_mapstore_client.hooksets.MapStoreHookSet") - - if "geonode_mapstore_client" not in INSTALLED_APPS: - INSTALLED_APPS += ( - "mapstore2_adapter", - "geonode_mapstore_client", - ) - - def get_geonode_catalogue_service(): - if PYCSW: - pycsw_config = PYCSW["CONFIGURATION"] - if pycsw_config: - pycsw_catalogue = { - ("%s" % pycsw_config["metadata"]["identification"]): { - "url": CATALOGUE["default"]["URL"], - "type": "csw", - "title": pycsw_config["metadata"]["identification"]["title"], - "autoload": True, - } - } - return pycsw_catalogue - return None - - GEONODE_CATALOGUE_SERVICE = get_geonode_catalogue_service() - - DEFAULT_MS2_BACKGROUNDS = [ - { - "type": "tileprovider", - "title": "Stamen Watercolor", - "provider": "Stamen.Watercolor", - "name": "Stamen.Watercolor", - "source": "Stamen", - "group": "background", - "thumbURL": "https://stamen-tiles-c.a.ssl.fastly.net/watercolor/0/0/0.jpg", - "visibility": False, - }, - { - "type": "tileprovider", - "title": "Stamen Terrain", - "provider": "Stamen.Terrain", - "name": "Stamen.Terrain", - "source": "Stamen", - "group": "background", - "thumbURL": "https://stamen-tiles-d.a.ssl.fastly.net/terrain/0/0/0.png", - "visibility": False, - }, - { - "type": "tileprovider", - "title": "Stamen Toner", - "provider": "Stamen.Toner", - "name": "Stamen.Toner", - "source": "Stamen", - "group": "background", - "thumbURL": "https://stamen-tiles-d.a.ssl.fastly.net/toner/0/0/0.png", - "visibility": False, - }, - { - "type": "osm", - "title": "Open Street Map", - "name": "mapnik", - "source": "osm", - "group": "background", - "visibility": True, - }, - { - "type": "tileprovider", - "title": "OpenTopoMap", - "provider": "OpenTopoMap", - "name": "OpenTopoMap", - "source": "OpenTopoMap", - "group": "background", - "visibility": False, - }, - { - "type": "wms", - "title": "Sentinel-2 cloudless - https://s2maps.eu", - "format": "image/jpeg", - "id": "s2cloudless", - "name": "s2cloudless:s2cloudless", - "url": [ - "https://maps1.geosolutionsgroup.com/geoserver/wms", - "https://maps2.geosolutionsgroup.com/geoserver/wms", - "https://maps3.geosolutionsgroup.com/geoserver/wms", - "https://maps4.geosolutionsgroup.com/geoserver/wms", - "https://maps5.geosolutionsgroup.com/geoserver/wms", - "https://maps6.geosolutionsgroup.com/geoserver/wms", - ], - "group": "background", - "thumbURL": f"{SITEURL}static/mapstorestyle/img/s2cloudless-s2cloudless.png", - "visibility": False, - "credits": { - "title": 'Sentinel-2 cloudless 2016 by EOX IT Services GmbH' - }, - }, - { - "source": "ol", - "group": "background", - "id": "none", - "name": "empty", - "title": "Empty Background", - "type": "empty", - "visibility": False, - "args": ["Empty Background", {"visibility": False}], - }, - ] - - if MAPBOX_ACCESS_TOKEN: - MAPBOX_BASEMAPS = { - "type": "tileprovider", - "title": "MapBox streets-v11", - "provider": "MapBoxStyle", - "name": "MapBox streets-v11", - "accessToken": "%s" % MAPBOX_ACCESS_TOKEN, - "source": "streets-v11", - "thumbURL": "https://api.mapbox.com/styles/v1/mapbox/streets-v11/tiles/256/6/33/23?access_token=%s" - % MAPBOX_ACCESS_TOKEN, - "group": "background", - "visibility": True, - } - DEFAULT_MS2_BACKGROUNDS = [ - MAPBOX_BASEMAPS, - ] + DEFAULT_MS2_BACKGROUNDS - - MAPSTORE_BASELAYERS = DEFAULT_MS2_BACKGROUNDS - # MAPSTORE_BASELAYERS_SOURCES allow to configure tilematrix sets for wmts layers - MAPSTORE_BASELAYERS_SOURCES = os.environ.get("MAPSTORE_BASELAYERS_SOURCES", {}) - -# -- END Client Hooksets Setup - -LOGGING = { - "version": 1, - "disable_existing_loggers": True, - "formatters": { - "verbose": {"format": "%(levelname)s %(asctime)s %(module)s %(process)d " "%(thread)d %(message)s"}, - "simple": { - "format": "%(message)s", - }, - }, - "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, - "handlers": { - "console": {"level": "INFO", "class": "logging.StreamHandler", "formatter": "simple"}, - "mail_admins": { - "level": "ERROR", - "filters": ["require_debug_false"], - "class": "django.utils.log.AdminEmailHandler", - }, - }, - "loggers": { - "django": { - "handlers": ["console"], - "level": "ERROR", - }, - "geonode": { - "handlers": ["console"], - "level": "INFO", - }, - "geoserver-restconfig.catalog": { - "handlers": ["console"], - "level": "ERROR", - }, - "owslib": { - "handlers": ["console"], - "level": "ERROR", - }, - "pycsw": { - "handlers": ["console"], - "level": "INFO", - }, - "celery": { - "handlers": ["console"], - "level": "ERROR", - }, - }, -} - -# Additional settings -X_FRAME_OPTIONS = "ALLOW-FROM %s" % SITEURL -CORS_ALLOW_ALL_ORIGINS = True - -GEOIP_PATH = "/usr/local/share/GeoIP" diff --git a/geonode/metadata/api/__init__.py b/geonode/metadata/api/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/metadata/handlers/__init__.py b/geonode/metadata/handlers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/metadata/schemas/__init__.py b/geonode/metadata/schemas/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/security/registry.py b/geonode/security/registry.py index 4c89e36db60..60ff75cb735 100644 --- a/geonode/security/registry.py +++ b/geonode/security/registry.py @@ -462,6 +462,13 @@ def delete_resource_permissions_cache(self, instance, user_clear_cache=True, gro else: pass + def clear_permissions_cache(self): + """ + Clear all permission cache entries. + """ + # This wipes everything in the default cache + cache.clear() + def __check_item(self, item): """ Ensure that the handler is a subclass of BasePermissionsHandler diff --git a/geonode/security/tests.py b/geonode/security/tests.py index f11ed8aef98..0f650b6f68b 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -903,7 +903,7 @@ def test_perm_specs_synchronization(self): self.assertEqual(rules_count, 0) @patch.dict(os.environ, {"ASYNC_SIGNALS": "False"}) - @override_settings(ASYNC_SIGNALS=False) + @override_settings(ASYNC_SIGNALS=False, IMPORTER_ENABLE_DYN_MODELS=False) @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_dataset_permissions(self): # Test permissions on a layer @@ -3736,3 +3736,60 @@ def test_delete_resource_permissions_cache_function(self): temp_user.delete() temp_group_profile.delete() temp_group.delete() + + def test_clear_permissions_cache(self): + """Test that clear_permissions_cache removes all cache entries.""" + test_resource = self.resources[0] + anonymous_user = Profile.objects.get(username="AnonymousUser") + + admin_key = f"resource_perms:{test_resource.pk}:user:{self.admin_user.pk}" + test_user_key = f"resource_perms:{test_resource.pk}:user:{self.test_user.pk}" + anonymous_key = f"resource_perms:{test_resource.pk}:anonymous" + group_key = f"resource_perms:{test_resource.pk}:group:{self.test_group.pk}" + all_key = f"resource_perms:{test_resource.pk}:__ALL__" + + permissions_registry.get_perms(instance=test_resource, user=self.admin_user, use_cache=True) + permissions_registry.get_perms(instance=test_resource, user=self.test_user, use_cache=True) + permissions_registry.get_perms(instance=test_resource, user=anonymous_user, use_cache=True) + permissions_registry.get_perms(instance=test_resource, group=self.test_group, use_cache=True) + permissions_registry.get_perms(instance=test_resource, use_cache=True) + + self.assertIsNotNone(cache.get(admin_key)) + self.assertIsNotNone(cache.get(test_user_key)) + self.assertIsNotNone(cache.get(anonymous_key)) + self.assertIsNotNone(cache.get(group_key)) + self.assertIsNotNone(cache.get(all_key)) + + permissions_registry.clear_permissions_cache() + + self.assertIsNone(cache.get(admin_key)) + self.assertIsNone(cache.get(test_user_key)) + self.assertIsNone(cache.get(anonymous_key)) + self.assertIsNone(cache.get(group_key)) + self.assertIsNone(cache.get(all_key)) + + def test_configuration_read_only_change_clears_permissions_cache(self): + """Permissions cache is cleared when read_only flag changes.""" + test_resource = self.resources[0] + user = self.admin_user + + config = Configuration.load() + original_read_only = config.read_only + + try: + cache.clear() + + config.read_only = False + config.save() + + permissions_registry.get_perms(instance=test_resource, user=user, use_cache=True) + cache_key = permissions_registry._get_cache_key([test_resource.pk], users=[user]) + self.assertIsNotNone(cache.get(cache_key)) + + config.read_only = True + config.save() + + self.assertIsNone(cache.get(cache_key)) + finally: + config.read_only = original_read_only + config.save() diff --git a/geonode/services/serviceprocessors/wms.py b/geonode/services/serviceprocessors/wms.py index ea1f8be7b25..2564ab2b430 100644 --- a/geonode/services/serviceprocessors/wms.py +++ b/geonode/services/serviceprocessors/wms.py @@ -46,7 +46,7 @@ from ..enumerations import INDEXED from .. import models from .. import utils -from . import base +from . import base, get_service_handler logger = logging.getLogger(__name__) @@ -288,7 +288,13 @@ def __init__(self, url, geonode_service_id=None, *args, **kwargs): @property def parsed_service(self): cleaned_url, service, version, request = WmsServiceHandler.get_cleaned_url_params(self.ows_endpoint()) - _url, _parsed_service = WebMapService(cleaned_url.geturl(), version=version) + _parsed_service = get_service_handler( + cleaned_url.geturl(), + service.type, + service.id, + username=service.username if service.needs_authentication else None, + password=service.get_password() if service.needs_authentication else None, + ) return _parsed_service def probe(self): diff --git a/geonode/services/templates/services/service_resources_harvest.html b/geonode/services/templates/services/service_resources_harvest.html index 8de1f454f43..8a0279e4427 100644 --- a/geonode/services/templates/services/service_resources_harvest.html +++ b/geonode/services/templates/services/service_resources_harvest.html @@ -177,8 +177,8 @@

{{service.title|default:service.name}}

resources.push({ id: '{{ resource_meta.id }}', name: '{{ resource_meta.unique_identifier }}', - title: '{{ resource_meta.title }}', - abstract: '{{ resource_meta.abstract|safe|truncatechars:20|striptags }}', + title: '{{ resource_meta.title|escapejs }}', + abstract: '{{ resource_meta.abstract|truncatechars:20|striptags|escapejs }}', type: '{{ resource_meta.remote_resource_type }}' }); {% endfor %} diff --git a/geonode/services/views.py b/geonode/services/views.py index 67f054e8a1a..915615e274e 100644 --- a/geonode/services/views.py +++ b/geonode/services/views.py @@ -97,7 +97,13 @@ def _get_service_handler(request, service): multiple Capabilities requests (this is a time saver on servers that feature many layers. """ - service_handler = get_service_handler(service.service_url, service.type, service.id) + service_handler = get_service_handler( + service.service_url, + service.type, + service.id, + username=service.username if service.needs_authentication else None, + password=service.get_password() if service.needs_authentication else None, + ) if not service_handler.geonode_service_id: service_handler.geonode_service_id = service.id # commented out due to jsonserializer error, will be replaced with cache @@ -320,6 +326,7 @@ def remove_service(request, service_id): elif request.method == "POST": service.dataset_set.all().delete() # by deleting the harvester we delete also the service + service_cache.delete(service.base_url) service.harvester.delete() messages.add_message(request, messages.INFO, _(f"Service {service.title} has been deleted")) return HttpResponseRedirect(reverse("services")) diff --git a/geonode/settings.py b/geonode/settings.py index 0c988cd7142..b9371d1fe11 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -76,6 +76,10 @@ else: EMAIL_BACKEND = os.getenv("DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend") + +# Email for users to contact admins. +THEME_ACCOUNT_CONTACT_EMAIL = os.getenv("THEME_ACCOUNT_CONTACT_EMAIL", "admin@example.com") + # Make this unique, and don't share it with anybody. _DEFAULT_SECRET_KEY = "myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a" SECRET_KEY = os.getenv("SECRET_KEY", _DEFAULT_SECRET_KEY) @@ -563,6 +567,21 @@ "basic": {"exclude[]": ["*"], "include[]": ["pk", "title", "abstract", "resource_type"]}, } +# If a command name is listed here, the command will be available to admins over http +# This list is used by the management_commands_http app +MANAGEMENT_COMMANDS_EXPOSED_OVER_HTTP = set( + [ + "ping_mngmt_commands_http", + "updatelayers", + "sync_geonode_datasets", + "sync_geonode_maps", + "importlayers", + "set_all_datasets_metadata", + "set_layers_permissions", + ] + + ast.literal_eval(os.getenv("MANAGEMENT_COMMANDS_EXPOSED_OVER_HTTP", "[]")) +) + DYNAMIC_REST = { # DEBUG: enable/disable internal debugging "DEBUG": False, @@ -814,7 +833,6 @@ }, } - MIDDLEWARE = ( "corsheaders.middleware.CorsMiddleware", "django.middleware.common.CommonMiddleware", @@ -845,6 +863,22 @@ if SESSION_ENGINE in ("django.contrib.sessions.backends.cached_db", "django.contrib.sessions.backends.cache"): SESSION_CACHE_ALIAS = "memcached" # use memcached cache if a cached backend is requested +# Add additional paths (as regular expressions) that don't require +# authentication. +# - authorized exempt urls needed for oauth when GeoNode is set to lockdown +AUTH_EXEMPT_URLS = ( + f"{FORCE_SCRIPT_NAME}/o/*", + f"{FORCE_SCRIPT_NAME}/gs/*", + f"{FORCE_SCRIPT_NAME}/account/*", + f"{FORCE_SCRIPT_NAME}/static/*", + f"{FORCE_SCRIPT_NAME}/api/o/*", + f"{FORCE_SCRIPT_NAME}/api/roles", + f"{FORCE_SCRIPT_NAME}/api/adminRole", + f"{FORCE_SCRIPT_NAME}/api/users", + f"{FORCE_SCRIPT_NAME}/api/datasets", + r"^/i18n/setlang/?$", +) + # Security stuff SESSION_EXPIRED_CONTROL_ENABLED = ast.literal_eval(os.environ.get("SESSION_EXPIRED_CONTROL_ENABLED", "True")) @@ -869,6 +903,26 @@ SECURE_HSTS_INCLUDE_SUBDOMAINS = ast.literal_eval(os.environ.get("SECURE_HSTS_INCLUDE_SUBDOMAINS", "True")) SECURE_REFERRER_POLICY = os.environ.get("SECURE_REFERRER_POLICY", "strict-origin-when-cross-origin") +# Settings for RECAPTCHA plugin +RECAPTCHA_ENABLED = ast.literal_eval(os.environ.get("RECAPTCHA_ENABLED", "False")) + +if RECAPTCHA_ENABLED: + if "django_recaptcha" not in INSTALLED_APPS: + INSTALLED_APPS += ("django_recaptcha",) + ACCOUNT_SIGNUP_FORM_CLASS = os.getenv( + "ACCOUNT_SIGNUP_FORM_CLASS", "geonode.people.forms.recaptcha.AllauthReCaptchaSignupForm" + ) + + # https://docs.allauth.org/en/dev/account/configuration.html + ACCOUNT_FORMS = dict(login="geonode.people.forms.recaptcha.AllauthRecaptchaLoginForm") + """ + In order to generate reCaptcha keys, please see: + - https://pypi.org/project/django-recaptcha/#installation + - https://pypi.org/project/django-recaptcha/#local-development-and-functional-testing + """ + RECAPTCHA_PUBLIC_KEY = os.getenv("RECAPTCHA_PUBLIC_KEY", "geonode_RECAPTCHA_PUBLIC_KEY") + RECAPTCHA_PRIVATE_KEY = os.getenv("RECAPTCHA_PRIVATE_KEY", "geonode_RECAPTCHA_PRIVATE_KEY") + # Replacement of the default authentication backend in order to support # permissions per object. AUTHENTICATION_BACKENDS = ( @@ -926,44 +980,98 @@ # 1 day expiration time by default ACCESS_TOKEN_EXPIRE_SECONDS = int(os.getenv("ACCESS_TOKEN_EXPIRE_SECONDS", "86400")) -# Require users to authenticate before using Geonode -LOCKDOWN_GEONODE = ast.literal_eval(os.getenv("LOCKDOWN_GEONODE", "False")) - -# Add additional paths (as regular expressions) that don't require -# authentication. -# - authorized exempt urls needed for oauth when GeoNode is set to lockdown -AUTH_EXEMPT_URLS = ( - f"{FORCE_SCRIPT_NAME}/o/*", - f"{FORCE_SCRIPT_NAME}/gs/*", - f"{FORCE_SCRIPT_NAME}/account/*", - f"{FORCE_SCRIPT_NAME}/static/*", - f"{FORCE_SCRIPT_NAME}/api/o/*", - f"{FORCE_SCRIPT_NAME}/api/roles", - f"{FORCE_SCRIPT_NAME}/api/adminRole", - f"{FORCE_SCRIPT_NAME}/api/users", - f"{FORCE_SCRIPT_NAME}/api/datasets", - r"^/i18n/setlang/?$", -) - ANONYMOUS_USER_ID = os.getenv("ANONYMOUS_USER_ID", "-1") GUARDIAN_GET_INIT_ANONYMOUS_USER = os.getenv( "GUARDIAN_GET_INIT_ANONYMOUS_USER", "geonode.people.models.get_anonymous_user_instance" ) -# Whether the uplaoded resources should be public and downloadable by default -# or not -DEFAULT_ANONYMOUS_VIEW_PERMISSION = ast.literal_eval(os.getenv("DEFAULT_ANONYMOUS_VIEW_PERMISSION", "True")) -DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION = ast.literal_eval(os.getenv("DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION", "True")) +try: + # try to parse python notation, default in dockerized env + ALLOWED_HOSTS = ast.literal_eval(os.getenv("ALLOWED_HOSTS")) +except (ValueError, SyntaxError): + # fallback to regular list of values separated with misc chars + ALLOWED_HOSTS = ( + [HOSTNAME, "localhost", "django", "geonode"] + if os.getenv("ALLOWED_HOSTS") is None + else re.split(r" *[,|:;] *", os.getenv("ALLOWED_HOSTS")) + ) +# AUTH_IP_WHITELIST property limits access to users/groups REST endpoints +# to only whitelisted IP addresses. +# +# Empty list means 'allow all' # -# Settings for default search size +# If you need to limit 'api' REST calls to only some specific IPs +# fill the list like below: # -DEFAULT_SEARCH_SIZE = int(os.getenv("DEFAULT_SEARCH_SIZE", "10")) +# AUTH_IP_WHITELIST = ['192.168.1.158', '192.168.1.159'] +AUTH_IP_WHITELIST = ( + [HOSTNAME, "localhost", "django", "geonode"] + if os.getenv("AUTH_IP_WHITELIST") is None + else re.split(r" *[,|:;] *", os.getenv("AUTH_IP_WHITELIST")) +) +# ADMIN_IP_WHITELIST property limits access as admin +# to only whitelisted IP addresses. +# +# Empty list means 'allow all' # -# Settings for third party apps +# If you need to limit admin access to some specific IPs +# fill the list like below: # +# ADMIN_IP_WHITELIST = ['192.168.1.158', '192.168.1.159'] +ADMIN_IP_WHITELIST = ( + [] if os.getenv("ADMIN_IP_WHITELIST") is None else re.split(r" *[,|:;] *", os.getenv("ADMIN_IP_WHITELIST")) +) +if len(ADMIN_IP_WHITELIST) > 0: + AUTHENTICATION_BACKENDS = ("geonode.security.backends.AdminRestrictedAccessBackend",) + AUTHENTICATION_BACKENDS + MIDDLEWARE += ("geonode.security.middleware.AdminAllowedMiddleware",) + +# LOCKDOWN API endpoints to prevent unauthenticated access. +# If set to True, search won't deliver results and filtering ResourceBase-objects is not possible for anonymous users +API_LOCKDOWN = ast.literal_eval(os.getenv("API_LOCKDOWN", "False")) + +# Require users to authenticate before using Geonode +LOCKDOWN_GEONODE = ast.literal_eval(os.getenv("LOCKDOWN_GEONODE", "False")) +# Require users to authenticate before using Geonode +if LOCKDOWN_GEONODE: + MIDDLEWARE += ("geonode.security.middleware.LoginRequiredMiddleware",) + +# A tuple of hosts the proxy can send requests to. +try: + # try to parse python notation, default in dockerized env + PROXY_ALLOWED_HOSTS = ast.literal_eval(os.getenv("PROXY_ALLOWED_HOSTS")) +except (ValueError, SyntaxError): + # fallback to regular list of values separated with misc chars + PROXY_ALLOWED_HOSTS = ( + [ + HOSTNAME, + "localhost", + "django", + "geonode", + "spatialreference.org", + "nominatim.openstreetmap.org", + "dev.openlayers.org", + ] + if os.getenv("PROXY_ALLOWED_HOSTS") is None + else re.split(r" *[,|:;] *", os.getenv("PROXY_ALLOWED_HOSTS")) + ) + +# Tuple with valid strings to be matched inside the request querystring to let it pass through the proxy +PROXY_ALLOWED_PARAMS_NEEDLES = ast.literal_eval(os.getenv("PROXY_ALLOWED_PARAMS_NEEDLES", "()")) +# Tuple with valid strings to be matched inside the request path to let it pass through the proxy +PROXY_ALLOWED_PATH_NEEDLES = ast.literal_eval(os.getenv("PROXY_ALLOWED_PATH_NEEDLES", "()")) + +# The proxy to use when making cross origin requests. +PROXY_URL = os.environ.get("PROXY_URL", "/proxy/?url=") + +# Avoid permissions prefiltering +SKIP_PERMS_FILTER = ast.literal_eval(os.getenv("SKIP_PERMS_FILTER", "False")) + +# Number of items returned by the apis 0 equals no limit +API_LIMIT_PER_PAGE = int(os.getenv("API_LIMIT_PER_PAGE", "200")) +API_INCLUDE_REGIONS_COUNT = ast.literal_eval(os.getenv("API_INCLUDE_REGIONS_COUNT", "False")) # Pinax Ratings PINAX_RATINGS_CATEGORY_CHOICES = { @@ -981,15 +1089,6 @@ "GFK_FETCH_DEPTH": 1, } - -# Email for users to contact admins. -THEME_ACCOUNT_CONTACT_EMAIL = os.getenv("THEME_ACCOUNT_CONTACT_EMAIL", "admin@example.com") - -# -# GeoNode specific settings -# -# per-deployment settings should go here - # Login and logout urls override LOGIN_URL = os.getenv("LOGIN_URL", f"{SITEURL}account/login/") LOGOUT_URL = os.getenv("LOGOUT_URL", f"{SITEURL}account/logout/") @@ -1001,16 +1100,6 @@ DEFAULT_WORKSPACE = os.getenv("DEFAULT_WORKSPACE", "geonode") CASCADE_WORKSPACE = os.getenv("CASCADE_WORKSPACE", "geonode") -OGP_URL = os.getenv("OGP_URL", "http://geodata.tufts.edu/solr/select") - -# Topic Categories list should not be modified (they are ISO). In case you -# absolutely need it set to True this variable -MODIFY_TOPICCATEGORY = ast.literal_eval(os.getenv("MODIFY_TOPICCATEGORY", "True")) - -# If this option is enabled, Topic Categories will become strictly Mandatory on -# Metadata Wizard -TOPICCATEGORY_MANDATORY = ast.literal_eval(os.environ.get("TOPICCATEGORY_MANDATORY", "False")) - MISSING_THUMBNAIL = os.getenv("MISSING_THUMBNAIL", "geonode/img/missing_thumb.png") GEOSERVER_LOCATION = os.getenv("GEOSERVER_LOCATION", "http://localhost:8080/geoserver/") @@ -1085,11 +1174,8 @@ # Uploader Settings DATA_UPLOAD_MAX_NUMBER_FIELDS = 100000 -""" - DEFAULT_BACKEND_UPLOADER = {'geonode.importer'} -""" UPLOADER = { - "BACKEND": os.getenv("DEFAULT_BACKEND_UPLOADER", "geonode.upload"), + "BACKEND": "geonode.upload", } EPSG_CODE_MATCHES = { @@ -1198,37 +1284,6 @@ _DATETIME_INPUT_FORMATS = ["%Y-%m-%d %H:%M:%S.%f %Z", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S%Z"] DATETIME_INPUT_FORMATS = DATETIME_INPUT_FORMATS + _DATETIME_INPUT_FORMATS -DISPLAY_SOCIAL = ast.literal_eval(os.getenv("DISPLAY_SOCIAL", "True")) -DISPLAY_RATINGS = ast.literal_eval(os.getenv("DISPLAY_RATINGS", "True")) -DISPLAY_WMS_LINKS = ast.literal_eval(os.getenv("DISPLAY_WMS_LINKS", "True")) - -SOCIAL_ORIGINS = [ - {"label": "Email", "url": "mailto:?subject={name}&body={url}", "css_class": "email"}, - {"label": "Facebook", "url": "http://www.facebook.com/sharer.php?u={url}", "css_class": "fb"}, - {"label": "Twitter", "url": "https://twitter.com/share?url={url}&hashtags={hashtags}", "css_class": "tw"}, -] - -# CKAN Query String Parameters names pulled from -# http://tinyurl.com/og2jofn -CKAN_ORIGINS = [ - { - "label": "Humanitarian Data Exchange (HDX)", - "url": "https://data.hdx.rwlabs.org/dataset/new?title={name}&" - "dataset_date={date}¬es={abstract}&caveats={caveats}", - "css_class": "hdx", - } -] -# SOCIAL_ORIGINS.extend(CKAN_ORIGINS) - -# Setting TWITTER_CARD to True will enable Twitter Cards -# https://dev.twitter.com/cards/getting-started -# Be sure to replace @GeoNode with your organization or site's twitter handle. -TWITTER_CARD = ast.literal_eval(os.getenv("TWITTER_CARD", "True")) -TWITTER_SITE = "@GeoNode" -TWITTER_HASHTAGS = ["geonode"] - -OPENGRAPH_ENABLED = ast.literal_eval(os.getenv("OPENGRAPH_ENABLED", "True")) - # Enable Licenses User Interface # Regardless of selection, license field stil exists as a field in the # Resourcebase model. @@ -1244,153 +1299,53 @@ "DETAIL": "never", } -try: - # try to parse python notation, default in dockerized env - ALLOWED_HOSTS = ast.literal_eval(os.getenv("ALLOWED_HOSTS")) -except ValueError: - # fallback to regular list of values separated with misc chars - ALLOWED_HOSTS = ( - [HOSTNAME, "localhost", "django", "geonode"] - if os.getenv("ALLOWED_HOSTS") is None - else re.split(r" *[,|:;] *", os.getenv("ALLOWED_HOSTS")) - ) -# AUTH_IP_WHITELIST property limits access to users/groups REST endpoints -# to only whitelisted IP addresses. -# -# Empty list means 'allow all' -# -# If you need to limit 'api' REST calls to only some specific IPs -# fill the list like below: -# -# AUTH_IP_WHITELIST = ['192.168.1.158', '192.168.1.159'] -AUTH_IP_WHITELIST = ( - [HOSTNAME, "localhost", "django", "geonode"] - if os.getenv("AUTH_IP_WHITELIST") is None - else re.split(r" *[,|:;] *", os.getenv("AUTH_IP_WHITELIST")) -) +# Available download formats +DOWNLOAD_FORMATS_METADATA = [ + "Atom", + "DIF", + "Dublin Core", + "ebRIM", + "FGDC", + "ISO", +] +DOWNLOAD_FORMATS_VECTOR = [ + "JPEG", + "PDF", + "PNG", + "Zipped Shapefile", + "GML 2.0", + "GML 3.1.1", + "CSV", + "Excel", + "GeoJSON", + "KML", + "View in Google Earth", + "Tiles", +] +DOWNLOAD_FORMATS_RASTER = [ + "JPEG", + "PDF", + "PNG", + "ArcGrid", + "GeoTIFF", + "Gtopo30", + "ImageMosaic", + "KML", + "View in Google Earth", + "Tiles", + "GML", + "GZIP", + "Zipped All Files", +] - -# ADMIN_IP_WHITELIST property limits access as admin -# to only whitelisted IP addresses. -# -# Empty list means 'allow all' -# -# If you need to limit admin access to some specific IPs -# fill the list like below: -# -# ADMIN_IP_WHITELIST = ['192.168.1.158', '192.168.1.159'] -ADMIN_IP_WHITELIST = ( - [] if os.getenv("ADMIN_IP_WHITELIST") is None else re.split(r" *[,|:;] *", os.getenv("ADMIN_IP_WHITELIST")) -) -if len(ADMIN_IP_WHITELIST) > 0: - AUTHENTICATION_BACKENDS = ("geonode.security.backends.AdminRestrictedAccessBackend",) + AUTHENTICATION_BACKENDS - MIDDLEWARE += ("geonode.security.middleware.AdminAllowedMiddleware",) - -# A tuple of hosts the proxy can send requests to. -try: - # try to parse python notation, default in dockerized env - PROXY_ALLOWED_HOSTS = ast.literal_eval(os.getenv("PROXY_ALLOWED_HOSTS")) -except ValueError: - # fallback to regular list of values separated with misc chars - PROXY_ALLOWED_HOSTS = ( - [ - HOSTNAME, - "localhost", - "django", - "geonode", - "spatialreference.org", - "nominatim.openstreetmap.org", - "dev.openlayers.org", - ] - if os.getenv("PROXY_ALLOWED_HOSTS") is None - else re.split(r" *[,|:;] *", os.getenv("PROXY_ALLOWED_HOSTS")) - ) - -# Tuple with valid strings to be matched inside the request querystring to let it pass through the proxy -PROXY_ALLOWED_PARAMS_NEEDLES = ast.literal_eval(os.getenv("PROXY_ALLOWED_PARAMS_NEEDLES", "()")) -# Tuple with valid strings to be matched inside the request path to let it pass through the proxy -PROXY_ALLOWED_PATH_NEEDLES = ast.literal_eval(os.getenv("PROXY_ALLOWED_PATH_NEEDLES", "()")) - -# The proxy to use when making cross origin requests. -PROXY_URL = os.environ.get("PROXY_URL", "/proxy/?url=") - -# Avoid permissions prefiltering -SKIP_PERMS_FILTER = ast.literal_eval(os.getenv("SKIP_PERMS_FILTER", "False")) -# Available download formats -DOWNLOAD_FORMATS_METADATA = [ - "Atom", - "DIF", - "Dublin Core", - "ebRIM", - "FGDC", - "ISO", -] -DOWNLOAD_FORMATS_VECTOR = [ - "JPEG", - "PDF", - "PNG", - "Zipped Shapefile", - "GML 2.0", - "GML 3.1.1", - "CSV", - "Excel", - "GeoJSON", - "KML", - "View in Google Earth", - "Tiles", -] -DOWNLOAD_FORMATS_RASTER = [ - "JPEG", - "PDF", - "PNG", - "ArcGrid", - "GeoTIFF", - "Gtopo30", - "ImageMosaic", - "KML", - "View in Google Earth", - "Tiles", - "GML", - "GZIP", - "Zipped All Files", -] - -ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE = ast.literal_eval(os.getenv("ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE", "False")) +ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE = ast.literal_eval(os.getenv("ACCOUNT_NOTIFY_ON_PASSWORD_CHANGE", "False")) TASTYPIE_DEFAULT_FORMATS = ["json"] -# gravatar settings -AUTO_GENERATE_AVATAR_SIZES = (20, 30, 32, 40, 50, 65, 70, 80, 100, 140, 200, 240) -AVATAR_GRAVATAR_SSL = ast.literal_eval(os.getenv("AVATAR_GRAVATAR_SSL", "False")) - -AVATAR_DEFAULT_URL = os.getenv("AVATAR_DEFAULT_URL", "/geonode/img/avatar.png") - -try: - # try to parse python notation, default in dockerized env - AVATAR_PROVIDERS = ast.literal_eval(os.getenv("AVATAR_PROVIDERS")) -except ValueError: - # fallback to regular list of values separated with misc chars - AVATAR_PROVIDERS = ( - ( - "avatar.providers.PrimaryAvatarProvider", - "avatar.providers.DefaultAvatarProvider", - ) - if os.getenv("AVATAR_PROVIDERS") is None - else re.split(r" *[,|:;] *", os.getenv("AVATAR_PROVIDERS")) - ) - # Number of results per page listed in the GeoNode search pages CLIENT_RESULTS_LIMIT = int(os.getenv("CLIENT_RESULTS_LIMIT", "16")) -# LOCKDOWN API endpoints to prevent unauthenticated access. -# If set to True, search won't deliver results and filtering ResourceBase-objects is not possible for anonymous users -API_LOCKDOWN = ast.literal_eval(os.getenv("API_LOCKDOWN", "False")) - -# Number of items returned by the apis 0 equals no limit -API_LIMIT_PER_PAGE = int(os.getenv("API_LIMIT_PER_PAGE", "200")) -API_INCLUDE_REGIONS_COUNT = ast.literal_eval(os.getenv("API_INCLUDE_REGIONS_COUNT", "False")) - # Settings for EXIF plugin EXIF_ENABLED = ast.literal_eval(os.getenv("EXIF_ENABLED", "True")) @@ -1405,26 +1360,6 @@ if "geonode.geoserver.createlayer" not in INSTALLED_APPS: INSTALLED_APPS += ("geonode.geoserver.createlayer",) -# Settings for RECAPTCHA plugin -RECAPTCHA_ENABLED = ast.literal_eval(os.environ.get("RECAPTCHA_ENABLED", "False")) - -if RECAPTCHA_ENABLED: - if "django_recaptcha" not in INSTALLED_APPS: - INSTALLED_APPS += ("django_recaptcha",) - ACCOUNT_SIGNUP_FORM_CLASS = os.getenv( - "ACCOUNT_SIGNUP_FORM_CLASS", "geonode.people.forms.recaptcha.AllauthReCaptchaSignupForm" - ) - - # https://docs.allauth.org/en/dev/account/configuration.html - ACCOUNT_FORMS = dict(login="geonode.people.forms.recaptcha.AllauthRecaptchaLoginForm") - """ - In order to generate reCaptcha keys, please see: - - https://pypi.org/project/django-recaptcha/#installation - - https://pypi.org/project/django-recaptcha/#local-development-and-functional-testing - """ - RECAPTCHA_PUBLIC_KEY = os.getenv("RECAPTCHA_PUBLIC_KEY", "geonode_RECAPTCHA_PUBLIC_KEY") - RECAPTCHA_PRIVATE_KEY = os.getenv("RECAPTCHA_PRIVATE_KEY", "geonode_RECAPTCHA_PRIVATE_KEY") - GEONODE_CATALOGUE_METADATA_XSL = ast.literal_eval(os.getenv("GEONODE_CATALOGUE_METADATA_XSL", "True")) # -- START Client Hooksets Setup @@ -1457,8 +1392,6 @@ GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY = os.getenv("GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY", "mapstore") -MAP_BASELAYERS = [{}] - """ MapStore2 REACT based Client parameters """ @@ -1618,7 +1551,7 @@ def get_geonode_catalogue_service(): MAPSTORE_PLUGINS_CONFIG_PATCH_RULES = [] # Extensions path to use in importing custom extensions into geonode - MAPSTORE_EXTENSIONS_FOLDER_PATH = "/static/mapstore/extensions/" + MAPSTORE_EXTENSIONS_FOLDER_PATH = "mapstore/extensions/" # Supported Dataset file types for uploading Datasets. This setting is being from from the client @@ -1641,47 +1574,6 @@ def get_geonode_catalogue_service(): "GROUP_CATEGORIES_ENABLED": True, } -# HTML WYSIWYG Editor (TINYMCE) Menu Bar Settings -TINYMCE_DEFAULT_CONFIG = { - "theme": "silver", - "height": 200, - "plugins": "preview paste searchreplace autolink directionality code visualblocks visualchars fullscreen image link media template codesample table charmap hr pagebreak nonbreaking insertdatetime advlist lists wordcount imagetools textpattern noneditable help charmap quickbars", # noqa - "imagetools_cors_hosts": ["picsum.photos"], - "menubar": False, - "statusbar": False, - "toolbar": "bold italic underline | formatselect removeformat | outdent indent | numlist bullist | insertfile image media link codesample | preview", # noqa - "toolbar_sticky": "true", - "autosave_ask_before_unload": "true", - "autosave_interval": "30s", - "autosave_prefix": "{path}{query}-{id}-", - "autosave_restore_when_empty": "false", - "autosave_retention": "2m", - "image_advtab": "true", - "content_css": "//www.tiny.cloud/css/codepen.min.css", - "importcss_append": "true", - "image_caption": "true", - "quickbars_selection_toolbar": "bold italic | quicklink h2 h3 blockquote quickimage quicktable", - "noneditable_noneditable_class": "mceNonEditable", - "toolbar_mode": "sliding", - "contextmenu": "link image imagetools table", - "templates": [ - { - "title": "New Table", - "description": "creates a new table", - "content": '
', # noqa - }, - {"title": "Starting my story", "description": "A cure for writers block", "content": "Once upon a time..."}, - { - "title": "New list with dates", - "description": "New List with dates", - "content": '
cdate
mdate

My List

', # noqa - }, - ], - "template_cdate_format": "[Date Created (CDATE): %m/%d/%Y : %H:%M:%S]", - "template_mdate_format": "[Date Modified (MDATE): %m/%d/%Y : %H:%M:%S]", - "setup": 'function(editor) {editor.on("input", onInputChange)}', -} - # ########################################################################### # # ASYNC SETTINGS @@ -1784,6 +1676,61 @@ def get_geonode_catalogue_service(): Queue("geonode.layer.viewer", GEOSERVER_EXCHANGE, routing_key="geonode.viewer"), ) +CELERY_TASK_QUEUES += ( + Queue("geonode.upload.import_orchestrator", GEONODE_EXCHANGE, routing_key="geonode.upload.import_orchestrator"), + Queue( + "geonode.upload.import_resource", GEONODE_EXCHANGE, routing_key="geonode.upload.import_resource", max_priority=8 + ), + Queue( + "geonode.upload.publish_resource", + GEONODE_EXCHANGE, + routing_key="geonode.upload.publish_resource", + max_priority=8, + ), + Queue( + "geonode.upload.create_geonode_resource", + GEONODE_EXCHANGE, + routing_key="geonode.upload.create_geonode_resource", + max_priority=8, + ), + Queue( + "geonode.upload.import_with_ogr2ogr", + GEONODE_EXCHANGE, + routing_key="geonode.upload.import_with_ogr2ogr", + max_priority=10, + ), + Queue( + "geonode.upload.import_next_step", + GEONODE_EXCHANGE, + routing_key="geonode.upload.import_next_step", + max_priority=3, + ), + Queue( + "geonode.upload.create_dynamic_structure", + GEONODE_EXCHANGE, + routing_key="geonode.upload.create_dynamic_structure", + max_priority=10, + ), + Queue( + "geonode.upload.copy_geonode_resource", + GEONODE_EXCHANGE, + routing_key="geonode.upload.copy_geonode_resource", + max_priority=0, + ), + Queue("geonode.upload.copy_dynamic_model", GEONODE_EXCHANGE, routing_key="geonode.upload.copy_dynamic_model"), + Queue( + "geonode.upload.copy_geonode_data_table", GEONODE_EXCHANGE, routing_key="geonode.upload.copy_geonode_data_table" + ), + Queue("geonode.upload.copy_raster_file", GEONODE_EXCHANGE, routing_key="geonode.upload.copy_raster_file"), + Queue("geonode.upload.rollback", GEONODE_EXCHANGE, routing_key="geonode.upload.rollback"), + Queue("geonode.upload.upsert_data", GEONODE_EXCHANGE, routing_key="geonode.upload.upsert_data"), + Queue( + "geonode.upload.refresh_geonode_resource", + GEONODE_EXCHANGE, + routing_key="geonode.upload.refresh_geonode_resource", + ), +) + # from celery.schedules import crontab # EXAMPLES # ... @@ -1876,32 +1823,11 @@ def get_geonode_catalogue_service(): INSTALLED_APPS += (NOTIFICATIONS_MODULE,) # ########################################################################### # -# SECURITY SETTINGS +# START SECURITY SETTINGS # ########################################################################### # ENABLE_APIKEY_LOGIN = ast.literal_eval(os.getenv("ENABLE_APIKEY_LOGIN", "False")) -# Require users to authenticate before using Geonode -if LOCKDOWN_GEONODE: - MIDDLEWARE += ("geonode.security.middleware.LoginRequiredMiddleware",) - -# for windows users check if they didn't set GEOS and GDAL in local_settings.py -# maybe they set it as a windows environment -if os.name == "nt": - if "GEOS_LIBRARY_PATH" not in locals() or "GDAL_LIBRARY_PATH" not in locals(): - if os.environ.get("GEOS_LIBRARY_PATH", None) and os.environ.get("GDAL_LIBRARY_PATH", None): - GEOS_LIBRARY_PATH = os.environ.get("GEOS_LIBRARY_PATH") - GDAL_LIBRARY_PATH = os.environ.get("GDAL_LIBRARY_PATH") - else: - # maybe it will be found regardless if not it will throw 500 error - from django.contrib.gis.geos import GEOSGeometry # noqa - -# Keywords thesauri -# e.g. THESAURUS = {'name':'inspire_themes', 'required':True, 'filter':True} -# Required: (boolean, optional, default false) mandatory while editing metadata (not implemented yet) -# Filter: (boolean, optional, default false) a filter option on that thesaurus will appear in the main search page -# THESAURUS = {'name': 'inspire_themes', 'required': True, 'filter': True} - # ######################################################## # # Advanced Resource Publishing Worklow Settings - START # # ######################################################## # @@ -1924,13 +1850,62 @@ def get_geonode_catalogue_service(): # Advanced Resource Publishing Worklow Settings - END # # ######################################################## # +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS = ast.literal_eval( + os.getenv("AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS", "True") +) + +# Whether the uplaoded resources should be public and downloadable by default +# or not +DEFAULT_ANONYMOUS_VIEW_PERMISSION = ast.literal_eval(os.getenv("DEFAULT_ANONYMOUS_VIEW_PERMISSION", "True")) +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION = ast.literal_eval(os.getenv("DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION", "True")) + +EDITORS_CAN_MANAGE_ANONYMOUS_PERMISSIONS = ast.literal_eval( + os.getenv("EDITORS_CAN_MANAGE_ANONYMOUS_PERMISSIONS", "True") +) +EDITORS_CAN_MANAGE_REGISTERED_MEMBERS_PERMISSIONS = ast.literal_eval( + os.getenv("EDITORS_CAN_MANAGE_REGISTERED_MEMBERS_PERMISSIONS", "True") +) + +PERMISSIONS_HANDLERS = [ + "geonode.security.handlers.GroupManagersPermissionsHandler", + "geonode.security.handlers.SpecialGroupsPermissionsHandler", + "geonode.security.handlers.AdvancedWorkflowPermissionsHandler", +] + +# ########################################################################### # +# END SECURITY SETTINGS +# ########################################################################### # + # A boolean which specifies wether to display the email in user's profile SHOW_PROFILE_EMAIL = ast.literal_eval(os.environ.get("SHOW_PROFILE_EMAIL", "False")) -# Enables cross origin requests for geonode-client -MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS = ast.literal_eval( - os.getenv("MAP_CLIENT_USE_CROSS_ORIGIN_CREDENTIALS", "False") -) +# gravatar settings +AUTO_GENERATE_AVATAR_SIZES = (20, 30, 32, 40, 50, 65, 70, 80, 100, 140, 200, 240) +AVATAR_GRAVATAR_SSL = ast.literal_eval(os.getenv("AVATAR_GRAVATAR_SSL", "False")) + +AVATAR_DEFAULT_URL = os.getenv("AVATAR_DEFAULT_URL", "/geonode/img/avatar.png") + +# Django-Avatar - Change default templates to Geonode based +AVATAR_ADD_TEMPLATE = "people/avatar/add.html" +AVATAR_CHANGE_TEMPLATE = "people/avatar/change.html" +AVATAR_DELETE_TEMPLATE = "people/avatar/confirm_delete.html" + +# Group default logo url +GROUP_LOGO_URL = os.getenv("GROUP_LOGO_URL", "/geonode/img/group_logo.png") + +try: + # try to parse python notation, default in dockerized env + AVATAR_PROVIDERS = ast.literal_eval(os.getenv("AVATAR_PROVIDERS")) +except (ValueError, SyntaxError): + # fallback to regular list of values separated with misc chars + AVATAR_PROVIDERS = ( + ( + "avatar.providers.PrimaryAvatarProvider", + "avatar.providers.DefaultAvatarProvider", + ) + if os.getenv("AVATAR_PROVIDERS") is None + else re.split(r" *[,|:;] *", os.getenv("AVATAR_PROVIDERS")) + ) ACCOUNT_OPEN_SIGNUP = ast.literal_eval(os.environ.get("ACCOUNT_OPEN_SIGNUP", "True")) # ref https://github.com/GeoNode/geonode/issues/12967 @@ -1942,6 +1917,10 @@ def get_geonode_catalogue_service(): ACCOUNT_EMAIL_REQUIRED = ast.literal_eval(os.environ.get("ACCOUNT_EMAIL_REQUIRED", "True")) ACCOUNT_EMAIL_VERIFICATION = os.environ.get("ACCOUNT_EMAIL_VERIFICATION", "none") +# Invitation Adapter +INVITATIONS_ADAPTER = ACCOUNT_ADAPTER +INVITATIONS_CONFIRMATION_URL_NAME = "geonode.invitations:accept-invite" + # Since django-allauth 0.43.0. ACCOUNT_SIGNUP_REDIRECT_URL = os.environ.get("ACCOUNT_SIGNUP_REDIRECT_URL", os.getenv("SITEURL", _default_siteurl)) ACCOUNT_RATE_LIMITS = {"login_failed": os.getenv("ACCOUNT_LOGIN_ATTEMPTS_LIMIT", "10/m/ip,5/5m/key")} @@ -2023,9 +2002,17 @@ def get_geonode_catalogue_service(): SOCIALACCOUNT_OIDC_PROVIDER: SOCIALACCOUNT_PROVIDERS_DEFS.get(_SOCIALACCOUNT_PROVIDER), } -# Invitation Adapter -INVITATIONS_ADAPTER = ACCOUNT_ADAPTER -INVITATIONS_CONFIRMATION_URL_NAME = "geonode.invitations:accept-invite" +DISPLAY_RATINGS = ast.literal_eval(os.getenv("DISPLAY_RATINGS", "True")) +DISPLAY_WMS_LINKS = ast.literal_eval(os.getenv("DISPLAY_WMS_LINKS", "True")) + +# Setting TWITTER_CARD to True will enable Twitter Cards +# https://dev.twitter.com/cards/getting-started +# Be sure to replace @GeoNode with your organization or site's twitter handle. +TWITTER_CARD = ast.literal_eval(os.getenv("TWITTER_CARD", "True")) +TWITTER_SITE = "@GeoNode" +TWITTER_HASHTAGS = ["geonode"] + +OPENGRAPH_ENABLED = ast.literal_eval(os.getenv("OPENGRAPH_ENABLED", "True")) # Choose thumbnail generator -- this is the default generator THUMBNAIL_GENERATOR = os.environ.get("THUMBNAIL_GENERATOR", "geonode.thumbs.thumbnails.create_gs_thumbnail_geonode") @@ -2054,75 +2041,23 @@ def get_geonode_catalogue_service(): # }, } -# define the urls after the settings are overridden -if USE_GEOSERVER: - LOCAL_GXP_PTYPE = "gxp_wmscsource" - PUBLIC_GEOSERVER = { - "source": { - "title": "GeoServer - Public Layers", - "attribution": f"© {SITEURL}", - "ptype": LOCAL_GXP_PTYPE, - "url": f"{OGC_SERVER['default']['PUBLIC_LOCATION']}ows", - "restUrl": "/gs/rest", - } - } - LOCAL_GEOSERVER = { - "source": { - "title": "GeoServer - Private Layers", - "attribution": f"© {SITEURL}", - "ptype": LOCAL_GXP_PTYPE, - "url": "/gs/ows", - "restUrl": "/gs/rest", - } - } - baselayers = MAP_BASELAYERS - MAP_BASELAYERS = [PUBLIC_GEOSERVER] - MAP_BASELAYERS.extend(baselayers) - -CATALOG_METADATA_TEMPLATE = os.getenv("CATALOG_METADATA_TEMPLATE", "catalogue/full_metadata.xml") - -DEFAULT_AUTO_FIELD = "django.db.models.AutoField" -UI_DEFAULT_MANDATORY_FIELDS = [ - "id_resource-title", - "id_resource-abstract", - "id_resource-language", - "id_resource-license", - "id_resource-regions", - "id_resource-date_type", - "id_resource-date", - "category_form", - "id_resource-attribution", - "id_resource-constraints_other", - "id_resource-data_quality_statement", - "id_resource-restriction_code_type", -] -UI_REQUIRED_FIELDS = ast.literal_eval(os.getenv("UI_REQUIRED_FIELDS ", "[]")) +# Keywords thesauri +# e.g. THESAURUS = {'name':'inspire_themes', 'required':True, 'filter':True} +# Required: (boolean, optional, default false) mandatory while editing metadata (not implemented yet) +# Filter: (boolean, optional, default false) a filter option on that thesaurus will appear in the main search page +# THESAURUS = {'name': 'inspire_themes', 'required': True, 'filter': True} -# If a command name is listed here, the command will be available to admins over http -# This list is used by the management_commands_http app -MANAGEMENT_COMMANDS_EXPOSED_OVER_HTTP = set( - [ - "ping_mngmt_commands_http", - "updatelayers", - "sync_geonode_datasets", - "sync_geonode_maps", - "importlayers", - "set_all_datasets_metadata", - "set_layers_permissions", - ] - + ast.literal_eval(os.getenv("MANAGEMENT_COMMANDS_EXPOSED_OVER_HTTP ", "[]")) -) +# Topic Categories list should not be modified (they are ISO). In case you +# absolutely need it set to True this variable +MODIFY_TOPICCATEGORY = ast.literal_eval(os.getenv("MODIFY_TOPICCATEGORY", "True")) +# If this option is enabled, Topic Categories will become strictly Mandatory on +# Metadata Wizard +TOPICCATEGORY_MANDATORY = ast.literal_eval(os.environ.get("TOPICCATEGORY_MANDATORY", "False")) -FILE_UPLOAD_HANDLERS = [ - "geonode.upload.uploadhandler.SizeRestrictedFileUploadHandler", - "django.core.files.uploadhandler.TemporaryFileUploadHandler", - "django.core.files.uploadhandler.MemoryFileUploadHandler", -] +CATALOG_METADATA_TEMPLATE = os.getenv("CATALOG_METADATA_TEMPLATE", "catalogue/full_metadata.xml") -DEFAULT_MAX_UPLOAD_SIZE = 104857600 # 100 MB -DEFAULT_BUFFER_CHUNK_SIZE = int(os.getenv("DEFAULT_BUFFER_CHUNK_SIZE", 64 * 1024)) -DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER = int(os.getenv("DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER", 5)) +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" """ Default schema used to store extra and dynamic metadata for the resource @@ -2136,9 +2071,6 @@ def get_geonode_catalogue_service(): "field_value": object, } -GEOIP_PATH = os.getenv("GEOIP_PATH", os.path.join(PROJECT_ROOT, "GeoIPCities.dat")) -# This controls if tastypie search on resourches is performed only with titles -SEARCH_RESOURCES_EXTENDED = ast.literal_eval(os.getenv("SEARCH_RESOURCES_EXTENDED", "True")) """ If present, will extend the available metadata schema used for store new value for each resource. By default overrided the existing one. @@ -2167,6 +2099,18 @@ def get_geonode_catalogue_service(): # 'geonode.resource.regions_storer.spatial_predicate_region_assignor', ] +FILE_UPLOAD_HANDLERS = [ + "geonode.upload.uploadhandler.SizeRestrictedFileUploadHandler", + "django.core.files.uploadhandler.TemporaryFileUploadHandler", + "django.core.files.uploadhandler.MemoryFileUploadHandler", +] + +DEFAULT_MAX_UPLOAD_SIZE = 104857600 # 100 MB +DEFAULT_BUFFER_CHUNK_SIZE = int(os.getenv("DEFAULT_BUFFER_CHUNK_SIZE", 64 * 1024)) +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER = int(os.getenv("DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER", 5)) + +# This controls if tastypie search on resourches is performed only with titles +SEARCH_RESOURCES_EXTENDED = ast.literal_eval(os.getenv("SEARCH_RESOURCES_EXTENDED", "True")) """ List of modules that implement the deletion rules for a user @@ -2194,66 +2138,11 @@ def get_geonode_catalogue_service(): "geonode.upload.handlers", ) -CELERY_TASK_QUEUES += ( - Queue("geonode.upload.import_orchestrator", GEONODE_EXCHANGE, routing_key="geonode.upload.import_orchestrator"), - Queue( - "geonode.upload.import_resource", GEONODE_EXCHANGE, routing_key="geonode.upload.import_resource", max_priority=8 - ), - Queue( - "geonode.upload.publish_resource", - GEONODE_EXCHANGE, - routing_key="geonode.upload.publish_resource", - max_priority=8, - ), - Queue( - "geonode.upload.create_geonode_resource", - GEONODE_EXCHANGE, - routing_key="geonode.upload.create_geonode_resource", - max_priority=8, - ), - Queue( - "geonode.upload.import_with_ogr2ogr", - GEONODE_EXCHANGE, - routing_key="geonode.upload.import_with_ogr2ogr", - max_priority=10, - ), - Queue( - "geonode.upload.import_next_step", - GEONODE_EXCHANGE, - routing_key="geonode.upload.import_next_step", - max_priority=3, - ), - Queue( - "geonode.upload.create_dynamic_structure", - GEONODE_EXCHANGE, - routing_key="geonode.upload.create_dynamic_structure", - max_priority=10, - ), - Queue( - "geonode.upload.copy_geonode_resource", - GEONODE_EXCHANGE, - routing_key="geonode.upload.copy_geonode_resource", - max_priority=0, - ), - Queue("geonode.upload.copy_dynamic_model", GEONODE_EXCHANGE, routing_key="geonode.upload.copy_dynamic_model"), - Queue( - "geonode.upload.copy_geonode_data_table", GEONODE_EXCHANGE, routing_key="geonode.upload.copy_geonode_data_table" - ), - Queue("geonode.upload.copy_raster_file", GEONODE_EXCHANGE, routing_key="geonode.upload.copy_raster_file"), - Queue("geonode.upload.rollback", GEONODE_EXCHANGE, routing_key="geonode.upload.rollback"), - Queue("geonode.upload.upsert_data", GEONODE_EXCHANGE, routing_key="geonode.upload.upsert_data"), - Queue( - "geonode.upload.refresh_geonode_resource", - GEONODE_EXCHANGE, - routing_key="geonode.upload.refresh_geonode_resource", - ), -) - DATABASE_ROUTERS = ["geonode.upload.db_router.DatastoreRouter"] IMPORTER_HANDLERS = ast.literal_eval(os.getenv("IMPORTER_HANDLERS", "[]")) -IMPORTER_ENABLE_DYN_MODELS = ast.literal_eval(os.getenv("IMPORTER_ENABLE_DYN_MODELS", "True")) +IMPORTER_ENABLE_DYN_MODELS = True INSTALLED_APPS += ("geonode.facets",) GEONODE_APPS += ("geonode.facets",) @@ -2273,10 +2162,6 @@ def get_geonode_catalogue_service(): DATASET_DOWNLOAD_HANDLERS = ast.literal_eval(os.getenv("DATASET_DOWNLOAD_HANDLERS", "[]")) -AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS = ast.literal_eval( - os.getenv("AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS", "True") -) - DEFAULT_ASSET_HANDLER = "geonode.assets.local.LocalAssetHandler" ASSET_HANDLERS = [ DEFAULT_ASSET_HANDLER, @@ -2284,34 +2169,13 @@ def get_geonode_catalogue_service(): INSTALLED_APPS += ("geonode.assets",) GEONODE_APPS += ("geonode.assets",) -PERMISSIONS_HANDLERS = [ - "geonode.security.handlers.GroupManagersPermissionsHandler", - "geonode.security.handlers.SpecialGroupsPermissionsHandler", - "geonode.security.handlers.AdvancedWorkflowPermissionsHandler", -] - FEATURE_VALIDATORS = [ "geonode.upload.feature_validators.GeoserverFeatureValidator", ] -# Django-Avatar - Change default templates to Geonode based -AVATAR_ADD_TEMPLATE = "people/avatar/add.html" -AVATAR_CHANGE_TEMPLATE = "people/avatar/change.html" -AVATAR_DELETE_TEMPLATE = "people/avatar/confirm_delete.html" - -# Group default logo url -GROUP_LOGO_URL = os.getenv("GROUP_LOGO_URL", "/geonode/img/group_logo.png") - UPSERT_CHUNK_SIZE = ast.literal_eval(os.getenv("UPSERT_CHUNK_SIZE", "1000")) UPSERT_LIMIT_ERROR_LOG = ast.literal_eval(os.getenv("UPSERT_LIMIT_ERROR_LOG", "1000")) UPSERT_LOG_LOCATION = os.getenv("UPSERT_LOG_LOCATION", "/tmp") FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o777 FILE_UPLOAD_PERMISSIONS = 0o777 - -EDITORS_CAN_MANAGE_ANONYMOUS_PERMISSIONS = ast.literal_eval( - os.getenv("EDITORS_CAN_MANAGE_ANONYMOUS_PERMISSIONS", "True") -) -EDITORS_CAN_MANAGE_REGISTERED_MEMBERS_PERMISSIONS = ast.literal_eval( - os.getenv("EDITORS_CAN_MANAGE_REGISTERED_MEMBERS_PERMISSIONS", "True") -) diff --git a/geonode/upload/api/tests_old.py b/geonode/upload/api/tests_old.py index bada533c0ea..9cf4ae3584b 100644 --- a/geonode/upload/api/tests_old.py +++ b/geonode/upload/api/tests_old.py @@ -16,38 +16,14 @@ # along with this program. If not, see . # ######################################################################### - -from geonode.base.models import ResourceBase -from geonode.geoserver.helpers import gs_catalog import os -import shutil import logging -import tempfile -from io import IOBase -from urllib.request import urljoin - -from django.conf import settings from django.urls import reverse -from django.contrib.auth import authenticate, get_user_model -from django.test.utils import override_settings - -from requests_toolbelt.multipart.encoder import MultipartEncoder +from django.contrib.auth import get_user_model from rest_framework.test import APITestCase -from seleniumrequests import Firefox - -# from selenium.common import exceptions -# from selenium.webdriver.common.by import By -from selenium.webdriver import FirefoxOptions -from selenium.webdriver.support.wait import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.firefox.firefox_binary import FirefoxBinary - -from webdriver_manager.firefox import GeckoDriverManager - -from geonode.tests.base import GeoNodeLiveTestSupport from geonode.geoserver.helpers import ogc_server_settings from geonode.upload.models import UploadSizeLimit, UploadParallelismLimit @@ -60,180 +36,6 @@ logger = logging.getLogger("importer") -@override_settings( - DEBUG=True, - ALLOWED_HOSTS=["*"], - SITEURL=LIVE_SERVER_URL, - CSRF_COOKIE_SECURE=False, - CSRF_COOKIE_HTTPONLY=False, - CORS_ORIGIN_ALLOW_ALL=True, - SESSION_COOKIE_SECURE=False, - DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5, -) -class UploadApiTests(GeoNodeLiveTestSupport, APITestCase): - port = 0 - - @classmethod - def setUpClass(cls): - super().setUpClass() - - try: - """Instantiate selenium driver instance""" - binary = FirefoxBinary("/usr/bin/firefox") - opts = FirefoxOptions() - opts.add_argument("--headless") - executable_path = GeckoDriverManager().install() - cls.selenium = Firefox(firefox_binary=binary, firefox_options=opts, executable_path=executable_path) - cls.selenium.implicitly_wait(10) - except Exception as e: - logger.error(e) - - @classmethod - def tearDownClass(cls): - """Quit selenium driver instance""" - try: - cls.selenium.quit() - except Exception as e: - logger.debug(e) - super().tearDownClass() - - def setUp(self): - super().setUp() - self.temp_folder = tempfile.mkdtemp(dir=CURRENT_LOCATION) - self.session_id = None - self.csrf_token = None - - def tearDown(self): - shutil.rmtree(self.temp_folder, ignore_errors=True) - return super().tearDown() - - def set_session_cookies(self, url=None): - # selenium will set cookie domain based on current page domain - self.selenium.get(url or f"{self.live_server_url}/") - self.csrf_token = self.selenium.get_cookie("csrftoken")["value"] - self.session_id = self.selenium.get_cookie(settings.SESSION_COOKIE_NAME)["value"] - self.selenium.add_cookie( - {"name": settings.SESSION_COOKIE_NAME, "value": self.session_id, "secure": False, "path": "/"} - ) - self.selenium.add_cookie({"name": "csrftoken", "value": self.csrf_token, "secure": False, "path": "/"}) - - def click_button(self, label): - selector = f"//button[contains(., '{label}')]" - self.selenium.find_element_by_xpath(selector).click() - - def do_login(self, username="admin", password="admin"): - """Method to login the GeoNode site""" - assert authenticate(username=username, password=password) - self.assertTrue(self.client.login(username=username, password=password)) # Native django test client - - url = urljoin(settings.SITEURL, f"{reverse('account_login')}?next=/layers") - self.set_session_cookies(url) - self.selenium.save_screenshot(os.path.join(self.temp_folder, "login.png")) - - title = self.selenium.title - current_url = self.selenium.current_url - logger.debug(f" ---- title: {title} / current_url: {current_url}") - - username_input = self.selenium.find_element_by_xpath('//input[@id="id_login"][@type="text"]') - username_input.send_keys(username) - password_input = self.selenium.find_element_by_xpath('//input[@id="id_password"][@type="password"]') - password_input.send_keys(password) - - self.selenium.save_screenshot(os.path.join(self.temp_folder, "login-set_fields.png")) - self.click_button("Sign In") - self.selenium.save_screenshot(os.path.join(self.temp_folder, "login-sign_in.png")) - - title = self.selenium.title - current_url = self.selenium.current_url - logger.debug(f" ---- title: {title} / current_url: {current_url}") - - # Wait until the response is received - WebDriverWait(self.selenium, 10).until(EC.title_contains("Explore Layers")) - self.set_session_cookies(url) - - def do_logout(self): - url = urljoin(settings.SITEURL, f"{reverse('account_logout')}") - self.selenium.get(url) - self.click_button("Log out") - - def do_upload_step(self, step=None): - step = urljoin(settings.SITEURL, reverse("data_upload", args=[step] if step else [])) - return step - - def live_upload_file(self, _file): - """function that uploads a file, or a collection of files, to - the GeoNode""" - spatial_files = ("dbf_file", "shx_file", "prj_file") - base, ext = os.path.splitext(_file) - params = { - # make public since wms client doesn't do authentication - "csrfmiddlewaretoken": self.csrf_token, - "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}', - "time": "false", - "charset": "UTF-8", - } - cookies = {settings.SESSION_COOKIE_NAME: self.session_id, "csrftoken": self.csrf_token} - headers = { - "X-CSRFToken": self.csrf_token, - "X-Requested-With": "XMLHttpRequest", - "Set-Cookie": f"csrftoken={self.csrf_token}; sessionid={self.session_id}", - } - url = self.do_upload_step() - logger.debug(f" ---- UPLOAD URL: {url} / cookies: {cookies} / headers: {headers}") - - # deal with shapefiles - if ext.lower() == ".shp": - for spatial_file in spatial_files: - ext, _ = spatial_file.split("_") - file_path = f"{base}.{ext}" - # sometimes a shapefile is missing an extra file, - # allow for that - if os.path.exists(file_path): - params[spatial_file] = open(file_path, "rb") - - with open(_file, "rb") as base_file: - params["base_file"] = base_file - for name, value in params.items(): - if isinstance(value, IOBase): - params[name] = (os.path.basename(value.name), value) - - # refresh to exchange cookies with the server. - self.selenium.refresh() - self.selenium.get(url) - self.selenium.save_screenshot(os.path.join(self.temp_folder, "upload-page.png")) - logger.debug(f" ------------ UPLOAD FORM: {params}") - encoder = MultipartEncoder(fields=params) - headers["Content-Type"] = encoder.content_type - response = self.selenium.request("POST", url, data=encoder, headers=headers) - - # Closes the files - for spatial_file in spatial_files: - if isinstance(params.get(spatial_file), IOBase): - params[spatial_file].close() - - try: - logger.error(f" -- response: {response.status_code} / {response.json()}") - return response, response.json() - except ValueError: - logger.exception(ValueError(f"probably not json, status {response.status_code} / {response.content}")) - return response, response.content - - def _cleanup_layer(self, layer_name): - # removing the layer from geonode - x = ResourceBase.objects.filter(alternate__icontains=layer_name) - if x.exists(): - for el in x.iterator(): - el.delete() - # removing the layer from geoserver - dataset = gs_catalog.get_layer(layer_name) - if dataset: - gs_catalog.delete(dataset, purge="all", recurse=True) - # removing the layer from geoserver - store = gs_catalog.get_store(layer_name, workspace="geonode") - if store: - gs_catalog.delete(store, purge="all", recurse=True) - - class UploadSizeLimitTests(APITestCase): fixtures = [ "group_test_data.json", diff --git a/geonode/upload/celery_tasks.py b/geonode/upload/celery_tasks.py index ac7fc7970ed..daa433020ad 100644 --- a/geonode/upload/celery_tasks.py +++ b/geonode/upload/celery_tasks.py @@ -62,6 +62,7 @@ find_key_recursively, ImporterRequestAction as ira, ) +from geonode.upload.handlers.base import BaseHandler logger = logging.getLogger("importer") @@ -533,6 +534,9 @@ def create_geonode_resource( else: handler.create_resourcehandlerinfo(handler_module_path, resource, _exec, **kwargs) + if _overwrite and handler.have_table: + handler.fixup_dynamic_model_fields(_exec, _files, resource=resource) + # at the end recall the import_orchestrator for the next step import_orchestrator.apply_async( ( @@ -714,6 +718,9 @@ def _create_field(dynamic_model_schema, field, _kwargs): dynamic_model_schema = dynamic_model_schema.first() + # clearing existing fields for this chunk + FieldSchema.objects.filter(model_schema=dynamic_model_schema, name__in=(x["name"] for x in fields)).delete() + row_to_insert = [] for field in fields: # setup kwargs for the class provided @@ -733,20 +740,13 @@ def _create_field(dynamic_model_schema, field, _kwargs): # setting the dimension for the gemetry. So that we can handle also 3d geometries _kwargs = {**_kwargs, **{"dim": field.get("dim")}} + if authority := field.get("authority"): + srid_str = authority.split(":")[-1] + if srid_str.isdigit(): + _kwargs["srid"] = int(srid_str) + # if is a new creation we generate the field model from scratch - if not overwrite: - row_to_insert.append(_create_field(dynamic_model_schema, field, _kwargs)) - else: - # otherwise if is an overwrite, we update the existing one and create the one that does not exists - _field_exists = FieldSchema.objects.filter(name=field["name"], model_schema=dynamic_model_schema) - if _field_exists.exists(): - _field_exists.update( - class_name=field["class_name"], - model_schema=dynamic_model_schema, - kwargs=_kwargs, - ) - else: - row_to_insert.append(_create_field(dynamic_model_schema, field, _kwargs)) + row_to_insert.append(_create_field(dynamic_model_schema, field, _kwargs)) if row_to_insert: if dynamic_model_schema.managed: @@ -765,7 +765,12 @@ def _create_field(dynamic_model_schema, field, _kwargs): field.save() else: # the build creation improves the overall permformance with the DB - FieldSchema.objects.bulk_create(row_to_insert, 30) + FieldSchema.objects.bulk_create( + row_to_insert, + update_conflicts=True, + update_fields=["name", "model_schema_id", "class_name", "kwargs"], + unique_fields=["id"], + ) # fixing the schema model in django return "dynamic_model", layer_name, execution_id @@ -807,7 +812,8 @@ def copy_dynamic_model(self, exec_id, actual_step, layer_name, alternate, handle resource = resource.first() - new_dataset_alternate = create_alternate(resource.title, exec_id).lower() + sanitized_title = BaseHandler().fixup_name(resource.title) + new_dataset_alternate = create_alternate(sanitized_title, exec_id) if settings.IMPORTER_ENABLE_DYN_MODELS: dynamic_schema = ModelSchema.objects.filter(name=alternate.split(":")[1]) diff --git a/geonode/upload/handlers/common/raster.py b/geonode/upload/handlers/common/raster.py index 7c36ae47592..798bc378565 100644 --- a/geonode/upload/handlers/common/raster.py +++ b/geonode/upload/handlers/common/raster.py @@ -569,6 +569,11 @@ def _publish_resource_rollback(self, exec_id, instance_name=None, *args, **kwarg publisher = DataPublisher(handler_module_path=handler_module_path) publisher.delete_resource(instance_name) + def fixup_dynamic_model_fields(self, _exec, files, **kwargs): + """ + Raster dataset does not have the dynamic model, so this can be skept + """ + @importer_app.task( base=UpdateTaskClass, @@ -609,7 +614,8 @@ def copy_raster_file(exec_id, actual_step, layer_name, alternate, handler_module if not new_file_location["files"]: raise InvalidGeoTiffException("Could not determine the location of the copied file") - new_dataset_alternate = create_alternate(original_dataset.title, exec_id) + sanitized_title = BaseHandler().fixup_name(original_dataset.title) + new_dataset_alternate = create_alternate(sanitized_title, exec_id) additional_kwargs = { "original_dataset_alternate": original_dataset.alternate, diff --git a/geonode/upload/handlers/common/remote.py b/geonode/upload/handlers/common/remote.py index 36699f20bb1..f0439c7cada 100755 --- a/geonode/upload/handlers/common/remote.py +++ b/geonode/upload/handlers/common/remote.py @@ -219,6 +219,9 @@ def create_geonode_resource( resource_type=resource_type, defaults=self.generate_resource_payload(layer_name, alternate, asset, _exec, None, **params), ) + + # The thumbnail is not created for the following data types: "3dtiles", "cog", "flatgeobuf" + # because of the can_have_thumbnail property resource_manager.set_thumbnail(None, instance=resource) resource = self.create_link(resource, params, alternate) diff --git a/geonode/upload/handlers/common/tests_vector.py b/geonode/upload/handlers/common/tests_vector.py index bf6209439cd..9e3c4389f50 100644 --- a/geonode/upload/handlers/common/tests_vector.py +++ b/geonode/upload/handlers/common/tests_vector.py @@ -353,29 +353,34 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command(self, self.assertEqual(str(_uuid), execution_id) _datastore = settings.DATABASES["datastore"] + + # Build the expected list to match the "Actual" output in your trace + expected_cmd_list = [ + "/usr/bin/ogr2ogr", + "--config", + "PG_USE_COPY", + "YES", + "-f", + "PostgreSQL", + f"PG: dbname='{_datastore['NAME']}' host={os.getenv('DATABASE_HOST', 'localhost')} port=5432 user='{_datastore['USER']}' password='{_datastore['PASSWORD']}' ", + self.valid_files.get("base_file"), + "-lco", + "FID=fid", + "-nln", + "alternate", + "dataset", + ] + _open.assert_called_once() - _open.assert_called_with( - "/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" - + os.getenv("DATABASE_HOST", "localhost") - + " port=5432 user='" - + _datastore["USER"] - + "' password='" - + _datastore["PASSWORD"] - + '\' " "' - + self.valid_files.get("base_file") - + '" -lco FID=fid' - + ' -nln alternate "dataset"', - stdout=-1, - stderr=-1, - shell=True, # noqa - ) + _open.assert_called_with(expected_cmd_list, stdout=-1, stderr=-1, shell=False) # Changed from True to False @patch("geonode.upload.handlers.common.vector.Popen") def test_import_with_ogr2ogr_with_errors_should_raise_exception(self, _open): _uuid = uuid.uuid4() comm = MagicMock() - comm.communicate.return_value = b"", b"ERROR: some error here" + # Mocking the communicate to return an error in stderr + comm.communicate.return_value = (b"", b"ERROR: some error here") _open.return_value = comm with self.assertRaises(Exception): @@ -388,30 +393,31 @@ def test_import_with_ogr2ogr_with_errors_should_raise_exception(self, _open): alternate="alternate", ) - _datastore = settings.DATABASES["datastore"] - + # Verification of the NEW secure call _open.assert_called_once() - _open.assert_called_with( - "/usr/bin/ogr2ogr --config PG_USE_COPY YES -f PostgreSQL PG:\" dbname='test_geonode_data' host=" - + os.getenv("DATABASE_HOST", "localhost") - + " port=5432 user='" - + _datastore["USER"] - + "' password='" - + _datastore["PASSWORD"] - + '\' " "' - + self.valid_files.get("base_file") - + '" -lco FID=fid' - + ' -nln alternate "dataset"', - stdout=-1, - stderr=-1, - shell=True, # noqa - ) - @patch.dict(os.environ, {"OGR2OGR_COPY_WITH_DUMP": "True"}, clear=True) + # Get the actual call arguments to inspect them + args, kwargs = _open.call_args + cmd_list = args[0] + + # Check that it's a list (not a string) + self.assertIsInstance(cmd_list, list) + + # Check for specific safe flags in the list + self.assertIn("--config", cmd_list) + self.assertIn("PG_USE_COPY", cmd_list) + self.assertIn("alternate", cmd_list) + self.assertIn("dataset", cmd_list) + + # Verify shell=False is now the standard (either explicitly False or not present) + self.assertFalse(kwargs.get("shell", False)) + + @patch.dict(os.environ, {"OGR2OGR_COPY_WITH_DUMP": "True"}) @patch("geonode.upload.handlers.common.vector.Popen") def test_import_with_ogr2ogr_without_errors_should_call_the_right_command_if_dump_is_enabled(self, _open): _uuid = uuid.uuid4() + # We need the second process (psql) to return the communicate values comm = MagicMock() comm.communicate.return_value = b"", b"" _open.return_value = comm @@ -429,12 +435,28 @@ def test_import_with_ogr2ogr_without_errors_should_call_the_right_command_if_dum self.assertEqual(alternate, "alternate") self.assertEqual(str(_uuid), execution_id) - _open.assert_called_once() - _call_as_string = _open.mock_calls[0][1][0] + # 1. Verify Popen was called twice (once for ogr2ogr, once for psql) + self.assertEqual(_open.call_count, 2) + + # 2. Check the first call (ogr2ogr) + # mock_calls[0] = ((args), {kwargs}) + ogr_args = _open.mock_calls[0][1][0] + self.assertIn("-f", ogr_args) + self.assertIn("PGDump", ogr_args) + self.assertIn("/vsistdout/", ogr_args) + self.assertNotIn("PostgreSQL", ogr_args) + + # 3. Check the second call (psql) + psql_args = _open.mock_calls[1][1][0] + psql_kwargs = _open.mock_calls[1][2] - self.assertTrue("-f PGDump /vsistdout/" in _call_as_string) - self.assertTrue("psql -d" in _call_as_string) - self.assertFalse("-f PostgreSQL PG" in _call_as_string) + self.assertIn("psql", psql_args) + self.assertIn("-d", psql_args) + + # 4. Verify the password is passed securely in 'env', not in the command list + self.assertIn("PGPASSWORD", psql_kwargs["env"]) + # Ensure the password is NOT in the actual command list (the security fix!) + self.assertFalse(any("PGPASSWORD" in str(arg) for arg in psql_args)) def test_select_valid_layers(self): """ @@ -497,34 +519,6 @@ def test_upsert_validation_should_fail(self): "The Dynamic model generation must be enabled to perform the upsert IMPORTER_ENABLE_DYN_MODELS=True", ) - @override_settings(IMPORTER_ENABLE_DYN_MODELS=True) - @patch("geonode.upload.handlers.common.vector.ModelSchema") - @patch("geonode.upload.handlers.common.vector.BaseVectorFileHandler.extract_upsert_key") - def test_upsert_data_should_fail_if_upsertkey_is_not_provided(self, upsert_function, schema): - """ - The test should fail since the upsert key provided is empty/Null and - was not possible to extract the key from the DB schema - """ - schema.return_value = MagicMock() - data = create_single_dataset("example_upsert_dataset") - exec_id = orchestrator.create_execution_request( - user=self.user, - func_name="funct1", - step="step", - input_params={"files": self.valid_files, "skip_existing_layer": True, "resource_pk": data.pk}, - ) - - upsert_function.return_value = None - handler = ShapeFileHandler() - with self.assertRaises(Exception) as exept: - handler.upsert_data(["files"], exec_id) - - self.assertIsNotNone(exept) - self.assertEqual( - str(exept.exception), - "Was not possible to find the upsert key, upsert is aborted", - ) - def test_get_error_file_csv_headers(self): handler = BaseVectorFileHandler() mock_validator = MagicMock() @@ -600,29 +594,6 @@ def test_upsert_data_without_dynamic_model_schema(self): "This dataset does't support updates. Please upload the dataset again to have the upsert operations enabled", ) - def test_upsert_data_raise_error_if_upsert_key_is_not_defined(self): - """ - Should raise error if the dynamic model schema is not present - """ - data = create_single_dataset("example_upsert_dataset") - exec_id = orchestrator.create_execution_request( - user=self.user, - func_name="funct1", - step="step", - input_params={ - "files": self.original, - "skip_existing_layer": True, - "resource_pk": data.pk, - "upsert_key": None, - }, - ) - ModelSchema.objects.create(name="example_upsert_dataset", db_name="datastore", managed=True) - - with self.assertRaises(UpsertException) as exp: - self.handler.upsert_data(self.original, exec_id) - - self.assertEqual(str(exp.exception), "Was not possible to find the upsert key, upsert is aborted") - def test_validate_single_feature_raise_error(self): """ Should raise error if the dynamic model schema is not present @@ -643,7 +614,6 @@ def test_validate_single_feature_raise_error(self): with self.assertRaises(Exception) as exp: self.json_handler.upsert_data(self.original, exec_id) - self.assertEqual( str(exp.exception), "An internal error occurred during upsert save. All features are rolled back." ) diff --git a/geonode/upload/handlers/common/vector.py b/geonode/upload/handlers/common/vector.py index ff8eb9efcef..fec6987d3ce 100644 --- a/geonode/upload/handlers/common/vector.py +++ b/geonode/upload/handlers/common/vector.py @@ -18,7 +18,7 @@ ######################################################################### import ast import csv - +import re from datetime import datetime from itertools import islice from pathlib import Path @@ -34,6 +34,7 @@ import os from subprocess import PIPE, Popen from typing import List, Optional, Tuple +import shlex from celery import chord, group from django.db import transaction @@ -258,6 +259,8 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate, ** This is a default command that is needed to import a vector file """ _datastore = settings.DATABASES["datastore"] + layers = ogr.Open(files.get("base_file")) + layer = layers.GetLayer(original_name) options = "--config PG_USE_COPY YES" copy_with_dump = ast.literal_eval(os.getenv("OGR2OGR_COPY_WITH_DUMP", "False")) @@ -277,9 +280,13 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate, ** # vrt file is aready created in import_resource and vrt will be auto detected by ogr2ogr # and also the base_file will work so can be used as alternative for fallback which will also be autodeteced by ogr2ogr. input_file = files.get("temp_vrt_file") or files.get("base_file") - options += f'"{input_file}"' + f" -lco FID={DEFAULT_PK_COLUMN_NAME} " - options += f'-nln {alternate} "{original_name}"' + options += f" {shlex.quote(input_file)} -lco FID={DEFAULT_PK_COLUMN_NAME} " + + options += f" -nln {shlex.quote(alternate)} {shlex.quote(original_name)}" + + if layer is not None and "Point" not in ogr.GeometryTypeToName(layer.GetGeomType()): + options += " -nlt PROMOTE_TO_MULTI" if ovverwrite_layer: options += " -overwrite" @@ -448,7 +455,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str: data inside the geonode_data database """ all_layers = self.get_ogr2ogr_driver().Open(files.get("base_file")) - layers = self._select_valid_layers(all_layers) + layers = self._select_valid_layers(all_layers, execution_id=execution_id) # for the moment we skip the dyanamic model creation layer_count = len(layers) logger.info(f"Total number of layers available: {layer_count}") @@ -549,7 +556,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str: raise e return layer_names, alternates, execution_id - def _select_valid_layers(self, all_layers): + def _select_valid_layers(self, all_layers, **kwargs): layers = [] for layer in all_layers: try: @@ -731,15 +738,28 @@ def _define_dynamic_layer_schema(self, layer, **kwargs): "authority": self.identify_authority(layer), } ] - + # if the layer comes from a DB, the fid column is not included in the schema, but we need to add it as primary key for the dynamic model + fid_in_schema = any(x["name"] == DEFAULT_PK_COLUMN_NAME for x in layer_schema) + if not fid_in_schema and layer.GetFIDColumn(): + layer_schema += [ + { + "name": layer.GetFIDColumn(), + "class_name": "django.db.models.BigAutoField", + "null": False, + "primary_key": True, + } + ] return layer_schema - def promote_to_multi(self, geometry_name: str): + def promote_to_multi(self, geometry_name): """ If needed change the name of the geometry, by promoting it to Multi example if is Point -> MultiPoint Needed for the shapefiles + Later this is used to map the geometry coming from ogr2ogr with a django class """ + if "Multi" not in geometry_name and "Point" not in geometry_name and "3D" not in geometry_name: + return f"Multi {geometry_name.title()}" return geometry_name def promote_geom_to_multi(self, geom): @@ -748,7 +768,21 @@ def promote_geom_to_multi(self, geom): example if is Point -> MultiPoint Needed for the shapefiles """ - return geom + match geom.GetGeometryType(): + case ogr.wkbMultiLineString | ogr.wkbMultiPolygon: + # if is already multi, we dont need to do anything + return geom + case ogr.wkbLineString: + new_multi_geom = ogr.Geometry(ogr.wkbMultiLineString) + new_multi_geom.AddGeometry(geom) + return new_multi_geom + case ogr.wkbPolygon: + new_multi_geom = ogr.Geometry(ogr.wkbMultiPolygon) + new_multi_geom.AddGeometry(geom) + return new_multi_geom + case _: + # we dont convert points + return geom def create_geonode_resource( self, @@ -796,37 +830,7 @@ def create_geonode_resource( saved_dataset.refresh_from_db() - # if dynamic model is enabled, we can save up with is the primary key of the table - if settings.IMPORTER_ENABLE_DYN_MODELS and self.have_table: - from django.db import connections - - # then we can check for the PK - column = None - connection = connections["datastore"] - table_name = saved_dataset.alternate.split(":")[1] - - schema = ModelSchema.objects.filter(name=table_name).first() - schema.managed = False - schema.save() - - with connection.cursor() as cursor: - column = connection.introspection.get_primary_key_columns(cursor, table_name) - if column: - # getting the relative model schema - # better to always ensure that the schema is NOT managed - field = FieldSchema.objects.filter(name=column[0], model_schema__name=table_name).first() - if field: - field.kwargs.update({"primary_key": True}) - field.save() - else: - # creating the field needed as primary key - pk_field = FieldSchema( - name=column[0], - model_schema=schema, - class_name="django.db.models.BigAutoField", - kwargs={"null": False, "primary_key": True}, - ) - pk_field.save() + self.__fixup_primary_key(saved_dataset) return saved_dataset @@ -1135,7 +1139,7 @@ def __get_new_and_original_schema(self, files, execution_id): # use ogr2ogr to read the uploaded files for the upsert all_layers = self.get_ogr2ogr_driver().Open(files.get("base_file")) - layers = self._select_valid_layers(all_layers) + layers = self._select_valid_layers(all_layers, execution_id=execution_id) if not layers: raise UpsertException("No valid layers found in the provided file for upsert.") @@ -1159,6 +1163,45 @@ def __get_new_and_original_schema(self, files, execution_id): return target_schema_fields, new_file_schema_fields + def __fixup_primary_key(self, saved_dataset): + + # if dynamic model is enabled, we can save up with is the primary key of the table + if settings.IMPORTER_ENABLE_DYN_MODELS and self.have_table: + from django.db import connections + + # then we can check for the PK + column = None + connection = connections["datastore"] + table_name = saved_dataset.alternate.split(":")[1] + + schema = ModelSchema.objects.filter(name=table_name).first() + if not schema: + logger.warning( + "No ModelSchema for %s.", + table_name, + ) + return + schema.managed = False + schema.save() + with connection.cursor() as cursor: + column = connection.introspection.get_primary_key_columns(cursor, table_name) + if column: + # getting the relative model schema + # better to always ensure that the schema is NOT managed + field = FieldSchema.objects.filter(name=column[0], model_schema__name=table_name).first() + if field: + field.kwargs.update({"primary_key": True}) + field.save() + else: + # creating the field needed as primary key + pk_field = FieldSchema( + name=column[0], + model_schema=schema, + class_name="django.db.models.BigAutoField", + kwargs={"null": False, "primary_key": True}, + ) + pk_field.save() + def upsert_data(self, files, execution_id, **kwargs): """ Function used to upsert the data for a vector resource. @@ -1175,9 +1218,7 @@ def upsert_data(self, files, execution_id, **kwargs): exec_obj = orchestrator.get_execution_object(execution_id) # getting the related model schema for the resource - original_resource = ResourceBase.objects.filter(pk=exec_obj.input_params.get("resource_pk")).first() - self.real_instance = original_resource.get_real_instance() - model = ModelSchema.objects.filter(name=original_resource.alternate.split(":")[-1]).first() + original_resource, model = self.___get_dynamic_schema(exec_obj) if not model: raise UpsertException( "This dataset does't support updates. Please upload the dataset again to have the upsert operations enabled" @@ -1186,20 +1227,20 @@ def upsert_data(self, files, execution_id, **kwargs): # get the rows that match the upsert key OriginalResource = model.as_model() - # retrieve the upsert key. - upsert_key = self.extract_upsert_key(exec_obj, dynamic_model_instance=model) - if not upsert_key: - # if for any reason the key is not present, better to raise an error - raise UpsertException("Was not possible to find the upsert key, upsert is aborted") # use ogr2ogr to read the uploaded files values for the upsert all_layers = self.get_ogr2ogr_driver().Open(files.get("base_file")) valid_create = 0 valid_update = 0 - layers = self._select_valid_layers(all_layers) + layers = self._select_valid_layers(all_layers, execution_id=execution_id) if not layers: raise UpsertException("No valid layers were found in the file provided") # we can upsert just 1 layer at time + upsert_key = self.extract_upsert_key(layers[0]) + if not upsert_key: + # if for any reason the key is not present, better to raise an error + raise UpsertException("Was not possible to find the upsert key, upsert is aborted") + self._validate_single_feature(exec_obj, OriginalResource, upsert_key, layers, iter(layers[0])) valid_create, valid_update = self._commit_upsert(model, OriginalResource, upsert_key, iter(layers[0])) @@ -1218,6 +1259,12 @@ def upsert_data(self, files, execution_id, **kwargs): "layer_name": original_resource.title, } + def ___get_dynamic_schema(self, exec_obj): + original_resource = ResourceBase.objects.filter(pk=exec_obj.input_params.get("resource_pk")).first() + self.real_instance = original_resource.get_real_instance() + model = ModelSchema.objects.filter(name=original_resource.alternate.split(":")[-1]).first() + return original_resource, model + def _commit_upsert(self, model_obj, OriginalResource, upsert_key, layer_iterator): valid_create = 0 valid_update = 0 @@ -1310,7 +1357,11 @@ def _validate_feature(self, data_chunk, model_instance, upsert_key, errors): # need to simulate the "promote to multi" used by the upload process. # here we cannot rely on ogr2ogr so we need to do it manually geom = feature.GetGeometryRef() - feature_as_dict.update({self.default_geometry_column_name: self.promote_geom_to_multi(geom).ExportToWkt()}) + if geom: + wkt = self.promote_geom_to_multi(geom).ExportToWkt() + if code := geom.GetSpatialReference().GetAuthorityCode(None): + wkt = f"SRID={code};{wkt}" + feature_as_dict.update({self.default_geometry_column_name: wkt}) feature_as_dict, is_valid = self.validate_feature(feature_as_dict) if not is_valid: @@ -1322,18 +1373,40 @@ def _validate_feature(self, data_chunk, model_instance, upsert_key, errors): def _save_feature(self, data_chunk, model_obj, model_instance, upsert_key, valid_update, valid_create): # getting all the upsert_key value from the data chunk # retrieving the data from the DB - value_in_db = model_instance.objects.filter( - **{f"{upsert_key}__in": (getattr(feature, upsert_key) for feature in data_chunk)} - ).in_bulk(field_name=upsert_key) + use_get_fid = False # flag to understand if we need to use the GetFID as upsert key, this is needed for DB drivers with FID columns that hide the FID field from the schema + filters = [] + for feature in data_chunk: + # DB drivers with FID columns hide the FID field from the schema, so we need to check if the FID is present and use it as upsert key if the upsert key is the default one + if not getattr(feature, upsert_key, None) and feature.GetFID() != ogr.NullFID: + filters.append(feature.GetFID()) + use_get_fid = True + else: + filters.append(getattr(feature, upsert_key)) + value_in_db = model_instance.objects.filter(**{f"{upsert_key}__in": filters}).in_bulk(field_name=upsert_key) # looping over the chunk data to_process = [] feature_to_save = [] for feature in data_chunk: - feature_as_dict = feature.items() + feature_as_dict = {self.fixup_name(key): value for key, value in feature.items().items()} + # evaluate if there is any date in the schema of the feature + schema = feature.DumpReadableAsString().split("\n") + if any(date_fields := [f for f in schema if ("(Date)" in f or "(DateTime)" in f) and "(null)" not in f]): + # if any field schema as date is found, we can normalize the date + pattern = re.compile(r"^\s*(?P