diff --git a/app/src/main/cpp/AudioBeacon.cpp b/app/src/main/cpp/AudioBeacon.cpp index 275713622..ae3ba8f46 100644 --- a/app/src/main/cpp/AudioBeacon.cpp +++ b/app/src/main/cpp/AudioBeacon.cpp @@ -17,6 +17,8 @@ PositionedAudio::PositionedAudio(AudioEngine *engine, m_Eof(false), m_Dimmable(dimmable) { + static std::atomic s_nextHandle{1}; + m_Handle = s_nextHandle.fetch_add(1); m_pEngine = engine; m_UtteranceId = std::move(utterance_id); } diff --git a/app/src/main/cpp/AudioBeacon.h b/app/src/main/cpp/AudioBeacon.h index 520b136eb..8f79285f5 100644 --- a/app/src/main/cpp/AudioBeacon.h +++ b/app/src/main/cpp/AudioBeacon.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include "AudioBeaconBuffer.h" @@ -47,6 +48,7 @@ namespace soundscape { AudioEngine *m_pEngine; std::string m_UtteranceId; + uint64_t m_Handle; protected: void Init(double degrees_off_axis, diff --git a/app/src/main/cpp/AudioEngine.cpp b/app/src/main/cpp/AudioEngine.cpp index 079dda6ce..1abf923a5 100644 --- a/app/src/main/cpp/AudioEngine.cpp +++ b/app/src/main/cpp/AudioEngine.cpp @@ -386,6 +386,17 @@ const BeaconDescriptor AudioEngine::msc_BeaconDescriptors[] = return m_QueuedBeacons.size(); } + bool AudioEngine::IsHandleActive(uint64_t handle) { + std::lock_guard guard(m_BeaconsMutex); + for (const auto &queued : m_QueuedBeacons) { + if (queued->m_Handle == handle) return true; + } + for (const auto &beacon : m_Beacons) { + if (beacon->m_Handle == handle) return true; + } + return false; + } + void AudioEngine::UpdateAudioConfig(std::string &utterance_id, int sample_rate, int audio_format, @@ -399,7 +410,7 @@ const BeaconDescriptor AudioEngine::msc_BeaconDescriptors[] = } } - void AudioEngine::AddBeacon(PositionedAudio *beacon, bool queued) + uint64_t AudioEngine::AddBeacon(PositionedAudio *beacon, bool queued) { std::lock_guard guard(m_BeaconsMutex); if(queued) @@ -418,6 +429,7 @@ const BeaconDescriptor AudioEngine::msc_BeaconDescriptors[] = m_Beacons.insert(beacon); TRACE("AddBeacon -> %zu beacons", m_Beacons.size()); } + return beacon->m_Handle; } void AudioEngine::RemoveBeacon(PositionedAudio *beacon) @@ -586,6 +598,20 @@ Java_org_scottishtecharmy_soundscape_audio_NativeAudioEngine_getQueueDepth(JNIEn } return 0L; } + +extern "C" +JNIEXPORT jboolean JNICALL +Java_org_scottishtecharmy_soundscape_audio_NativeAudioEngine_isHandleActive(JNIEnv *env MAYBE_UNUSED, + jobject thiz MAYBE_UNUSED, + jlong engine_handle, + jlong handle) { + auto* ae = reinterpret_cast(engine_handle); + if(ae) { + return ae->IsHandleActive(static_cast(handle)) ? JNI_TRUE : JNI_FALSE; + } + return JNI_FALSE; +} + extern "C" JNIEXPORT void JNICALL Java_org_scottishtecharmy_soundscape_audio_NativeAudioEngine_destroyNativeBeacon(JNIEnv *env MAYBE_UNUSED, @@ -643,9 +669,11 @@ Java_org_scottishtecharmy_soundscape_audio_NativeAudioEngine_createNativeTextToS if (not tts) { TRACE("Failed to create text to speech"); tts.reset(nullptr); + return 0L; } - auto ret = reinterpret_cast(tts.release()); - return ret; + auto handle = tts->m_Handle; + tts.release(); + return static_cast(handle); } return 0L; } @@ -692,9 +720,11 @@ Java_org_scottishtecharmy_soundscape_audio_NativeAudioEngine_createNativeEarcon( if (not earcon) { TRACE("Failed to create Earcon"); earcon.reset(nullptr); + return 0L; } - auto ret = reinterpret_cast(earcon.release()); - return ret; + auto handle = earcon->m_Handle; + earcon.release(); + return static_cast(handle); } return 0L; } diff --git a/app/src/main/cpp/AudioEngine.h b/app/src/main/cpp/AudioEngine.h index c22216af7..06640cc7a 100644 --- a/app/src/main/cpp/AudioEngine.h +++ b/app/src/main/cpp/AudioEngine.h @@ -69,7 +69,7 @@ namespace soundscape { void SetBeaconType(int beaconType); const BeaconDescriptor *GetBeaconDescriptor() const; - void AddBeacon(PositionedAudio *beacon, bool queued = false); + uint64_t AddBeacon(PositionedAudio *beacon, bool queued = false); void RemoveBeacon(PositionedAudio *beacon); bool ToggleBeaconMute(); @@ -91,6 +91,7 @@ namespace soundscape { void ClearQueue(); unsigned int GetQueueDepth(); + bool IsHandleActive(uint64_t handle); void SetUseHrtf(bool use) { if (m_pMixer) m_pMixer->setUseHrtf(use); } void SetSuppressRestart(bool suppress) { if (m_pMixer) m_pMixer->setSuppressRestart(suppress); } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/audio/AudioEngine.kt b/app/src/main/java/org/scottishtecharmy/soundscape/audio/AudioEngine.kt index c688f7990..cfd702bdc 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/audio/AudioEngine.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/audio/AudioEngine.kt @@ -14,6 +14,7 @@ interface AudioEngine { fun createEarcon(asset: String, type: AudioType, latitude: Double = Double.NaN, longitude: Double = Double.NaN, heading: Double = Double.NaN) : Long fun clearTextToSpeechQueue() fun getQueueDepth() : Long + fun isHandleActive(handle: Long) : Boolean fun updateGeometry(listenerLatitude: Double, listenerLongitude: Double, listenerHeading: Double?, focusGained: Boolean, duckingAllowed: Boolean, proximityNear: Double) fun setBeaconType(beaconType: String) fun getListOfBeaconTypes() : Array diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/audio/NativeAudioEngine.kt b/app/src/main/java/org/scottishtecharmy/soundscape/audio/NativeAudioEngine.kt index 8fd35b060..a3c066eef 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/audio/NativeAudioEngine.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/audio/NativeAudioEngine.kt @@ -68,6 +68,7 @@ class NativeAudioEngine @Inject constructor(val service: SoundscapeService? = nu private external fun createNativeEarcon(engineHandle: Long, asset:String, mode: Int, latitude: Double, longitude: Double, heading: Double) : Long private external fun clearNativeTextToSpeechQueue(engineHandle: Long) private external fun getQueueDepth(engineHandle: Long) : Long + private external fun isHandleActive(engineHandle: Long, handle: Long) : Boolean private external fun updateGeometry(engineHandle: Long, latitude: Double, longitude: Double, heading: Double, focusGained: Boolean, duckingAllowed: Boolean, proximityNear: Double) private external fun setBeaconType(engineHandle: Long, beaconType: String) private external fun getListOfBeacons() : Array @@ -338,6 +339,15 @@ class NativeAudioEngine @Inject constructor(val service: SoundscapeService? = nu return 0 } + override fun isHandleActive(handle: Long) : Boolean { + synchronized(engineMutex) { + if (engineHandle != 0L) { + return isHandleActive(engineHandle, handle) + } + } + return false + } + override fun getAvailableSpeechEngines() : List { return ttsEngine.getAvailableEngines() } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt b/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt index f8ba87552..c8944cceb 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/geoengine/GeoEngine.kt @@ -848,8 +848,15 @@ class GeoEngine { val gridFinishTime = timeSource.markNow() Log.e(GridState.TAG, "Time to calculate NearbyMarkers: ${gridFinishTime - gridStartTime}") - if(results.isEmpty()) - return null + if(results.isEmpty()) { + results.add( + PositionedString( + text = localizedContext.getString(R.string.callouts_no_nearby_markers), + type = AudioType.STANDARD + ) + ) + } + return TrackedCallout( userGeometry = userGeometry, diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/RoutePlayer.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/RoutePlayer.kt index 504602f50..8e1688de6 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/RoutePlayer.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/RoutePlayer.kt @@ -152,7 +152,7 @@ class RoutePlayer(val service: SoundscapeService, context: Context) { if ((currentMarker + 1) < route.markers.size) { // We're within 12m of the marker, move on to the next one Log.d(TAG, "Moving to next waypoint ${coroutineContext[Job]}") - moveToNext() + moveToNext(false) } else { // We've reached the end of the route // Announce the end of the route @@ -179,7 +179,7 @@ class RoutePlayer(val service: SoundscapeService, context: Context) { } } - private fun createBeaconAtWaypoint(index: Int) { + private fun createBeaconAtWaypoint(index: Int, userInitiated: Boolean) { currentRouteData?.let { route -> if (index < route.markers.size) { val location = route.markers[index].getLngLatAlt() @@ -206,7 +206,7 @@ class RoutePlayer(val service: SoundscapeService, context: Context) { route.markers[index].name, formatDistanceAndDirection(distance, null, localizedContext)) } - + if(userInitiated) service.audioEngine.clearTextToSpeechQueue() service.speakText( beaconSetText, AudioType.LOCALIZED, location.latitude, location.longitude, 0.0 @@ -229,18 +229,18 @@ class RoutePlayer(val service: SoundscapeService, context: Context) { } fun play() { - createBeaconAtWaypoint(currentMarker) + createBeaconAtWaypoint(currentMarker, true) Log.d(TAG, toString()) } - fun moveToNext() : Boolean { + fun moveToNext(userInitiated: Boolean) : Boolean { currentRouteData?.let { route -> if(route.markers.size > 1) { if ((currentMarker + 1) < route.markers.size) { currentMarker++ _currentRouteFlow.update { it.copy(currentWaypoint = currentMarker) } - createBeaconAtWaypoint(currentMarker) + createBeaconAtWaypoint(currentMarker, userInitiated) } return true } @@ -249,13 +249,13 @@ class RoutePlayer(val service: SoundscapeService, context: Context) { return false } - fun moveToPrevious() : Boolean{ + fun moveToPrevious(userInitiated: Boolean) : Boolean{ currentRouteData?.let { route -> if(route.markers.size > 1) { if (currentMarker > 0) { currentMarker-- _currentRouteFlow.update { it.copy(currentWaypoint = currentMarker) } - createBeaconAtWaypoint(currentMarker) + createBeaconAtWaypoint(currentMarker, userInitiated) return true } } diff --git a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt index 76da4630a..955189be7 100644 --- a/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt +++ b/app/src/main/java/org/scottishtecharmy/soundscape/services/SoundscapeService.kt @@ -36,6 +36,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.cancelChildren import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.collectLatest @@ -103,6 +104,9 @@ class SoundscapeService : MediaSessionService() { // secondary service private var timerJob: Job? = null + // Guard to prevent duplicate user-triggered callouts + private var calloutJob: Job? = null + // Wake lock — keeps CPU running while screen is off so audio callbacks continue private var wakeLock: PowerManager.WakeLock? = null @@ -538,43 +542,80 @@ class SoundscapeService : MediaSessionService() { geoEngine.updateBeaconLocation(null) } + private suspend fun awaitHandle(handle: Long) { + while (handle != 0L && audioEngine.isHandleActive(handle)) { + delay(100) + } + } + + private fun cancelCallout(): Boolean { + val wasActive = calloutJob?.isActive == true + if (wasActive) + calloutJob?.cancel() + + // Always clear the TTS queue as there's been a user action that requires a response + audioEngine.clearTextToSpeechQueue() + return wasActive + } + fun myLocation() { - coroutineScope.launch { - val results = geoEngine.myLocation() - if(results != null) { - audioEngine.clearTextToSpeechQueue() - speakCallout(results, true) + if (cancelCallout()) return + calloutJob = coroutineScope.launch { + if (requestAudioFocus()) { + // The call to myLocation can take a second or so as it might be doing network + // based reverse geocoding. Ensure that the user has feedback that the action is + // taking place by immediately playing the earcon. + audioEngine.createEarcon(EARCON_MODE_ENTER, AudioType.STANDARD) + val results = geoEngine.myLocation() + ensureActive() + var lastHandle = 0L + if (results != null) { + lastHandle = speakCallout(results, false) + } + audioEngine.createEarcon(EARCON_MODE_EXIT, AudioType.STANDARD) + awaitHandle(lastHandle) + } else { + Log.w(TAG, "myLocation: Could not get audio focus.") } } } fun whatsAroundMe() { - coroutineScope.launch { + if (cancelCallout()) return + calloutJob = coroutineScope.launch { val results = geoEngine.whatsAroundMe() + ensureActive() + var lastHandle = 0L if(results.positionedStrings.isNotEmpty()) { - audioEngine.clearTextToSpeechQueue() - speakCallout(results, true) + lastHandle = speakCallout(results, true) } + awaitHandle(lastHandle) } } fun aheadOfMe() { - coroutineScope.launch { + if (cancelCallout()) return + calloutJob = coroutineScope.launch { val results = geoEngine.aheadOfMe() + ensureActive() + var lastHandle = 0L if(results != null) { - audioEngine.clearTextToSpeechQueue() - speakCallout(results, true) + lastHandle = speakCallout(results, true) } + awaitHandle(lastHandle) } } fun nearbyMarkers() { - coroutineScope.launch { + if (cancelCallout()) return + calloutJob = coroutineScope.launch { val results = geoEngine.nearbyMarkers() + ensureActive() + var lastHandle = 0L if(results != null) { - audioEngine.clearTextToSpeechQueue() - speakCallout(results, true) + lastHandle = speakCallout(results, true) } + awaitHandle(lastHandle) } } @@ -623,10 +664,10 @@ class SoundscapeService : MediaSessionService() { routePlayer.stopRoute() } fun routeSkipPrevious(): Boolean { - return routePlayer.moveToPrevious() + return routePlayer.moveToPrevious(true) } fun routeSkipNext(): Boolean { - return routePlayer.moveToNext() + return routePlayer.moveToNext(true) } fun routeMute(): Boolean { if(routePlayer.isPlaying()) { @@ -720,24 +761,25 @@ class SoundscapeService : MediaSessionService() { audioEngine.createTextToSpeech(text, AudioType.STANDARD) } - fun speakCallout(callout: TrackedCallout?, addModeEarcon: Boolean) { + fun speakCallout(callout: TrackedCallout?, addModeEarcon: Boolean) : Long { - if(callout == null) return + if(callout == null) return 0L if (!requestAudioFocus()) { Log.w(TAG, "SpeakCallout: Could not get audio focus.") - return + return 0L } - if(addModeEarcon) audioEngine.createEarcon(EARCON_MODE_ENTER, AudioType.STANDARD) + var lastHandle = 0L + if(addModeEarcon) lastHandle = audioEngine.createEarcon(EARCON_MODE_ENTER, AudioType.STANDARD) for(result in callout.positionedStrings) { if(result.location == null) { var type = result.type if(type == AudioType.LOCALIZED) type = AudioType.STANDARD - if(result.earcon != null) { + if(result.earcon != null) audioEngine.createEarcon(result.earcon, type, 0.0, 0.0, result.heading?:0.0) - } - audioEngine.createTextToSpeech(result.text, type, 0.0, 0.0, result.heading?:0.0) + + lastHandle = audioEngine.createTextToSpeech(result.text, type, 0.0, 0.0, result.heading?:0.0) } else { if(result.earcon != null) { @@ -748,7 +790,7 @@ class SoundscapeService : MediaSessionService() { result.location.longitude, result.heading?:0.0) } - audioEngine.createTextToSpeech( + lastHandle = audioEngine.createTextToSpeech( result.text, result.type, result.location.latitude, @@ -757,10 +799,13 @@ class SoundscapeService : MediaSessionService() { ) } } + // Don't set lastHandle on the earcon, we don't really care if this has finished or not, + // we just want to wait on the TTS. if(addModeEarcon) audioEngine.createEarcon(EARCON_MODE_EXIT, AudioType.STANDARD) callout.calloutHistory?.add(callout) callout.locationFilter?.update(callout.userGeometry) + return lastHandle } fun toggleAutoCallouts() { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 6e46a8f1a..48dcc24a2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -304,6 +304,8 @@ "There is nothing to call out right now" "Nearby Markers" + +"There are no nearby markers" "Sleep" diff --git a/docs/Download_on_the_App_Store_Badge_US-UK_RGB_blk_092917.svg b/docs/Download_on_the_App_Store_Badge_US-UK_RGB_blk_092917.svg new file mode 100755 index 000000000..984e5e994 --- /dev/null +++ b/docs/Download_on_the_App_Store_Badge_US-UK_RGB_blk_092917.svg @@ -0,0 +1,46 @@ + + Download_on_the_App_Store_Badge_US-UK_RGB_blk_4SVG_092917 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/GetItOnGooglePlay_Badge_Web_color_English.png b/docs/GetItOnGooglePlay_Badge_Web_color_English.png deleted file mode 100644 index 7a06997a5..000000000 Binary files a/docs/GetItOnGooglePlay_Badge_Web_color_English.png and /dev/null differ diff --git a/docs/GetItOnGooglePlay_Badge_Web_color_English.svg b/docs/GetItOnGooglePlay_Badge_Web_color_English.svg new file mode 100644 index 000000000..68efbf11a --- /dev/null +++ b/docs/GetItOnGooglePlay_Badge_Web_color_English.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index e267c2ab6..e2db2744c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,11 +6,24 @@ permalink: / --- # What is Soundscape? -Soundscape is an Android app to help people with visual impairment explore and navigate the world. It does this by describing nearby features using mapping data from OpenStreetMap. +Soundscape is a free, open-source app that uses 3D spatial audio to help people with visual impairments build awareness of their surroundings and navigate more confidently. - Get Soundscape on Google Play Store +Rather than giving turn-by-turn directions, Soundscape acts like a lighthouse for the ears. Through stereo headphones, users hear sounds coming from the direction of real-world places, building a richer mental map of their environment. -The initial release is based on the original [iOS Soundscape app](https://www.scottishtecharmy.org/soundscape). We are now at the point where we can publish openly on the Play Store, though we will be continuing to work on and improve the app based on user feedback. +- **Automatic callouts** - As you walk, the app announces nearby intersections, shops, bus stops, parks, and other points of interest. +- **Audio beacon** - Set a destination and hear a continuous spatial sound pulling you toward it. The tone changes when you're pointing the right way. +- **Location awareness** - Tap a button to hear what's around you in each direction, what road you're on, or what's coming up ahead. +- **Markers and routes** - Save favourite locations and create walking routes between them. The app guides you along the route, automatically moving to the next waypoint as you arrive. +- **Street preview** - Explore an unfamiliar area virtually before visiting, to build confidence and familiarity. + +The app carries on working in the background when the phone in a pocket or bag - no need to hold or look at a screen. It is completely free, with no ads or data collection, and the Android version can work offline using downloaded map regions. + +Soundscape was originally developed on iOS by [Microsoft Research](https://www.microsoft.com/en-us/research/product/soundscape/) and was widely used before being open sourced in 2022. The [Scottish Tech Army](https://www.scottishtecharmy.org) has kept it available on iOS and also rewritten it as an open-source Android app. + + Get Soundscape on Google Play Store + Get Soundscape on Apple App +Store # Soundscape support The email address links straight into our help desk system. Please send in any feedback you have, whether it's a feature request or a bug report. You'll receive an automated email in response and we'll get pinged that there's a new ticket to look at and we'll try and answer it as quickly as possible.