diff --git a/WebDriverAgentLib/Routing/FBTCPSocket.h b/WebDriverAgentLib/Routing/FBTCPSocket.h index 31adc7e22..72947e2a0 100644 --- a/WebDriverAgentLib/Routing/FBTCPSocket.h +++ b/WebDriverAgentLib/Routing/FBTCPSocket.h @@ -38,7 +38,11 @@ NS_ASSUME_NONNULL_BEGIN @interface FBTCPSocket : NSObject -@property (nullable, nonatomic) id delegate; +#if __has_feature(objc_arc_weak) +@property (nullable, nonatomic, weak) id delegate; +#else +@property (nullable, nonatomic, assign) id delegate; +#endif /** Creates TCP socket isntance which is going to be started on the specified port diff --git a/WebDriverAgentLib/Routing/FBTCPSocket.m b/WebDriverAgentLib/Routing/FBTCPSocket.m index fabd22074..25a286ddd 100644 --- a/WebDriverAgentLib/Routing/FBTCPSocket.m +++ b/WebDriverAgentLib/Routing/FBTCPSocket.m @@ -48,11 +48,14 @@ - (BOOL)startWithError:(NSError **)error - (void)stop { @synchronized(self.connectedClients) { - for (NSUInteger i = 0; i < [self.connectedClients count]; i++) { - [[self.connectedClients objectAtIndex:i] disconnect]; + NSArray *clients = self.connectedClients.copy; + [self.connectedClients removeAllObjects]; + for (GCDAsyncSocket *client in clients) { + [client disconnect]; } } + self.delegate = nil; [self.listeningSocket disconnect]; } @@ -66,12 +69,18 @@ - (void)socket:(GCDAsyncSocket *)sock didAcceptNewSocket:(GCDAsyncSocket *)newSo @synchronized(self.connectedClients) { [self.connectedClients addObject:newSocket]; } - [self.delegate didClientConnect:newSocket]; + id delegate = self.delegate; + if (nil != delegate) { + [delegate didClientConnect:newSocket]; + } } - (void)socket:(GCDAsyncSocket *)sock didReadData:(NSData *)data withTag:(long)tag { - [self.delegate didClientSendData:sock]; + id delegate = self.delegate; + if (nil != delegate) { + [delegate didClientSendData:sock]; + } } - (void)socketDidDisconnect:(GCDAsyncSocket *)sock withError:(NSError *)err @@ -79,7 +88,10 @@ - (void)socketDidDisconnect:(GCDAsyncSocket *)sock withError:(NSError *)err @synchronized(self.connectedClients) { [self.connectedClients removeObject:sock]; } - [self.delegate didClientDisconnect:sock]; + id delegate = self.delegate; + if (nil != delegate) { + [delegate didClientDisconnect:sock]; + } } @end diff --git a/WebDriverAgentLib/Routing/FBWebServer.m b/WebDriverAgentLib/Routing/FBWebServer.m index 0b82eaff5..29a2e16e6 100644 --- a/WebDriverAgentLib/Routing/FBWebServer.m +++ b/WebDriverAgentLib/Routing/FBWebServer.m @@ -47,10 +47,16 @@ @interface FBWebServer () @property (nonatomic, strong) RoutingHTTPServer *server; @property (atomic, assign) BOOL keepAlive; @property (nonatomic, nullable) FBTCPSocket *screenshotsBroadcaster; +@property (nonatomic, nullable, strong) FBMjpegServer *mjpegServer; @end @implementation FBWebServer +- (void)dealloc +{ + [self stopScreenshotsBroadcaster]; +} + + (NSArray> *)collectCommandHandlerClasses { NSArray *handlersClasses = FBClassesThatConformsToProtocol(@protocol(FBCommandHandler)); @@ -97,7 +103,7 @@ - (void)startHTTPServer [self.server setInterface:bindingIP]; [FBLogger logFmt:@"Using custom binding IP address: %@", bindingIP]; } - + NSError *error; BOOL serverStarted = NO; @@ -117,7 +123,7 @@ - (void)startHTTPServer [FBLogger logFmt:@"Last attempt to start web server failed with error %@", [error description]]; abort(); } - + NSString *serverHost = bindingIP ?: ([XCUIDevice sharedDevice].fb_wifiIPAddress ?: @"127.0.0.1"); [FBLogger logFmt:@"%@http://%@:%d%@", FBServerURLBeginMarker, serverHost, [self.server port], FBServerURLEndMarker]; } @@ -125,12 +131,15 @@ - (void)startHTTPServer - (void)initScreenshotsBroadcaster { [self readMjpegSettingsFromEnv]; + self.mjpegServer = [[FBMjpegServer alloc] init]; self.screenshotsBroadcaster = [[FBTCPSocket alloc] initWithPort:(uint16_t)FBConfiguration.mjpegServerPort]; - self.screenshotsBroadcaster.delegate = [[FBMjpegServer alloc] init]; + self.screenshotsBroadcaster.delegate = self.mjpegServer; NSError *error; if (![self.screenshotsBroadcaster startWithError:&error]) { [FBLogger logFmt:@"Cannot init screenshots broadcaster service on port %@. Original error: %@", @(FBConfiguration.mjpegServerPort), error.description]; + [self.mjpegServer stopStreaming]; + self.mjpegServer = nil; self.screenshotsBroadcaster = nil; } } @@ -138,10 +147,18 @@ - (void)initScreenshotsBroadcaster - (void)stopScreenshotsBroadcaster { if (nil == self.screenshotsBroadcaster) { + self.mjpegServer = nil; return; } + id delegate = self.screenshotsBroadcaster.delegate; + if ([(NSObject *)delegate respondsToSelector:@selector(stopStreaming)]) { + [(FBMjpegServer *)delegate stopStreaming]; + } + self.screenshotsBroadcaster.delegate = nil; [self.screenshotsBroadcaster stop]; + self.screenshotsBroadcaster = nil; + self.mjpegServer = nil; } - (void)readMjpegSettingsFromEnv @@ -164,6 +181,8 @@ - (void)stopServing if (self.server.isRunning) { [self.server stop:NO]; } + self.server = nil; + self.exceptionHandler = nil; self.keepAlive = NO; } @@ -192,10 +211,15 @@ - (BOOL)attemptToStartServer:(RoutingHTTPServer *)server onPort:(NSInteger)port - (void)registerRouteHandlers:(NSArray *)commandHandlerClasses { + __weak typeof(self) weakSelf = self; for (Class commandHandler in commandHandlerClasses) { NSArray *routes = [commandHandler routes]; for (FBRoute *route in routes) { [self.server handleMethod:route.verb withPath:route.path block:^(RouteRequest *request, RouteResponse *response) { + __strong typeof(weakSelf) strongSelf = weakSelf; + if (nil == strongSelf) { + return; + } NSDictionary *arguments = [NSJSONSerialization JSONObjectWithData:request.body options:NSJSONReadingMutableContainers error:NULL]; FBRouteRequest *routeParams = [FBRouteRequest routeRequestWithURL:request.url @@ -209,7 +233,7 @@ - (void)registerRouteHandlers:(NSArray *)commandHandlerClasses [route mountRequest:routeParams intoResponse:response]; } @catch (NSException *exception) { - [self handleException:exception forResponse:response]; + [strongSelf handleException:exception forResponse:response]; } }]; } @@ -237,9 +261,14 @@ - (void)registerServerKeyRouteHandlers [response respondWithString:calibrationPage]; }]; + __weak typeof(self) weakSelf = self; [self.server get:@"/wda/shutdown" withBlock:^(RouteRequest *request, RouteResponse *response) { + __strong typeof(weakSelf) strongSelf = weakSelf; + if (nil == strongSelf) { + return; + } [response respondWithString:@"Shutting down"]; - [self.delegate webServerDidRequestShutdown:self]; + [strongSelf.delegate webServerDidRequestShutdown:strongSelf]; }]; [self registerRouteHandlers:@[FBUnknownCommands.class]]; diff --git a/WebDriverAgentLib/Utilities/FBImageProcessor.m b/WebDriverAgentLib/Utilities/FBImageProcessor.m index 7d2ada759..951568a8e 100644 --- a/WebDriverAgentLib/Utilities/FBImageProcessor.m +++ b/WebDriverAgentLib/Utilities/FBImageProcessor.m @@ -27,6 +27,7 @@ @interface FBImageProcessor () @property (nonatomic) NSData *nextImage; @property (nonatomic, readonly) NSLock *nextImageLock; @property (nonatomic, readonly) dispatch_queue_t scalingQueue; +@property (atomic, assign) BOOL isScalingScheduled; @end @@ -38,6 +39,7 @@ - (id)init if (self) { _nextImageLock = [[NSLock alloc] init]; _scalingQueue = dispatch_queue_create("image.scaling.queue", NULL); + _isScalingScheduled = NO; } return self; } @@ -51,32 +53,45 @@ - (void)submitImageData:(NSData *)image [FBLogger verboseLog:@"Discarding screenshot"]; } self.nextImage = image; + BOOL shouldSchedule = !self.isScalingScheduled; + if (shouldSchedule) { + self.isScalingScheduled = YES; + } [self.nextImageLock unlock]; + if (!shouldSchedule) { + return; + } #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wcompletion-handler" dispatch_async(self.scalingQueue, ^{ - [self.nextImageLock lock]; - NSData *nextImageData = self.nextImage; - self.nextImage = nil; - [self.nextImageLock unlock]; - if (nextImageData == nil) { - return; - } + while (YES) { + @autoreleasepool { + [self.nextImageLock lock]; + NSData *nextImageData = self.nextImage; + self.nextImage = nil; + if (nextImageData == nil) { + self.isScalingScheduled = NO; + [self.nextImageLock unlock]; + return; + } + [self.nextImageLock unlock]; - // We do not want this value to be too high because then we get images larger in size than original ones - // Although, we also don't want to lose too much of the quality on recompression - CGFloat recompressionQuality = MAX(0.9, - MIN(FBMaxCompressionQuality, FBConfiguration.mjpegServerScreenshotQuality / 100.0)); - NSData *thumbnailData = [self.class fixedImageDataWithImageData:nextImageData - scalingFactor:scalingFactor - uti:UTTypeJPEG - compressionQuality:recompressionQuality - // iOS always returns screnshots in portrait orientation, but puts the real value into the metadata - // Use it with care. See https://github.com/appium/WebDriverAgent/pull/812 - fixOrientation:FBConfiguration.mjpegShouldFixOrientation - desiredOrientation:nil]; - completionHandler(thumbnailData ?: nextImageData); + // We do not want this value to be too high because then we get images larger in size than original ones + // Although, we also don't want to lose too much of the quality on recompression + CGFloat recompressionQuality = MAX(0.9, + MIN(FBMaxCompressionQuality, FBConfiguration.mjpegServerScreenshotQuality / 100.0)); + NSData *thumbnailData = [self.class fixedImageDataWithImageData:nextImageData + scalingFactor:scalingFactor + uti:UTTypeJPEG + compressionQuality:recompressionQuality + // iOS always returns screenshots in portrait orientation, but puts the real value into the metadata + // Use it with care. See https://github.com/appium/WebDriverAgent/pull/812 + fixOrientation:FBConfiguration.mjpegShouldFixOrientation + desiredOrientation:nil]; + completionHandler(thumbnailData ?: nextImageData); + } + } }); #pragma clang diagnostic pop } diff --git a/WebDriverAgentLib/Utilities/FBMjpegServer.h b/WebDriverAgentLib/Utilities/FBMjpegServer.h index 294c399f8..a9b47cada 100644 --- a/WebDriverAgentLib/Utilities/FBMjpegServer.h +++ b/WebDriverAgentLib/Utilities/FBMjpegServer.h @@ -19,6 +19,11 @@ NS_ASSUME_NONNULL_BEGIN */ - (instancetype)init; +/** + Stops screenshot broadcasting and prevents future scheduling. + */ +- (void)stopStreaming; + @end NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBMjpegServer.m b/WebDriverAgentLib/Utilities/FBMjpegServer.m index 4f061d2e3..75389df50 100644 --- a/WebDriverAgentLib/Utilities/FBMjpegServer.m +++ b/WebDriverAgentLib/Utilities/FBMjpegServer.m @@ -27,6 +27,11 @@ static NSString *const SERVER_NAME = @"WDA MJPEG Server"; static const char *QUEUE_NAME = "JPEG Screenshots Provider Queue"; +static NSUInteger FBNormalizedMjpegFramerate(NSUInteger framerate) +{ + return (0 == framerate || framerate > MAX_FPS) ? MAX_FPS : framerate; +} + @interface FBMjpegServer() @@ -35,6 +40,9 @@ @interface FBMjpegServer() @property (nonatomic, readonly) FBImageProcessor *imageProcessor; @property (nonatomic, readonly) long long mainScreenID; @property (nonatomic, assign) NSUInteger consecutiveScreenshotFailures; +@property (atomic, assign) BOOL isStreaming; +@property (nonatomic, assign) NSUInteger sentFramesCount; +@property (nonatomic, assign) NSUInteger sentBytesCount; @end @@ -45,38 +53,49 @@ - (instancetype)init { if ((self = [super init])) { _consecutiveScreenshotFailures = 0; + _isStreaming = YES; + _sentFramesCount = 0; + _sentBytesCount = 0; _listeningClients = [NSMutableArray array]; + _imageProcessor = [[FBImageProcessor alloc] init]; + _mainScreenID = [XCUIScreen.mainScreen displayID]; dispatch_queue_attr_t queueAttributes = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_UTILITY, 0); _backgroundQueue = dispatch_queue_create(QUEUE_NAME, queueAttributes); + __weak typeof(self) weakSelf = self; dispatch_async(_backgroundQueue, ^{ - [self streamScreenshot]; + [weakSelf streamScreenshot]; }); - _imageProcessor = [[FBImageProcessor alloc] init]; - _mainScreenID = [XCUIScreen.mainScreen displayID]; } return self; } - (void)scheduleNextScreenshotWithInterval:(uint64_t)timerInterval timeStarted:(uint64_t)timeStarted { + if (!self.isStreaming) { + return; + } uint64_t timeElapsed = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - timeStarted; - int64_t nextTickDelta = timerInterval - timeElapsed; + int64_t nextTickDelta = (int64_t)timerInterval - (int64_t)timeElapsed; + __weak typeof(self) weakSelf = self; if (nextTickDelta > 0) { dispatch_after(dispatch_time(DISPATCH_TIME_NOW, nextTickDelta), self.backgroundQueue, ^{ - [self streamScreenshot]; + [weakSelf streamScreenshot]; }); } else { // Try to do our best to keep the FPS at a decent level dispatch_async(self.backgroundQueue, ^{ - [self streamScreenshot]; + [weakSelf streamScreenshot]; }); } } - (void)streamScreenshot { - NSUInteger framerate = FBConfiguration.mjpegServerFramerate; - uint64_t timerInterval = (uint64_t)(1.0 / ((0 == framerate || framerate > MAX_FPS) ? MAX_FPS : framerate) * NSEC_PER_SEC); + if (!self.isStreaming) { + return; + } + NSUInteger framerate = FBNormalizedMjpegFramerate(FBConfiguration.mjpegServerFramerate); + uint64_t timerInterval = (uint64_t)(1.0 / framerate * NSEC_PER_SEC); uint64_t timeStarted = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW); @synchronized (self.listeningClients) { if (0 == self.listeningClients.count) { @@ -106,23 +125,41 @@ - (void)streamScreenshot self.consecutiveScreenshotFailures = 0; CGFloat scalingFactor = FBConfiguration.mjpegScalingFactor / 100.0; + __weak typeof(self) weakSelf = self; [self.imageProcessor submitImageData:screenshotData scalingFactor:scalingFactor completionHandler:^(NSData * _Nonnull scaled) { - [self sendScreenshot:scaled]; + [weakSelf sendScreenshot:scaled]; }]; [self scheduleNextScreenshotWithInterval:timerInterval timeStarted:timeStarted]; } - (void)sendScreenshot:(NSData *)screenshotData { + if (!self.isStreaming) { + return; + } NSString *chunkHeader = [NSString stringWithFormat:@"--BoundaryString\r\nContent-type: image/jpeg\r\nContent-Length: %@\r\n\r\n", @(screenshotData.length)]; NSMutableData *chunk = [[chunkHeader dataUsingEncoding:NSUTF8StringEncoding] mutableCopy]; [chunk appendData:screenshotData]; [chunk appendData:(id)[@"\r\n\r\n" dataUsingEncoding:NSUTF8StringEncoding]]; @synchronized (self.listeningClients) { + if (!self.isStreaming || 0 == self.listeningClients.count) { + return; + } + NSUInteger clientCount = self.listeningClients.count; for (GCDAsyncSocket *client in self.listeningClients) { - [client writeData:chunk withTimeout:-1 tag:0]; + // Slow clients should fail/close instead of buffering indefinitely. + [client writeData:chunk withTimeout:FRAME_TIMEOUT tag:0]; + } + self.sentFramesCount++; + self.sentBytesCount += chunk.length * clientCount; + NSUInteger framerate = FBNormalizedMjpegFramerate(FBConfiguration.mjpegServerFramerate); + if (0 == self.sentFramesCount % framerate) { + [FBLogger verboseLog:[NSString stringWithFormat:@"MJPEG stats: clients=%@ sentFrames=%@ sentBytes=%@", + @(clientCount), + @(self.sentFramesCount), + @(self.sentBytesCount)]]; } } } @@ -158,4 +195,22 @@ - (void)didClientDisconnect:(GCDAsyncSocket *)client [FBLogger log:@"Disconnected a client from screenshots broadcast"]; } +- (void)stopStreaming +{ + self.isStreaming = NO; + @synchronized (self.listeningClients) { + NSArray *clients = self.listeningClients.copy; + [self.listeningClients removeAllObjects]; + for (GCDAsyncSocket *client in clients) { + [client disconnect]; + } + } +} + +- (void)dealloc +{ + [self stopStreaming]; + [FBLogger verboseLog:@"FBMjpegServer deallocated"]; +} + @end diff --git a/lib/xcodebuild.ts b/lib/xcodebuild.ts index f21289906..754c87943 100644 --- a/lib/xcodebuild.ts +++ b/lib/xcodebuild.ts @@ -461,18 +461,18 @@ export class XcodeBuild { } const proxyTimeout = noSessionProxy.timeout; - noSessionProxy.timeout = 1000; + (noSessionProxy as any).timeout = 1000; try { currentStatus = (await noSessionProxy.command('/status', 'GET')) as StringRecord; - if (currentStatus && currentStatus.ios && (currentStatus.ios as any).ip) { - this.agentUrl = (currentStatus.ios as any).ip; + if (currentStatus?.ios?.ip) { + this.agentUrl = currentStatus.ios.ip as string; } this.log.debug(`WebDriverAgent information:`); this.log.debug(JSON.stringify(currentStatus, null, 2)); } catch (err: any) { throw new Error(`Unable to connect to running WebDriverAgent: ${err.message}`); } finally { - noSessionProxy.timeout = proxyTimeout; + (noSessionProxy as any).timeout = proxyTimeout; } });