From d2a2220596141f61f7200578d19eb8e4ea74855e Mon Sep 17 00:00:00 2001 From: Antoine Bollengier <44288655+b5i@users.noreply.github.com> Date: Fri, 30 Aug 2024 20:31:03 +0200 Subject: [PATCH] Various bug fixes and improvements on VideoInfos(WithDownloadFormats)Response. - Fixed removePlayerFilesFromDisk() not removing properly old players. - Fixed n-parameter extraction. - Improved VideoInfosResponse by adding aspectRatio and better HLS streaming file (allows selection of the language). Testing the scrapping of the downloadFormats and defaultFormats directly in the VideoInfosResponse instead of the VideoInfosWithDownloadFormatsResponse. --- Sources/YouTubeKit/YouTubeModel.swift | 41 +++++++++++++++---- .../VideoInfos/VideoInfosResponse.swift | 24 ++++++++++- ...ideoInfosWithDownloadFormatsResponse.swift | 12 +++--- Tests/YouTubeKitTests/YouTubeKitTests.swift | 6 ++- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/Sources/YouTubeKit/YouTubeModel.swift b/Sources/YouTubeKit/YouTubeModel.swift index be57a3c..4f96d9d 100644 --- a/Sources/YouTubeKit/YouTubeModel.swift +++ b/Sources/YouTubeKit/YouTubeModel.swift @@ -412,24 +412,49 @@ public class YouTubeModel { url: URL(string: "https://www.youtube.com/youtubei/v1/player")!, method: .POST, headers: [ - .init(name: "Accept", content: "*/*"), + .init(name: "Accept", content: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), .init(name: "Accept-Encoding", content: "gzip, deflate, br"), .init(name: "Host", content: "www.youtube.com"), - .init(name: "User-Agent", content: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15"), + .init(name: "User-Agent", content: "com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)"), .init(name: "Accept-Language", content: "\(self.selectedLocale);q=0.9"), .init(name: "Origin", content: "https://www.youtube.com/"), .init(name: "Referer", content: "https://www.youtube.com/"), .init(name: "Content-Type", content: "application/json"), - .init(name: "X-Origin", content: "https://www.youtube.com") + .init(name: "X-Origin", content: "https://www.youtube.com"), + .init(name: "X-Youtube-Client-Name", content: "5"), + .init(name: "X-Youtube-Client-Version", content: "19.29.1"), + .init(name: "Cookie", content: "PREF=hl=\(self.selectedLocaleLanguageCode)&tz=UTC; SOCS=CAI; GPS=1; VISITOR_INFO1_LIVE=X454mME5IB0; VISITOR_PRIVACY_METADATA=CgJDSBIEGgAgKQ%3D%3D") // yt-dlp ], addQueryAfterParts: [ - .init(index: 0, encode: true), - .init(index: 1, encode: true) + .init(index: 0, encode: false, content: .query) ], httpBody: [ - "{\"context\":{\"client\":{\"deviceMake\":\"Apple\",\"deviceModel\":\"\",\"userAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15,gzip(gfe)\",\"clientName\":\"WEB\",\"clientVersion\":\"2.20230602.01.00\",\"osName\":\"Macintosh\",\"osVersion\":\"10_15_7\",\"platform\":\"DESKTOP\",\"clientFormFactor\":\"UNKNOWN_FORM_FACTOR\",\"configInfo\":{},\"screenDensityFloat\":2,\"userInterfaceTheme\":\"USER_INTERFACE_THEME_DARK\",\"timeZone\":\"Europe/Zurich\",\"browserName\":\"Safari\",\"browserVersion\":\"16.5\",\"acceptHeader\":\"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\"utcOffsetMinutes\":120,\"clientScreen\":\"WATCH\",\"mainAppWebInfo\":{\"graftUrl\":\"/watch?v=", - "&pp=YAHIAQE%3D\",\"webDisplayMode\":\"WEB_DISPLAY_MODE_BROWSER\",\"isWebNativeShareAvailable\":true}},\"user\":{\"lockedSafetyMode\":false},\"request\":{\"useSsl\":true,\"internalExperimentFlags\":[],\"consistencyTokenJars\":[]}},\"videoId\":\"", - "\",\"params\":\"YAHIAQE%3D\",\"playbackContext\":{\"contentPlaybackContext\":{\"vis\":5,\"splay\":false,\"autoCaptionsDefaultOn\":false,\"autonavState\":\"STATE_NONE\",\"html5Preference\":\"HTML5_PREF_WANTS\",\"signatureTimestamp\":19508,\"autoplay\":true,\"autonav\":true,\"referer\":\"https://www.youtube.com/\",\"lactMilliseconds\":\"-1\",\"watchAmbientModeContext\":{\"hasShownAmbientMode\":true,\"watchAmbientModeEnabled\":true}}},\"racyCheckOk\":false,\"contentCheckOk\":false}" + #""" + { + "contentCheckOk": true, + "context": { + "client": { + "clientName": "IOS", + "clientVersion": "19.29.1", + "deviceMake": "Apple", + "deviceModel": "iPhone16,2", + "hl": "en", + "osName": "iPhone", + "osVersion": "17.5.1.21F90", + "timeZone": "UTC", + "userAgent": "com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)", + "utcOffsetMinutes": 0 + } + }, + "playbackContext": { + "contentPlaybackContext": { + "html5Preference": "HTML5_PREF_WANTS" + } + }, + "racyCheckOk": true, + "videoId": " + """#, + #""}"# ], parameters: [ .init(name: "prettyPrint", content: "false") diff --git a/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosResponse.swift b/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosResponse.swift index f715430..a3be41e 100644 --- a/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosResponse.swift +++ b/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosResponse.swift @@ -71,6 +71,17 @@ public struct VideoInfosResponse: YouTubeResponse { /// Count of view of the video, usually an integer in the string. public var viewCount: String? + + /// The aspect ratio of the video (width/height). + public var aspectRatio: Double? + + /// Array of formats used to download the video, they usually contain both audio and video data and the download speed is higher than the ``VideoInfosResponse/downloadFormats``. + @available(*, deprecated, message: "This property is unstable for the moment.") + public var defaultFormats: [any DownloadFormat] + + /// Array of formats used to download the video, usually sorted from highest video quality to lowest followed by audio formats. + @available(*, deprecated, message: "This property is unstable for the moment.") + public var downloadFormats: [any DownloadFormat] public init( captions: [YTCaption] = [], @@ -83,7 +94,10 @@ public struct VideoInfosResponse: YouTubeResponse { videoDescription: String? = nil, videoId: String? = nil, videoURLsExpireAt: Date? = nil, - viewCount: String? = nil + viewCount: String? = nil, + aspectRatio: Double? = nil, + defaultFormats: [any DownloadFormat] = [], + downloadFormats: [any DownloadFormat] = [] ) { self.captions = captions self.channel = channel @@ -96,6 +110,9 @@ public struct VideoInfosResponse: YouTubeResponse { self.videoId = videoId self.videoURLsExpireAt = videoURLsExpireAt self.viewCount = viewCount + self.aspectRatio = aspectRatio + self.defaultFormats = defaultFormats + self.downloadFormats = downloadFormats } /// Decode json to give an instance of ``VideoInfosResponse``. @@ -148,7 +165,10 @@ public struct VideoInfosResponse: YouTubeResponse { } return videoURLsExpireAt }(), - viewCount: videoDetailsJSON["viewCount"].string + viewCount: videoDetailsJSON["viewCount"].string, + aspectRatio: streamingJSON["aspectRatio"].double, + defaultFormats: streamingJSON["formats"].arrayValue.compactMap { VideoInfosWithDownloadFormatsResponse.decodeFormatFromJSON(json: $0) }, + downloadFormats: streamingJSON["adaptiveFormats"].arrayValue.compactMap { VideoInfosWithDownloadFormatsResponse.decodeFormatFromJSON(json: $0) } ) } diff --git a/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosWithDownloadFormatsResponse.swift b/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosWithDownloadFormatsResponse.swift index 194a259..3e2d55f 100644 --- a/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosWithDownloadFormatsResponse.swift +++ b/Sources/YouTubeKit/YouTubeResponseTypes/VideoInfos/VideoInfosWithDownloadFormatsResponse.swift @@ -95,7 +95,7 @@ public struct VideoInfosWithDownloadFormatsResponse: YouTubeResponse { nParameterString: String ) -> [DownloadFormat] { return json.map({ encodedItem in - var item = decodeFromJSON(json: encodedItem) + var item = decodeFormatFromJSON(json: encodedItem) if let cipher = encodedItem["signatureCipher"].string { guard let url = cipher.ytkFirstGroupMatch(for: "&?url=([^\\s|&]*)")?.removingPercentEncoding else { return item } @@ -167,7 +167,7 @@ public struct VideoInfosWithDownloadFormatsResponse: YouTubeResponse { /// - Parameter textString: YouTube's player code. /// - Returns: The Javascript code in Data format. private static func extractNParameterFunction(fromFileText textString: String) -> Data { - guard let functionContents = textString.replacingOccurrences(of: "\n", with: "").ytkFirstGroupMatch(for: "(var b=a.split\\(\"\"\\)[\\s\\S]*?return b.join\\(\"\"\\)\\})") else { return Data() } + guard let functionContents = textString.replacingOccurrences(of: "\n", with: "").ytkFirstGroupMatch(for: #"function\(a\)\{(var b=(?:a.split\((?:(?:\"\"\))|(?:a\.slice\(0\,0\)))|(?:String\.prototype\.split\.call))[\s\S]*?(?:(?:Array\.prototype\.join\.call)|(?:return b.join\(\"\"\)\}))[\s\S]*?;)"#) else { return Data() } return ("function processNParameter(a) {" + functionContents).data(using: .utf8) ?? Data() } @@ -517,7 +517,7 @@ public struct VideoInfosWithDownloadFormatsResponse: YouTubeResponse { /// Decode a ``DownloadFormat`` base informations from a JSON instance. /// - Parameter json: the JSON to be decoded. /// - Returns: A ``DownloadFormat``. - private static func decodeFromJSON(json: JSON) -> DownloadFormat { + static func decodeFormatFromJSON(json: JSON) -> DownloadFormat { if json["fps"].int != nil { /// Will return an instance of ``VideoInfosWithDownloadFormatsResponse/VideoDownloadFormat`` return VideoInfosWithDownloadFormatsResponse.VideoDownloadFormat( @@ -537,7 +537,7 @@ public struct VideoInfosWithDownloadFormatsResponse: YouTubeResponse { } }(), isCopyrightedMedia: json["signatureCipher"].string != nil, - url: nil, + url: json["signatureCipher"].string == nil ? json["url"].url : nil, mimeType: json["mimeType"].string?.ytkFirstGroupMatch(for: "([^;]*)"), width: json["width"].int, height: json["height"].int, @@ -563,7 +563,7 @@ public struct VideoInfosWithDownloadFormatsResponse: YouTubeResponse { } }(), isCopyrightedMedia: json["signatureCipher"].string != nil, - url: nil, + url: json["signatureCipher"].string == nil ? json["url"].url : nil, mimeType: json["mimeType"].string?.ytkFirstGroupMatch(for: "([^;]*)"), audioSampleRate: { if let audioSampleRate = json["audioSampleRate"].string { @@ -581,7 +581,7 @@ public struct VideoInfosWithDownloadFormatsResponse: YouTubeResponse { /// Remove all player mappings from disk. public static func removePlayerFilesFromDisk() throws { let playersDirectory = try getDocumentDirectory() - let filesInDir = FileManager.default.enumerator(atPath: playersDirectory.absoluteString) + let filesInDir = FileManager.default.enumerator(at: playersDirectory, includingPropertiesForKeys: nil) guard let filesInDir = filesInDir else { return } for file in filesInDir { if let file = file as? URL { diff --git a/Tests/YouTubeKitTests/YouTubeKitTests.swift b/Tests/YouTubeKitTests/YouTubeKitTests.swift index 7618446..2b174cb 100644 --- a/Tests/YouTubeKitTests/YouTubeKitTests.swift +++ b/Tests/YouTubeKitTests/YouTubeKitTests.swift @@ -615,6 +615,9 @@ final class YouTubeKitTests: XCTestCase { XCTAssertNotNil(requestResult.videoId, TEST_NAME + "Checking if requestResult.videoId is not nil.") XCTAssertNotNil(requestResult.videoURLsExpireAt, TEST_NAME + "Checking if requestResult.videoURLsExpireAt is not nil.") XCTAssertNotNil(requestResult.viewCount, TEST_NAME + "Checking if requestResult.viewCount is not nil.") + XCTAssertNotNil(requestResult.aspectRatio, TEST_NAME + "Checking if requestResult.aspectRatio is not nil.") + //XCTAssertNotEqual(requestResult.downloadFormats.count, 0, TEST_NAME + "Checking if requestResult.downloadFormats is empty") + //XCTAssertNotEqual(requestResult.defaultFormats.count, 0, TEST_NAME + "Checking if requestResult.defaultFormats is empty") let captionsResults = try await VideoCaptionsResponse.sendThrowingRequest(youtubeModel: YTM, data: [.customURL: requestResult.captions.first!.url.absoluteString]) @@ -685,7 +688,7 @@ final class YouTubeKitTests: XCTestCase { func testVideoInfosWithDownloadFormatsResponse() async throws { let TEST_NAME = "Test: testVideoInfosWithDownloadFormatsResponse() -> " - + try VideoInfosWithDownloadFormatsResponse.removePlayerFilesFromDisk() for video in [YTVideo(videoId: "dSDbwfXX5_I"), YTVideo(videoId: "3ryID_SwU5E")] as [YTVideo] { @@ -696,6 +699,7 @@ final class YouTubeKitTests: XCTestCase { XCTAssertNotEqual(requestResult.defaultFormats.count, 0, TEST_NAME + "Checking if requestResult.defaultFormats is empty") XCTAssertNotEqual(requestResult.videoInfos.streamingURL, nil, TEST_NAME + "Checking if requestResult.videoInfos.streamingURL is empty") } + } func testAutoCompletionResponse() async throws {