commit 26d2627a1c49f62565de53e7ddf015162bb77e62
parent c2918aaf16ec6729bf90517bfa8311ca62345039
Author: Daniel D’Aquino <daniel@daquino.me>
Date: Thu, 11 Apr 2024 00:41:54 +0000
Cache videos
This commit implements a simple but functional video cache.
It works by providing a method called `maybe_cached_url`, where a video
URL can be passed in, and this method will either return the URL of a
cached version of this video if available, or the original URL if not. It also
downloads new video URLs on the background into the cache folder for use
next time.
Functional testing
-------------------
PASS
Device: iPhone 15 simulator
iOS: 17.4
Damus: Approximately this commit
Setup:
- Debug connection
- Expiry time locally changed to 5 minutes
Steps:
1. Basic functionality
1. Go to a profile with lots of videos
2. Scroll down
3. Filter logs to only logs that start with "Loading video with URL"
4. Check that most videos are being loaded from external URLs. PASS
5. Now restart the app and go to that same profile
6. Scroll down and watch logs. Videos should now be loaded with an internal file URL. PASS
2. Automatic cache refresh after expiry
1. Go to the video-heavy profile, make note of the external URL.
2. Go to a different screen and then come back to that video. Make sure the file was loaded from cache. PASS
3. Now go to a different screen and wait 5 minutes.
4. Come back to the same video. It should be loaded from the external URL. PASS
3. "Clear cache" button functionality
1. Go to the video-heavy profile, make note of the external URL.
2. Go to a different screen and then come back to that video. Make sure the file was loaded from cache. PASS
3. Now quit the app (to ensure file is not in use when trying to delete it)
4. Clear cache in settings
5. Go back to the same video. It should now be loaded from the external URL. PASS
Performance testing
-----------------------
Device: iPhone 13 mini
iOS: 17.3.1
Damus: This commit
Baseline: 87de88861adb3b41d73998452e7c876ab5ee06bf
Setup:
- Debug connection
- Expiry time locally changed to 5 minutes
- Running on Profile mode, with XCode Instruments
Steps:
1. Start recording network activity with XCode Instruments
2. Go to a video-heavy profile (e.g. Julian Figueroa)
3. Scroll down to a specific video (Make sure to scroll through at least 5 videos)
4. Stop recording and measure the "Bytes In" from "Network connections"
5. Repeat this for all test configurations
Results:
- Baseline (No caching support): 26.74 MiB
- This commit (First run, cleared cache): 40.52 MiB
- This commit (Second run, cache filled with videos): 8.13 MiB
Automated test coverage
------------------------
PASS
Device: iPhone 15 simulator
iOS: 17.4
Damus: This commit
Coverage:
- Ran new automated tests multiple times. PASS 3/3 times
- Ran all other automated tests. PASS
Signed-off-by: Daniel D’Aquino <daniel@daquino.me>
Link: 20240411004129.84436-3-daniel@daquino.me
Signed-off-by: William Casarin <jb55@jb55.com>
Diffstat:
6 files changed, 242 insertions(+), 1 deletion(-)
diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj
@@ -490,6 +490,7 @@
D76556D62B1E6C08001B0CCC /* DamusPurpleWelcomeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D76556D52B1E6C08001B0CCC /* DamusPurpleWelcomeView.swift */; };
D76874F32AE3632B00FB0F68 /* ProfileZapLinkView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D76874F22AE3632B00FB0F68 /* ProfileZapLinkView.swift */; };
D77BFA0B2AE3051200621634 /* ProfileActionSheetView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D77BFA0A2AE3051200621634 /* ProfileActionSheetView.swift */; };
+ D7831AF82BBE11E2005DA780 /* VideoCacheTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7831AF72BBE11E2005DA780 /* VideoCacheTests.swift */; };
D783A63F2AD4E53D00658DDA /* SuggestedHashtagsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D783A63E2AD4E53D00658DDA /* SuggestedHashtagsView.swift */; };
D78525252A7B2EA4002FA637 /* NoteContentViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */; };
D7870BC12AC4750B0080BA88 /* MentionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7870BC02AC4750B0080BA88 /* MentionView.swift */; };
@@ -520,6 +521,7 @@
D7ADD3DE2B53854300F104C4 /* DamusPurpleURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7ADD3DD2B53854300F104C4 /* DamusPurpleURL.swift */; };
D7ADD3E02B538D4200F104C4 /* DamusPurpleURLSheetView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7ADD3DF2B538D4200F104C4 /* DamusPurpleURLSheetView.swift */; };
D7ADD3E22B538E3500F104C4 /* DamusPurpleVerifyNpubView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7ADD3E12B538E3500F104C4 /* DamusPurpleVerifyNpubView.swift */; };
+ D7C28E3B2BBB4D0000EE459F /* VideoCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7C28E3A2BBB4D0000EE459F /* VideoCache.swift */; };
D7C6787E2B2D34CC00BCEAFB /* NIP98AuthenticatedRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7C6787D2B2D34CC00BCEAFB /* NIP98AuthenticatedRequest.swift */; };
D7CB5D3B2B112FBB00AD4105 /* NotificationFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D70A3B162B02DCE5008BD568 /* NotificationFormatter.swift */; };
D7CB5D3C2B1130C600AD4105 /* LocalNotification.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CDA128B29EB19C40006FA5A /* LocalNotification.swift */; };
@@ -1402,6 +1404,7 @@
D76556D52B1E6C08001B0CCC /* DamusPurpleWelcomeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleWelcomeView.swift; sourceTree = "<group>"; };
D76874F22AE3632B00FB0F68 /* ProfileZapLinkView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileZapLinkView.swift; sourceTree = "<group>"; };
D77BFA0A2AE3051200621634 /* ProfileActionSheetView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileActionSheetView.swift; sourceTree = "<group>"; };
+ D7831AF72BBE11E2005DA780 /* VideoCacheTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoCacheTests.swift; sourceTree = "<group>"; };
D783A63E2AD4E53D00658DDA /* SuggestedHashtagsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestedHashtagsView.swift; sourceTree = "<group>"; };
D78525242A7B2EA4002FA637 /* NoteContentViewTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NoteContentViewTests.swift; sourceTree = "<group>"; };
D7870BC02AC4750B0080BA88 /* MentionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MentionView.swift; sourceTree = "<group>"; };
@@ -1417,6 +1420,7 @@
D7ADD3DD2B53854300F104C4 /* DamusPurpleURL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleURL.swift; sourceTree = "<group>"; };
D7ADD3DF2B538D4200F104C4 /* DamusPurpleURLSheetView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleURLSheetView.swift; sourceTree = "<group>"; };
D7ADD3E12B538E3500F104C4 /* DamusPurpleVerifyNpubView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurpleVerifyNpubView.swift; sourceTree = "<group>"; };
+ D7C28E3A2BBB4D0000EE459F /* VideoCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoCache.swift; sourceTree = "<group>"; };
D7C6787D2B2D34CC00BCEAFB /* NIP98AuthenticatedRequest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NIP98AuthenticatedRequest.swift; sourceTree = "<group>"; };
D7CB5D3D2B116DAD00AD4105 /* NotificationsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NotificationsManager.swift; sourceTree = "<group>"; };
D7CB5D442B116FE800AD4105 /* Contacts+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Contacts+.swift"; sourceTree = "<group>"; };
@@ -1650,6 +1654,7 @@
D74AAFC12B153395006CF0F4 /* HeadlessDamusState.swift */,
B5C60C1F2B530D5100C5ECA7 /* MuteItem.swift */,
B533694D2B66D791008A805E /* MutelistManager.swift */,
+ D7C28E3A2BBB4D0000EE459F /* VideoCache.swift */,
);
path = Models;
sourceTree = "<group>";
@@ -2553,6 +2558,7 @@
E06336A92B75832100A88E6B /* ImageMetadataTest.swift */,
D7CBD1D52B8D509800BFD889 /* DamusPurpleImpendingExpirationTests.swift */,
D72927AC2BAB515C00F93E90 /* RelayURLTests.swift */,
+ D7831AF72BBE11E2005DA780 /* VideoCacheTests.swift */,
);
path = damusTests;
sourceTree = "<group>";
@@ -3288,6 +3294,7 @@
4C1253522A76C6130004F4B8 /* ComposeNotify.swift in Sources */,
4C7D09662A0AE62100943473 /* AlbyButton.swift in Sources */,
D7100C582B76FC8400C59298 /* MarketingContentView.swift in Sources */,
+ D7C28E3B2BBB4D0000EE459F /* VideoCache.swift in Sources */,
4CAAD8AD298851D000060CEA /* AccountDeletion.swift in Sources */,
4CFF8F6329CC9AD7008DB934 /* ImageContextMenuModifier.swift in Sources */,
4C54AA0A29A55429003E4487 /* EventGroup.swift in Sources */,
@@ -3544,6 +3551,7 @@
75AD872B2AA23A460085EF2C /* Block+Tests.swift in Sources */,
E0E024112B7C19C20075735D /* TranslationTests.swift in Sources */,
F944F56E29EA9CCC0067B3BF /* DamusParseContentTests.swift in Sources */,
+ D7831AF82BBE11E2005DA780 /* VideoCacheTests.swift in Sources */,
3A5E47C72A4A76C800C0D090 /* TrieTests.swift in Sources */,
B501062D2B363036003874F5 /* AuthIntegrationTests.swift in Sources */,
4CB883AE2976FA9300DC99E7 /* FormatTests.swift in Sources */,
diff --git a/damus/ContentView.swift b/damus/ContentView.swift
@@ -516,6 +516,7 @@ struct ContentView: View {
print("txn: 📙 DAMUS BACKGROUNDED")
Task { @MainActor in
damus_state.ndb.close()
+ VideoCache.standard?.periodic_purge()
}
break
case .inactive:
diff --git a/damus/Models/VideoCache.swift b/damus/Models/VideoCache.swift
@@ -0,0 +1,114 @@
+//
+// VideoCache.swift
+// damus
+//
+// Created by Daniel D'Aquino on 2024-04-01.
+//
+import Foundation
+import CryptoKit
+
+// Default expiry time of only 1 day to prevent using too much storage
+fileprivate let DEFAULT_EXPIRY_TIME: TimeInterval = 60*60*24
+// Default cache directory is in the system-provided caches directory, so that the operating system can delete files when it needs storage space
+// (https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html)
+fileprivate let DEFAULT_CACHE_DIRECTORY_PATH: URL? = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("video_cache")
+
+struct VideoCache {
+ private let cache_url: URL
+ private let expiry_time: TimeInterval
+ static let standard: VideoCache? = try? VideoCache()
+
+ init?(cache_url: URL? = nil, expiry_time: TimeInterval = DEFAULT_EXPIRY_TIME) throws {
+ guard let cache_url_to_apply = cache_url ?? DEFAULT_CACHE_DIRECTORY_PATH else { return nil }
+ self.cache_url = cache_url_to_apply
+ self.expiry_time = expiry_time
+
+ // Create the cache directory if it doesn't exist
+ do {
+ try FileManager.default.createDirectory(at: self.cache_url, withIntermediateDirectories: true, attributes: nil)
+ } catch {
+ Log.error("Could not create cache directory: %s", for: .storage, error.localizedDescription)
+ throw error
+ }
+ }
+
+ /// Checks for a cached video and returns its URL if available, otherwise downloads and caches the video.
+ func maybe_cached_url_for(video_url: URL) throws -> URL {
+ let cached_url = url_to_cached_url(url: video_url)
+
+ if FileManager.default.fileExists(atPath: cached_url.path) {
+ // Check if the cached video has expired
+ let file_attributes = try FileManager.default.attributesOfItem(atPath: cached_url.path)
+ if let modification_date = file_attributes[.modificationDate] as? Date, Date().timeIntervalSince(modification_date) <= expiry_time {
+ // Video is not expired
+ return cached_url
+ } else {
+ Task {
+ // Video is expired, delete and re-download on the background
+ try FileManager.default.removeItem(at: cached_url)
+ return try await download_and_cache_video(from: video_url)
+ }
+ return video_url
+ }
+ } else {
+ Task {
+ // Video is not cached, download and cache on the background
+ return try await download_and_cache_video(from: video_url)
+ }
+ return video_url
+ }
+ }
+
+ /// Downloads video content using URLSession and caches it to disk.
+ private func download_and_cache_video(from url: URL) async throws -> URL {
+ let (data, response) = try await URLSession.shared.data(from: url)
+
+ guard let http_response = response as? HTTPURLResponse,
+ 200..<300 ~= http_response.statusCode else {
+ throw URLError(.badServerResponse)
+ }
+
+ let destination_url = url_to_cached_url(url: url)
+
+ try data.write(to: destination_url)
+ return destination_url
+ }
+
+ func url_to_cached_url(url: URL) -> URL {
+ let hashed_url = hash_url(url)
+ let file_extension = url.pathExtension
+ return self.cache_url.appendingPathComponent(hashed_url + "." + file_extension)
+ }
+
+ /// Deletes all cached videos older than the expiry time.
+ func periodic_purge(completion: ((Error?) -> Void)? = nil) {
+ DispatchQueue.global(qos: .background).async {
+ Log.info("Starting periodic video cache purge", for: .storage)
+ let file_manager = FileManager.default
+ do {
+ let cached_files = try file_manager.contentsOfDirectory(at: self.cache_url, includingPropertiesForKeys: [.contentModificationDateKey], options: .skipsHiddenFiles)
+
+ for file in cached_files {
+ let attributes = try file.resourceValues(forKeys: [.contentModificationDateKey])
+ if let modification_date = attributes.contentModificationDate, Date().timeIntervalSince(modification_date) > self.expiry_time {
+ try file_manager.removeItem(at: file)
+ }
+ }
+ DispatchQueue.main.async {
+ completion?(nil)
+ }
+ } catch {
+ DispatchQueue.main.async {
+ completion?(error)
+ }
+ }
+ }
+ }
+
+ /// Hashes the URL using SHA-256
+ private func hash_url(_ url: URL) -> String {
+ let data = Data(url.absoluteString.utf8)
+ let hashed_data = SHA256.hash(data: data)
+ return hashed_data.compactMap { String(format: "%02x", $0) }.joined()
+ }
+}
diff --git a/damus/Views/Video/DamusVideoPlayer.swift b/damus/Views/Video/DamusVideoPlayer.swift
@@ -33,7 +33,7 @@ struct DamusVideoPlayer: View {
else {
mute = nil
}
- _model = StateObject(wrappedValue: DamusVideoPlayerViewModel(url: url, video_size: video_size, controller: controller, mute: mute))
+ _model = StateObject(wrappedValue: DamusVideoPlayerViewModel.cached_video_model(url: url, video_size: video_size, controller: controller, mute: mute))
self.visibility_tracking_method = visibility_tracking_method
self.style = style
}
diff --git a/damus/Views/Video/DamusVideoPlayerViewModel.swift b/damus/Views/Video/DamusVideoPlayerViewModel.swift
@@ -57,6 +57,12 @@ final class DamusVideoPlayerViewModel: ObservableObject {
}
}
+ static func cached_video_model(url: URL, video_size: Binding<CGSize?>, controller: VideoController, mute: Bool? = nil) -> Self {
+ let maybe_cached_url = (try? VideoCache.standard?.maybe_cached_url_for(video_url: url)) ?? url
+ Log.info("Loading video with URL: %s",for: .render, maybe_cached_url.absoluteString)
+ return Self.init(url: maybe_cached_url, video_size: video_size, controller: controller, mute: mute)
+ }
+
init(url: URL, video_size: Binding<CGSize?>, controller: VideoController, mute: Bool? = nil) {
self.url = url
player_item = AVPlayerItem(url: url)
diff --git a/damusTests/VideoCacheTests.swift b/damusTests/VideoCacheTests.swift
@@ -0,0 +1,112 @@
+//
+// VideoCacheTests.swift
+// damusTests
+//
+// Created by Daniel D’Aquino on 2024-04-03.
+//
+
+import Foundation
+import XCTest
+@testable import damus
+
+// TODO: Reduce test dependency on external factors such as external URLs.
+let TEST_VIDEO_URL = "http://cdn.jb55.com/s/zaps-build.mp4"
+let LONG_TEST_EXPIRY_TIME: TimeInterval = 60 * 60 * 24 // A long expiry time for a video (in seconds).
+let SHORT_TEST_EXPIRY_TIME: TimeInterval = 15 // A short expiry time for a video (in seconds). Must be as short as possible but large enough to allow some test operations to occur
+let CACHE_SAVE_TIME_TIMEOUT: TimeInterval = 8 // How long the test will wait for the cache to save a file (in seconds)
+let EXPIRY_TIME_MARGIN: TimeInterval = 3 // The extra time we will wait after expected expiry, to avoid test timing issues. (in seconds)
+
+final class VideoCacheTests: XCTestCase {
+
+ func testCachedURLForExistingVideo() throws {
+ // Create a temporary directory for the cache
+ let test_cache_directory = FileManager.default.temporaryDirectory.appendingPathComponent("test_video_cache")
+
+ // Create a test video file
+ let original_video_url = URL(string: TEST_VIDEO_URL)!
+ FileManager.default.createFile(atPath: original_video_url.path, contents: Data(), attributes: nil)
+
+ // Create a VideoCache instance with the temporary cache directory
+ let test_expiry_time: TimeInterval = 10
+ let video_cache = try VideoCache(cache_url: test_cache_directory, expiry_time: test_expiry_time)!
+
+ // Call the maybe_cached_url_for method with the test video URL
+ let expected_cache_url = video_cache.url_to_cached_url(url: original_video_url)
+ let maybe_cached_url = try video_cache.maybe_cached_url_for(video_url: original_video_url)
+
+ // Assert that the returned URL is the same as the original
+ XCTAssertEqual(maybe_cached_url, original_video_url, "Returned URL should be the same as the original video URL on the first time we download it")
+
+ // Check that next time we get this video, we get the cached URL.
+ let cached_url_expectation = XCTestExpectation(description: "On second time we get a video, the cached URL should be returned")
+ let start_time = Date()
+ while Date().timeIntervalSince(start_time) < CACHE_SAVE_TIME_TIMEOUT {
+ let maybe_cached_url = try video_cache.maybe_cached_url_for(video_url: original_video_url)
+ if maybe_cached_url == expected_cache_url {
+ cached_url_expectation.fulfill()
+ break
+ }
+ sleep(1)
+ }
+ wait(for: [cached_url_expectation], timeout: CACHE_SAVE_TIME_TIMEOUT)
+
+ // Now wait for the remaining time until the expiry time + a margin
+ let remaining_time = test_expiry_time + EXPIRY_TIME_MARGIN - Date().timeIntervalSince(start_time)
+
+ // Wait for the expiry time to pass
+ sleep(UInt32(max(remaining_time, 0)))
+
+ // Call the periodic_purge method to purge expired video items
+ video_cache.periodic_purge()
+
+ // Call the maybe_cached_url_for method again
+ let maybe_cached_url_after_expiry = try video_cache.maybe_cached_url_for(video_url: original_video_url)
+
+ // Assert that the returned URL is the same as the original video URL, since the cache should have expired.
+ XCTAssertEqual(maybe_cached_url_after_expiry, original_video_url, "Video cache should expire after expiry time")
+
+ // Clean up the temporary files and directory
+ try FileManager.default.removeItem(at: test_cache_directory)
+ }
+
+ func testClearCache() throws {
+ // Create a temporary directory for the cache
+ let test_cache_directory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first!.appendingPathComponent("test_video_cache")
+ try FileManager.default.createDirectory(at: test_cache_directory, withIntermediateDirectories: true, attributes: nil)
+
+ // Create a test video file
+ let original_video_url = URL(string: TEST_VIDEO_URL)!
+ FileManager.default.createFile(atPath: original_video_url.path, contents: Data(), attributes: nil)
+
+ // Create a VideoCache instance with the temporary cache directory and a longer expiry time
+ let expiry_time: TimeInterval = LONG_TEST_EXPIRY_TIME
+ let video_cache = try VideoCache(cache_url: test_cache_directory, expiry_time: expiry_time)!
+
+ // Request the cached URL for the test video to create the cached file
+ let expected_cache_url = video_cache.url_to_cached_url(url: original_video_url)
+ let _ = try video_cache.maybe_cached_url_for(video_url: original_video_url)
+
+ // Check that next time we get this video, we get the cached URL.
+ let cached_url_expectation = XCTestExpectation(description: "On second time we get a video, the cached URL should be returned")
+ let start_time = Date()
+ while Date().timeIntervalSince(start_time) < CACHE_SAVE_TIME_TIMEOUT {
+ let maybe_cached_url = try video_cache.maybe_cached_url_for(video_url: original_video_url)
+ if maybe_cached_url == expected_cache_url {
+ cached_url_expectation.fulfill()
+ break
+ }
+ sleep(1)
+ }
+ wait(for: [cached_url_expectation], timeout: CACHE_SAVE_TIME_TIMEOUT)
+
+ // Call the periodic_purge method
+ DamusCacheManager.shared.clear_cache(damus_state: test_damus_state, completion: {
+ // Assert that fetching the cached URL after clearing cache will
+ let maybe_cached_url_after_purge = try? video_cache.maybe_cached_url_for(video_url: original_video_url)
+ XCTAssertEqual(maybe_cached_url_after_purge, original_video_url)
+
+ // Clean up the temporary directory
+ try? FileManager.default.removeItem(at: test_cache_directory)
+ })
+ }
+}
