commit 6de44223f2c16354124da23f8a0e2dbce080e355
parent 75d87fee9d1371d75bb4652aba48fab3ed768eed
Author: kernelkind <kernelkind@gmail.com>
Date: Tue, 27 Feb 2024 22:32:34 -0500
add performance upgrades to media picker
- Use jpeg instead of png data when processing a UIImage.
- Make processing of media occur after user confirms upload selection when possible for better responsiveness.
- Reduce redundant data fetching.
Lightning-address: kernelkind@getalby.com
Signed-off-by: kernelkind <kernelkind@gmail.com>
Link: 20240228033235.66935-2-kernelkind@gmail.com
Signed-off-by: William Casarin <jb55@jb55.com>
Diffstat:
8 files changed, 203 insertions(+), 149 deletions(-)
diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj
@@ -640,9 +640,10 @@
E02429952B7E97740088B16C /* CameraController.swift in Sources */ = {isa = PBXBuildFile; fileRef = E02429942B7E97740088B16C /* CameraController.swift */; };
E02B54182B4DFADA0077FF42 /* Bech32ObjectTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E02B54172B4DFADA0077FF42 /* Bech32ObjectTests.swift */; };
E04A37C62B544F090029650D /* URIParsing.swift in Sources */ = {isa = PBXBuildFile; fileRef = E04A37C52B544F090029650D /* URIParsing.swift */; };
- E0E024112B7C19C20075735D /* TranslationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E0E024102B7C19C20075735D /* TranslationTests.swift */; };
E06336AA2B75832100A88E6B /* ImageMetadataTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = E06336A92B75832100A88E6B /* ImageMetadataTest.swift */; };
E06336AB2B75850100A88E6B /* img_with_location.jpeg in Resources */ = {isa = PBXBuildFile; fileRef = E06336A82B7582E000A88E6B /* img_with_location.jpeg */; };
+ E0E024112B7C19C20075735D /* TranslationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E0E024102B7C19C20075735D /* TranslationTests.swift */; };
+ E0EE9DD42B8E5FEA00F3002D /* ImageProcessing.swift in Sources */ = {isa = PBXBuildFile; fileRef = E0EE9DD32B8E5FEA00F3002D /* ImageProcessing.swift */; };
E4FA1C032A24BB7F00482697 /* SearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */; };
E990020F2955F837003BBC5A /* EditMetadataView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E990020E2955F837003BBC5A /* EditMetadataView.swift */; };
E9E4ED0B295867B900DD7078 /* ThreadView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E9E4ED0A295867B900DD7078 /* ThreadView.swift */; };
@@ -1430,9 +1431,10 @@
E02429942B7E97740088B16C /* CameraController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraController.swift; sourceTree = "<group>"; };
E02B54172B4DFADA0077FF42 /* Bech32ObjectTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Bech32ObjectTests.swift; sourceTree = "<group>"; };
E04A37C52B544F090029650D /* URIParsing.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = URIParsing.swift; sourceTree = "<group>"; };
- E0E024102B7C19C20075735D /* TranslationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranslationTests.swift; sourceTree = "<group>"; };
E06336A82B7582E000A88E6B /* img_with_location.jpeg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = img_with_location.jpeg; sourceTree = "<group>"; };
E06336A92B75832100A88E6B /* ImageMetadataTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageMetadataTest.swift; sourceTree = "<group>"; };
+ E0E024102B7C19C20075735D /* TranslationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranslationTests.swift; sourceTree = "<group>"; };
+ E0EE9DD32B8E5FEA00F3002D /* ImageProcessing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageProcessing.swift; sourceTree = "<group>"; };
E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchSettingsView.swift; sourceTree = "<group>"; };
E990020E2955F837003BBC5A /* EditMetadataView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EditMetadataView.swift; sourceTree = "<group>"; };
E9E4ED0A295867B900DD7078 /* ThreadView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThreadView.swift; sourceTree = "<group>"; };
@@ -1687,6 +1689,7 @@
isa = PBXGroup;
children = (
4C198DF429F88D2E004C165C /* ImageMetadata.swift */,
+ E0EE9DD32B8E5FEA00F3002D /* ImageProcessing.swift */,
);
path = Images;
sourceTree = "<group>";
@@ -3388,6 +3391,7 @@
4C5E54062A9671F800FF6E60 /* UserStatusSheet.swift in Sources */,
F71694F42A6732B7001F4053 /* GradientFollowButton.swift in Sources */,
4C3AC7A728369BA200E1F516 /* SearchHomeView.swift in Sources */,
+ E0EE9DD42B8E5FEA00F3002D /* ImageProcessing.swift in Sources */,
4CB883B0297705DD00DC99E7 /* NoteZapButton.swift in Sources */,
4C363A922825FCF2006E126D /* ProfileUpdate.swift in Sources */,
4C3BEFDA281DCA1400B3DE84 /* LikeCounter.swift in Sources */,
diff --git a/damus/Models/ImageUploadModel.swift b/damus/Models/ImageUploadModel.swift
@@ -8,6 +8,13 @@
import Foundation
import UIKit
+enum PreUploadedMedia {
+ case uiimage(UIImage)
+ case processed_image(URL)
+ case unprocessed_image(URL)
+ case processed_video(URL)
+ case unprocessed_video(URL)
+}
enum MediaUpload {
case image(URL)
diff --git a/damus/Util/Images/ImageMetadata.swift b/damus/Util/Images/ImageMetadata.swift
@@ -210,51 +210,3 @@ func process_image_metadatas(cache: EventCache, ev: NostrEvent) {
}
}
}
-
-func canGetSourceTypeFromUrl(url: URL) -> Bool {
- guard let source = CGImageSourceCreateWithURL(url as CFURL, nil) else {
- print("Failed to create image source.")
- return false
- }
- return CGImageSourceGetType(source) != nil
-}
-
-func removeGPSDataFromImage(fromImageURL imageURL: URL) -> Bool {
- guard let source = CGImageSourceCreateWithURL(imageURL as CFURL, nil) else {
- print("Failed to create image source.")
- return false
- }
- let data = NSMutableData()
-
- let totalCount = CGImageSourceGetCount(source)
-
- guard totalCount > 0 else {
- print("No images found.")
- return false
- }
-
- guard let type = CGImageSourceGetType(source),
- let destination = CGImageDestinationCreateWithData(data, type, totalCount, nil) else {
- print("Failed to create image destination.")
- return false
- }
-
- let removeGPSProperties: CFDictionary = [kCGImagePropertyGPSDictionary as String: kCFNull] as CFDictionary
-
- for i in 0...totalCount {
- CGImageDestinationAddImageFromSource(destination, source, i, removeGPSProperties)
- }
-
- if CGImageDestinationFinalize(destination) {
- do {
- try data.write(to: imageURL, options: .atomic)
- return true
- } catch {
- print("Failed to write image data: \(error)")
- return false
- }
- } else {
- print("Failed to finalize image destination.")
- return false
- }
-}
diff --git a/damus/Util/Images/ImageProcessing.swift b/damus/Util/Images/ImageProcessing.swift
@@ -0,0 +1,151 @@
+//
+// ImageProcessing.swift
+// damus
+//
+// Created by KernelKind on 2/27/24.
+//
+
+import UIKit
+
+/// Removes GPS data from image at url and writes changes to new file
+func processImage(url: URL) -> URL? {
+ let fileExtension = url.pathExtension
+ guard let imageData = try? Data(contentsOf: url) else {
+ print("Failed to load image data from URL.")
+ return nil
+ }
+
+ guard let source = CGImageSourceCreateWithData(imageData as CFData, nil) else { return nil }
+
+ return processImage(source: source, fileExtension: fileExtension)
+}
+
+/// Removes GPS data from image and writes changes to new file
+func processImage(image: UIImage) -> URL? {
+ let fixedImage = image.fixOrientation()
+ guard let imageData = fixedImage.jpegData(compressionQuality: 1.0) else { return nil }
+ guard let source = CGImageSourceCreateWithData(imageData as CFData, nil) else { return nil }
+
+ return processImage(source: source, fileExtension: "jpeg")
+}
+
+fileprivate func processImage(source: CGImageSource, fileExtension: String) -> URL? {
+ let destinationURL = createMediaURL(fileExtension: fileExtension)
+
+ guard let destination = removeGPSDataFromImage(source: source, url: destinationURL) else { return nil }
+
+ if !CGImageDestinationFinalize(destination) { return nil }
+
+ return destinationURL
+}
+
+/// TODO: strip GPS data from video
+func processVideo(videoURL: URL) -> URL? {
+ saveVideoToTemporaryFolder(videoURL: videoURL)
+}
+
+fileprivate func saveVideoToTemporaryFolder(videoURL: URL) -> URL? {
+ let destinationURL = createMediaURL(fileExtension: videoURL.pathExtension)
+
+ do {
+ try FileManager.default.copyItem(at: videoURL, to: destinationURL)
+ return destinationURL
+ } catch {
+ print("Error copying file: \(error.localizedDescription)")
+ return nil
+ }
+}
+
+/// Generate a temporary URL with a unique filename
+fileprivate func createMediaURL(fileExtension: String) -> URL {
+ let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true)
+ let uniqueMediaName = "\(UUID().uuidString).\(fileExtension)"
+ let temporaryMediaURL = temporaryDirectoryURL.appendingPathComponent(uniqueMediaName)
+
+ return temporaryMediaURL
+}
+
+/**
+ Take the PreUploadedMedia payload, process it, if necessary, and convert it into a URL
+ which is ready to be uploaded to the upload service.
+
+ URLs containing media that hasn't been processed were generated from the system and were granted
+ access as a security scoped resource. The data will need to be processed to strip GPS data
+ and saved to a new location which isn't security scoped.
+ */
+func generateMediaUpload(_ media: PreUploadedMedia?) -> MediaUpload? {
+ guard let media else { return nil }
+
+ switch media {
+ case .uiimage(let image):
+ guard let url = processImage(image: image) else { return nil }
+ return .image(url)
+ case .unprocessed_image(let url):
+ guard let newUrl = processImage(url: url) else { return nil }
+ url.stopAccessingSecurityScopedResource()
+ return .image(newUrl)
+ case .processed_image(let url):
+ return .image(url)
+ case .processed_video(let url):
+ return .video(url)
+ case .unprocessed_video(let url):
+ guard let newUrl = processVideo(videoURL: url) else { return nil }
+ url.stopAccessingSecurityScopedResource()
+ return .video(newUrl)
+ }
+}
+
+extension UIImage {
+ func fixOrientation() -> UIImage {
+ guard imageOrientation != .up else { return self }
+
+ UIGraphicsBeginImageContextWithOptions(size, false, scale)
+ draw(in: CGRect(origin: .zero, size: size))
+ let normalizedImage = UIGraphicsGetImageFromCurrentImageContext()
+ UIGraphicsEndImageContext()
+
+ return normalizedImage ?? self
+ }
+}
+
+func canGetSourceTypeFromUrl(url: URL) -> Bool {
+ guard let source = CGImageSourceCreateWithURL(url as CFURL, nil) else {
+ print("Failed to create image source.")
+ return false
+ }
+ return CGImageSourceGetType(source) != nil
+}
+
+func removeGPSDataFromImageAndWrite(fromImageURL imageURL: URL) -> Bool {
+ guard let source = CGImageSourceCreateWithURL(imageURL as CFURL, nil) else {
+ print("Failed to create image source.")
+ return false
+ }
+
+ guard let destination = removeGPSDataFromImage(source: source, url: imageURL) else { return false }
+
+ return CGImageDestinationFinalize(destination)
+}
+
+fileprivate func removeGPSDataFromImage(source: CGImageSource, url: URL) -> CGImageDestination? {
+ let totalCount = CGImageSourceGetCount(source)
+
+ guard totalCount > 0 else {
+ print("No images found.")
+ return nil
+ }
+
+ guard let type = CGImageSourceGetType(source),
+ let destination = CGImageDestinationCreateWithURL(url as CFURL, type, totalCount, nil) else {
+ print("Failed to create image destination.")
+ return nil
+ }
+
+ let removeGPSProperties: CFDictionary = [kCGImageMetadataShouldExcludeGPS: kCFBooleanTrue] as CFDictionary
+
+ for i in 0..<totalCount {
+ CGImageDestinationAddImageFromSource(destination, source, i, removeGPSProperties)
+ }
+
+ return destination
+}
diff --git a/damus/Views/MediaPicker.swift b/damus/Views/MediaPicker.swift
@@ -16,7 +16,8 @@ struct MediaPicker: UIViewControllerRepresentable {
@Binding var image_upload_confirm: Bool
var imagesOnly: Bool = false
- let onMediaPicked: (MediaUpload) -> Void
+ let onMediaPicked: (PreUploadedMedia) -> Void
+
final class Coordinator: NSObject, PHPickerViewControllerDelegate {
let parent: MediaPicker
@@ -37,17 +38,17 @@ struct MediaPicker: UIViewControllerRepresentable {
if canGetSourceTypeFromUrl(url: url) {
// Media was not taken from camera
- if let savedURL = self.saveImageToTemporaryFolder(from: url) {
- self.chooseImage(url: savedURL)
- }
+ self.attemptAcquireResourceAndChooseMedia(
+ url: url,
+ fallback: processImage,
+ unprocessedEnum: {.unprocessed_image($0)},
+ processedEnum: {.processed_image($0)}
+ )
} else {
// Media was taken from camera
result.itemProvider.loadObject(ofClass: UIImage.self) { (image, error) in
- guard let image = image as? UIImage, error == nil else { return }
- let fixedImage = image.fixOrientation()
-
- if let savedURL = self.saveImageToTemporaryFolder(image: fixedImage) {
- self.chooseImage(url: savedURL)
+ if let image = image as? UIImage, error == nil {
+ self.chooseMedia(.uiimage(image))
}
}
}
@@ -56,80 +57,36 @@ struct MediaPicker: UIViewControllerRepresentable {
result.itemProvider.loadFileRepresentation(forTypeIdentifier: UTType.movie.identifier) { (url, error) in
guard let url, error == nil else { return }
- guard let url = self.saveVideoToTemporaryFolder(videoURL: url) else { return }
- self.parent.onMediaPicked(.video(url))
- self.parent.image_upload_confirm = true
+ self.attemptAcquireResourceAndChooseMedia(
+ url: url,
+ fallback: processVideo,
+ unprocessedEnum: {.unprocessed_video($0)},
+ processedEnum: {.processed_video($0)}
+ )
}
}
}
}
- func chooseImage(url: URL) {
- if removeGPSDataFromImage(fromImageURL: url) {
- self.parent.onMediaPicked(.image(url))
- self.parent.image_upload_confirm = true
- }
+ private func chooseMedia(_ media: PreUploadedMedia) {
+ self.parent.onMediaPicked(media)
+ self.parent.image_upload_confirm = true
}
- func saveImageToTemporaryFolder(from imageUrl: URL) -> URL? {
- let fileExtension = imageUrl.pathExtension
- guard let imageData = try? Data(contentsOf: imageUrl) else {
- print("Failed to load image data from URL.")
- return nil
- }
-
- return saveImageToTemporaryFolder(imageData: imageData, imageType: fileExtension)
- }
-
- func saveImageToTemporaryFolder(image: UIImage, imageType: String = "png") -> URL? {
- // Convert UIImage to Data
- let imageData: Data?
- if imageType.lowercased() == "jpeg" {
- imageData = image.jpegData(compressionQuality: 1.0)
+ private func attemptAcquireResourceAndChooseMedia(url: URL, fallback: (URL) -> URL?, unprocessedEnum: (URL) -> PreUploadedMedia, processedEnum: (URL) -> PreUploadedMedia) {
+ if url.startAccessingSecurityScopedResource() {
+ // Have permission from system to use url out of scope
+ print("Acquired permission to security scoped resource")
+ self.chooseMedia(unprocessedEnum(url))
} else {
- imageData = image.pngData()
- }
-
- guard let data = imageData else {
- print("Failed to convert UIImage to Data.")
- return nil
- }
-
- return saveImageToTemporaryFolder(imageData: data, imageType: imageType)
- }
-
- private func saveImageToTemporaryFolder(imageData: Data, imageType: String) -> URL? {
- // Generate a temporary URL with a unique filename
- let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true)
- let uniqueImageName = "\(UUID().uuidString).\(imageType)"
- let temporaryImageURL = temporaryDirectoryURL.appendingPathComponent(uniqueImageName)
-
- // Save the image data to the temporary URL
- do {
- try imageData.write(to: temporaryImageURL)
- return temporaryImageURL
- } catch {
- print("Error saving image data to temporary URL: \(error.localizedDescription)")
- return nil
- }
- }
-
- func saveVideoToTemporaryFolder(videoURL: URL) -> URL? {
- let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true)
- let fileExtension = videoURL.pathExtension
- let uniqueFileName = UUID().uuidString + (fileExtension.isEmpty ? "" : ".\(fileExtension)")
- let destinationURL = temporaryDirectoryURL.appendingPathComponent(uniqueFileName)
-
- do {
- try FileManager.default.copyItem(at: videoURL, to: destinationURL)
- return destinationURL
- } catch {
- print("Error copying file: \(error.localizedDescription)")
- return nil
+ // Need to copy URL to non-security scoped location
+ guard let newUrl = fallback(url) else { return }
+ self.chooseMedia(processedEnum(newUrl))
}
}
+
}
-
+
func makeCoordinator() -> Coordinator {
Coordinator(self)
}
@@ -147,16 +104,3 @@ struct MediaPicker: UIViewControllerRepresentable {
func updateUIViewController(_ uiViewController: PHPickerViewController, context: Context) {
}
}
-
-extension UIImage {
- func fixOrientation() -> UIImage {
- guard imageOrientation != .up else { return self }
-
- UIGraphicsBeginImageContextWithOptions(size, false, scale)
- draw(in: CGRect(origin: .zero, size: size))
- let normalizedImage = UIGraphicsGetImageFromCurrentImageContext()
- UIGraphicsEndImageContext()
-
- return normalizedImage ?? self
- }
-}
diff --git a/damus/Views/PostView.swift b/damus/Views/PostView.swift
@@ -57,7 +57,7 @@ struct PostView: View {
@State var newCursorIndex: Int?
@State var textHeight: CGFloat? = nil
- @State var mediaToUpload: MediaUpload? = nil
+ @State var preUploadedMedia: PreUploadedMedia? = nil
@StateObject var image_upload: ImageUploadModel = ImageUploadModel()
@StateObject var tagModel: TagModel = TagModel()
@@ -420,12 +420,12 @@ struct PostView: View {
}
.background(DamusColors.adaptableWhite.edgesIgnoringSafeArea(.all))
.sheet(isPresented: $attach_media) {
- MediaPicker(image_upload_confirm: $image_upload_confirm) { media in
- self.mediaToUpload = media
+ MediaPicker(image_upload_confirm: $image_upload_confirm){ media in
+ self.preUploadedMedia = media
}
.alert(NSLocalizedString("Are you sure you want to upload this media?", comment: "Alert message asking if the user wants to upload media."), isPresented: $image_upload_confirm) {
Button(NSLocalizedString("Upload", comment: "Button to proceed with uploading."), role: .none) {
- if let mediaToUpload {
+ if let mediaToUpload = generateMediaUpload(preUploadedMedia) {
self.handle_upload(media: mediaToUpload)
self.attach_media = false
}
diff --git a/damus/Views/Profile/EditPictureControl.swift b/damus/Views/Profile/EditPictureControl.swift
@@ -24,7 +24,7 @@ struct EditPictureControl: View {
@State private var show_library = false
@State var image_upload_confirm: Bool = false
- @State var mediaToUpload: MediaUpload? = nil
+ @State var preUploadedMedia: PreUploadedMedia? = nil
var body: some View {
Menu {
@@ -62,11 +62,11 @@ struct EditPictureControl: View {
.sheet(isPresented: $show_camera) {
MediaPicker(image_upload_confirm: $image_upload_confirm, imagesOnly: true) { media in
- self.mediaToUpload = media
+ self.preUploadedMedia = media
}
.alert(NSLocalizedString("Are you sure you want to upload this image?", comment: "Alert message asking if the user wants to upload an image."), isPresented: $image_upload_confirm) {
Button(NSLocalizedString("Upload", comment: "Button to proceed with uploading."), role: .none) {
- if let mediaToUpload {
+ if let mediaToUpload = generateMediaUpload(preUploadedMedia) {
self.handle_upload(media: mediaToUpload)
self.show_camera = false
}
@@ -76,15 +76,11 @@ struct EditPictureControl: View {
}
.sheet(isPresented: $show_library) {
MediaPicker(image_upload_confirm: $image_upload_confirm, imagesOnly: true) { media in
- if case .image = media {
- self.mediaToUpload = media
- } else {
- print("Cannot upload videos as profile image")
- }
+ self.preUploadedMedia = media
}
.alert(NSLocalizedString("Are you sure you want to upload this image?", comment: "Alert message asking if the user wants to upload an image."), isPresented: $image_upload_confirm) {
Button(NSLocalizedString("Upload", comment: "Button to proceed with uploading."), role: .none) {
- if let mediaToUpload {
+ if let mediaToUpload = generateMediaUpload(preUploadedMedia) {
self.handle_upload(media: mediaToUpload)
self.show_library = false
}
diff --git a/damusTests/ImageMetadataTest.swift b/damusTests/ImageMetadataTest.swift
@@ -29,7 +29,7 @@ final class ImageMetadataTest : XCTestCase {
return
}
- let removalSuccess = removeGPSDataFromImage(fromImageURL: testOutputURL)
+ let removalSuccess = removeGPSDataFromImageAndWrite(fromImageURL: testOutputURL)
XCTAssertTrue(removalSuccess, "GPS data removal was not successful")
