importer chnges
This commit is contained in:
+197
-9
@@ -15,7 +15,7 @@ struct ImportProgress: Sendable {
|
||||
struct Importer {
|
||||
let media: URL
|
||||
@discardableResult func run(_ urls: [URL],library: Library,copyMedia: Bool = true,cancelled: () -> Bool = { false },progress: (ImportProgress) -> Void = { _ in }) throws -> [Int64] {
|
||||
try FileManager.default.createDirectory(at: media, withIntermediateDirectories: true)
|
||||
try ManagedMediaStorage.prepare(media)
|
||||
var state = ImportProgress()
|
||||
var lastReport = Date.distantPast
|
||||
func report(_ current: String,force: Bool = false) {
|
||||
@@ -72,10 +72,46 @@ struct Importer {
|
||||
// Validate names before extraction; reject path traversal and symlinks afterwards.
|
||||
let details = try process(["-Z", "-l", zip.path])
|
||||
guard !details.split(separator: "\n").contains(where: { $0.hasPrefix("l") }) else { throw LibraryError.message("ZIP symlinks are unsupported") }
|
||||
let list = try process(["-Z1", zip.path])
|
||||
let names = list.split(separator: "\n").map(String.init)
|
||||
let list = try processData(["-Z1", zip.path])
|
||||
let rawNames = list.split(separator:0x0A).map { Data($0) }
|
||||
let names = rawNames.map { String(decoding:$0,as:UTF8.self) }
|
||||
guard !names.isEmpty, names.allSatisfy({ !$0.hasPrefix("/") && !$0.split(separator: "/").contains("..") && !$0.contains("\\") }) else { throw LibraryError.message("Unsafe or empty ZIP") }
|
||||
_ = try process(["-q", zip.path, "-d", folder.path])
|
||||
// ditto maps legacy ZIP filename bytes to valid macOS names. unzip can
|
||||
// fail with exit 50 ("disk full") on those names even when CRCs pass.
|
||||
_ = try process(["-x", "-k", "--noqtn", zip.path, folder.path],executable:"/usr/bin/ditto",operation:"extract ZIP archive")
|
||||
// Unflagged older Windows archives commonly use CP1252. ditto makes
|
||||
// their names filesystem-safe using Mac Roman; rename those components
|
||||
// to Unicode CP1252 spellings before matching MP3/CDG companions.
|
||||
var renamedComponents = [String:String]()
|
||||
for rawName in try archiveNames(zip) {
|
||||
var original = [String]()
|
||||
for rawComponent in rawName.split(separator:0x2F) {
|
||||
let bytes = Data(rawComponent)
|
||||
if let unicode = String(data:bytes,encoding:.utf8) { original.append(unicode);continue }
|
||||
guard let native = String(data:bytes,encoding:.macOSRoman),
|
||||
let unicode = String(data:bytes,encoding:.windowsCP1252) ?? String(data:bytes,encoding:.isoLatin1) else { continue }
|
||||
original.append(native)
|
||||
renamedComponents[original.joined(separator:"/")] = unicode.precomposedStringWithCanonicalMapping
|
||||
}
|
||||
}
|
||||
if !renamedComponents.isEmpty {
|
||||
let rootPath = folder.resolvingSymlinksInPath().path
|
||||
let entries = FileManager.default.enumerator(at:folder,includingPropertiesForKeys:[.isSymbolicLinkKey])!
|
||||
var pending = [(URL,String)]()
|
||||
for case let entry as URL in entries {
|
||||
guard try entry.resourceValues(forKeys:[.isSymbolicLinkKey]).isSymbolicLink != true else { throw LibraryError.message("ZIP symlinks are unsupported") }
|
||||
let path = entry.resolvingSymlinksInPath().path
|
||||
guard path.hasPrefix(rootPath+"/") else { throw LibraryError.message("Unsafe ZIP path") }
|
||||
let relative = String(path.dropFirst(rootPath.count+1))
|
||||
if let corrected = renamedComponents[relative] { pending.append((entry,corrected)) }
|
||||
}
|
||||
// Children move before their parents, keeping nested paths valid.
|
||||
for (entry,corrected) in pending.sorted(by:{$0.0.pathComponents.count > $1.0.pathComponents.count}) {
|
||||
if cancelled() { throw CancellationError() }
|
||||
let target = entry.deletingLastPathComponent().appendingPathComponent(corrected)
|
||||
if entry.lastPathComponent != corrected { try FileManager.default.moveItem(at:entry,to:target) }
|
||||
}
|
||||
}
|
||||
let extracted = FileManager.default.enumerator(at: folder, includingPropertiesForKeys: [.isSymbolicLinkKey,.isRegularFileKey])!
|
||||
for case let url as URL in extracted {
|
||||
let values = try url.resourceValues(forKeys: [.isSymbolicLinkKey,.isRegularFileKey])
|
||||
@@ -154,13 +190,165 @@ struct Importer {
|
||||
guard let file = try? AVAudioFile(forReading:url) else { return 0 }
|
||||
return Double(file.length)/file.processingFormat.sampleRate
|
||||
}
|
||||
private func process(_ arguments: [String]) throws -> String {
|
||||
let task = Process(); task.executableURL = URL(fileURLWithPath: "/usr/bin/unzip"); task.arguments = arguments
|
||||
/// Read original central-directory bytes: zipinfo translates legacy names
|
||||
/// for the terminal, so its output cannot be used to rename extracted files.
|
||||
private func archiveNames(_ url:URL) throws -> [Data] {
|
||||
let file = try FileHandle(forReadingFrom:url);defer { try? file.close() }
|
||||
let size = try file.seekToEnd(),tailSize = min(size,65_557)
|
||||
try file.seek(toOffset:size-tailSize)
|
||||
let tail = [UInt8](try file.readToEnd() ?? Data())
|
||||
func integer(_ bytes:[UInt8],_ offset:Int,_ length:Int) -> UInt64 {
|
||||
guard offset >= 0,offset+length <= bytes.count else { return 0 }
|
||||
return (0..<length).reduce(0) { $0 | UInt64(bytes[offset+$1]) << (8*$1) }
|
||||
}
|
||||
guard tail.count >= 22,let end = stride(from:tail.count-22,through:0,by:-1).first(where:{integer(tail,$0,4) == 0x06054b50 && $0+22+Int(integer(tail,$0+20,2)) == tail.count}) else { throw LibraryError.message("Invalid ZIP directory") }
|
||||
var count = integer(tail,end+10,2),offset = integer(tail,end+16,4)
|
||||
if count == 0xFFFF || offset == 0xFFFFFFFF {
|
||||
guard end >= 20,integer(tail,end-20,4) == 0x07064b50 else { throw LibraryError.message("Invalid ZIP64 directory") }
|
||||
try file.seek(toOffset:integer(tail,end-12,8))
|
||||
let record = [UInt8](try file.read(upToCount:56) ?? Data())
|
||||
guard record.count == 56,integer(record,0,4) == 0x06064b50 else { throw LibraryError.message("Invalid ZIP64 directory") }
|
||||
count = integer(record,32,8);offset = integer(record,48,8)
|
||||
}
|
||||
guard offset <= size,count <= size/46 else { throw LibraryError.message("Invalid ZIP directory bounds") }
|
||||
var names = [Data]()
|
||||
for _ in 0..<count {
|
||||
try file.seek(toOffset:offset)
|
||||
let header = [UInt8](try file.read(upToCount:46) ?? Data())
|
||||
guard header.count == 46,integer(header,0,4) == 0x02014b50 else { throw LibraryError.message("Invalid ZIP entry") }
|
||||
let length = integer(header,28,2),extra = integer(header,30,2),comment = integer(header,32,2)
|
||||
guard offset+46+length+extra+comment <= size else { throw LibraryError.message("Invalid ZIP entry bounds") }
|
||||
let name = try file.read(upToCount:Int(length)) ?? Data()
|
||||
guard name.count == Int(length) else { throw LibraryError.message("Incomplete ZIP filename") }
|
||||
names.append(name);offset += 46+length+extra+comment
|
||||
}
|
||||
return names
|
||||
}
|
||||
private func process(_ arguments: [String],executable: String = "/usr/bin/unzip",operation: String = "read ZIP archive") throws -> String {
|
||||
String(decoding:try processData(arguments,executable:executable,operation:operation),as:UTF8.self)
|
||||
}
|
||||
private func processData(_ arguments: [String],executable: String = "/usr/bin/unzip",operation: String = "read ZIP archive") throws -> Data {
|
||||
let task = Process(); task.executableURL = URL(fileURLWithPath:executable); task.arguments = arguments
|
||||
let out = Pipe(); task.standardOutput = out
|
||||
let err = FileHandle.nullDevice; task.standardError = err
|
||||
task.standardError = out
|
||||
try task.run()
|
||||
let bytes = out.fileHandleForReading.readDataToEndOfFile(); task.waitUntilExit()
|
||||
guard task.terminationStatus == 0 else { throw LibraryError.message("Cannot read ZIP archive") }
|
||||
return String(decoding: bytes, as: UTF8.self)
|
||||
guard task.terminationStatus == 0 else {
|
||||
let detail = String(decoding:bytes,as:UTF8.self).trimmingCharacters(in:.whitespacesAndNewlines)
|
||||
throw LibraryError.message("Could not \(operation) (exit \(task.terminationStatus))."+(detail.isEmpty ? "" : " "+String(detail.prefix(600))))
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
}
|
||||
|
||||
struct MediaStorageLocation: Codable, Sendable {
|
||||
var path: String
|
||||
var bookmark: Data?
|
||||
var roots: [String]
|
||||
static let key = "media.storage.v1"
|
||||
}
|
||||
|
||||
enum ManagedMediaStorage {
|
||||
static func prepare(_ directory:URL) throws {
|
||||
if directory.path.hasPrefix("/Volumes/") {
|
||||
let components = directory.pathComponents
|
||||
guard components.count >= 3 else { throw LibraryError.message("Choose a folder on the external drive") }
|
||||
let mount = URL(fileURLWithPath:"/Volumes/"+components[2]).resolvingSymlinksInPath()
|
||||
let mounted = FileManager.default.mountedVolumeURLs(includingResourceValuesForKeys:nil,options:[]) ?? []
|
||||
guard mounted.contains(where:{$0.resolvingSymlinksInPath().path == mount.path}) else { throw LibraryError.message("Reconnect the media drive at \(mount.path). Imports will wait for that drive.") }
|
||||
}
|
||||
try FileManager.default.createDirectory(at:directory,withIntermediateDirectories:true)
|
||||
guard FileManager.default.isWritableFile(atPath:directory.path) else { throw LibraryError.message("The media folder is not writable: \(directory.path)") }
|
||||
}
|
||||
static func location(_ directory:URL,roots:[String]) -> MediaStorageLocation {
|
||||
let bookmark = (try? directory.bookmarkData(options:.withSecurityScope,includingResourceValuesForKeys:nil,relativeTo:nil)) ?? (try? directory.bookmarkData(options:.minimalBookmark,includingResourceValuesForKeys:nil,relativeTo:nil))
|
||||
return .init(path:directory.path,bookmark:bookmark,roots:Array(Set(roots+[directory.path])).sorted())
|
||||
}
|
||||
static func save(_ location:MediaStorageLocation,to library:Library) throws {
|
||||
try library.setSetting(MediaStorageLocation.key,String(decoding:try JSONEncoder().encode(location),as:UTF8.self))
|
||||
}
|
||||
static func hash(_ url:URL,cancelled:()->Bool) throws -> SHA256.Digest {
|
||||
let file = try FileHandle(forReadingFrom:url);defer { try? file.close() }
|
||||
var result = SHA256()
|
||||
while let data = try file.read(upToCount:1_048_576),!data.isEmpty { if cancelled() { throw CancellationError() };result.update(data:data) }
|
||||
return result.finalize()
|
||||
}
|
||||
static func move(database:URL,roots:[URL],destination:URL,cancelled:()->Bool = {false},progress:(ImportProgress)->Void = {_ in}) throws -> ImportProgress {
|
||||
try prepare(destination)
|
||||
let targetRoot = destination.resolvingSymlinksInPath()
|
||||
let sources = Set(roots.map{$0.resolvingSymlinksInPath()}).filter{$0 != targetRoot}
|
||||
guard !sources.contains(where:{targetRoot.path.hasPrefix($0.path+"/") || $0.path.hasPrefix(targetRoot.path+"/")}) else { throw LibraryError.message("Choose a separate media folder, outside the existing media folders") }
|
||||
let db = try Library(url:database),songs = try db.songs()
|
||||
var files = Set<URL>()
|
||||
for song in songs {
|
||||
for path in [song.audio,song.cdg].compactMap({$0}) {
|
||||
let file = URL(fileURLWithPath:path).resolvingSymlinksInPath()
|
||||
if sources.contains(file.deletingLastPathComponent()) { files.insert(file) }
|
||||
}
|
||||
}
|
||||
// Include managed hash-named remnants of previous cancelled imports.
|
||||
for root in sources where FileManager.default.fileExists(atPath:root.path) {
|
||||
for file in try FileManager.default.contentsOfDirectory(at:root,includingPropertiesForKeys:[.isRegularFileKey,.isSymbolicLinkKey]) {
|
||||
let stem = file.deletingPathExtension().lastPathComponent
|
||||
if [64,129].contains(stem.count),stem.allSatisfy({$0.isHexDigit || $0 == "-"}),["mp3","wav","m4a","mp4","m4v","mov","cdg"].contains(file.pathExtension.lowercased()) {
|
||||
let values = try file.resourceValues(forKeys:[.isRegularFileKey,.isSymbolicLinkKey])
|
||||
if values.isRegularFile == true,values.isSymbolicLink != true { files.insert(file.resolvingSymlinksInPath()) }
|
||||
}
|
||||
}
|
||||
}
|
||||
var state = ImportProgress(total:files.count),verified = [(URL,URL)](),lastReport = Date.distantPast
|
||||
defer { progress(state) }
|
||||
var sourceStamps = [URL:String]()
|
||||
func stamp(_ url:URL) throws -> String { let info = try FileManager.default.attributesOfItem(atPath:url.path);return "\(info[.size] ?? 0)|\(info[.modificationDate] ?? "")|\(info[.systemFileNumber] ?? 0)" }
|
||||
progress(state)
|
||||
for source in files.sorted(by:{$0.path < $1.path}) {
|
||||
if cancelled() { throw CancellationError() }
|
||||
state.current = "Copying "+source.lastPathComponent
|
||||
if Date().timeIntervalSince(lastReport) >= 0.1 { progress(state);lastReport = Date() }
|
||||
let target = targetRoot.appendingPathComponent(source.lastPathComponent)
|
||||
sourceStamps[source] = try stamp(source)
|
||||
let expected = try hash(source,cancelled:cancelled)
|
||||
if FileManager.default.fileExists(atPath:target.path) {
|
||||
guard try hash(target,cancelled:cancelled) == expected else { throw LibraryError.message("A different file already exists at \(target.path). Choose an empty folder or resolve the conflict first.") }
|
||||
state.skipped += 1
|
||||
} else {
|
||||
let temporary = targetRoot.appendingPathComponent(".kjmix-moving-"+UUID().uuidString)
|
||||
defer { try? FileManager.default.removeItem(at:temporary) }
|
||||
try FileManager.default.copyItem(at:source,to:temporary)
|
||||
guard try hash(temporary,cancelled:cancelled) == expected else { throw LibraryError.message("Verification failed for \(source.lastPathComponent). The original has been kept.") }
|
||||
try FileManager.default.moveItem(at:temporary,to:target)
|
||||
}
|
||||
verified.append((source,target));state.completed += 1
|
||||
if Date().timeIntervalSince(lastReport) >= 0.1 || state.completed == 1 || state.completed == state.total { progress(state);lastReport = Date() }
|
||||
}
|
||||
if cancelled() { throw CancellationError() }
|
||||
state.current = "Updating library links…";progress(state)
|
||||
let targets = Dictionary(uniqueKeysWithValues:verified.map { ($0.0.path,$0.1.path) })
|
||||
let studioURL = database.deletingLastPathComponent().appendingPathComponent("StudioSettings.json")
|
||||
if let data = try? Data(contentsOf:studioURL),var settings = try JSONSerialization.jsonObject(with:data) as? [String:Any],let background = settings["backgroundPath"] as? String,let target = targets[URL(fileURLWithPath:background).resolvingSymlinksInPath().path] {
|
||||
settings["backgroundPath"] = target;settings.removeValue(forKey:"backgroundBookmark")
|
||||
try JSONSerialization.data(withJSONObject:settings,options:[.sortedKeys]).write(to:studioURL,options:.atomic)
|
||||
}
|
||||
try db.execute("BEGIN IMMEDIATE")
|
||||
do {
|
||||
for song in try db.songs() {
|
||||
if let target = targets[URL(fileURLWithPath:song.audio).resolvingSymlinksInPath().path] { try db.execute("UPDATE songs SET audio=? WHERE id=?",[target,String(song.id)]) }
|
||||
if let graphics = song.cdg,let target = targets[URL(fileURLWithPath:graphics).resolvingSymlinksInPath().path] { try db.execute("UPDATE songs SET cdg=? WHERE id=?",[target,String(song.id)]) }
|
||||
}
|
||||
try save(location(targetRoot,roots:roots.map(\.path)),to:db)
|
||||
try db.execute("COMMIT")
|
||||
} catch { try? db.execute("ROLLBACK");throw error }
|
||||
// Every destination is verified and database links are committed before
|
||||
// any original is removed. A failed/cancelled copy keeps originals intact.
|
||||
var retained = 0
|
||||
for (source,_) in verified {
|
||||
do {
|
||||
guard try stamp(source) == sourceStamps[source] else { retained += 1;continue }
|
||||
try FileManager.default.removeItem(at:source)
|
||||
} catch { retained += 1 }
|
||||
}
|
||||
state.finished = true;state.current = "Media moved to "+targetRoot.path
|
||||
if retained > 0 { state.failure = "Media links were updated, but \(retained) original files could not be removed. You can retry the move to clean them up." }
|
||||
progress(state);return state
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user