749 lines
51 KiB
Swift
749 lines
51 KiB
Swift
import Foundation
|
|
import CryptoKit
|
|
import AVFoundation
|
|
import Darwin
|
|
|
|
struct ImportProgress: Sendable {
|
|
var completed = 0
|
|
var skipped = 0
|
|
var total = 0
|
|
var current = "Preparing import…"
|
|
var finished = false
|
|
var cancelled = false
|
|
var failure: String?
|
|
var logLines = [String]()
|
|
var reportLines: [String] {
|
|
if let failure { return logLines+["[FAILED] "+current+": "+failure] }
|
|
return logLines
|
|
}
|
|
var logText: String {
|
|
"Import results: \(completed-skipped) imported, \(skipped) skipped\n"+reportLines.joined(separator:"\n")
|
|
}
|
|
}
|
|
|
|
struct Importer {
|
|
let media: URL
|
|
var audioExtensions: Set<String> = ["mp3","wav","m4a","mp4","m4v","mov"]
|
|
var includeGraphics = true
|
|
var waveformCache: URL? = nil
|
|
var continueOnUnreadableAudio = false
|
|
var metadataTimeout: TimeInterval = 2
|
|
var metadataLoader: (@Sendable (URL) async throws -> [AVMetadataKey:String])? = nil
|
|
@discardableResult func run(_ urls: [URL],library: Library,copyMedia: Bool = true,cancelled: () -> Bool = { false },progress: (ImportProgress) -> Void = { _ in }) throws -> [Int64] {
|
|
if copyMedia { try ManagedMediaStorage.prepare(media) }
|
|
var state = ImportProgress()
|
|
var lastReport = Date.distantPast
|
|
func report(_ current: String,force: Bool = false) {
|
|
state.current = current
|
|
if force || Date().timeIntervalSince(lastReport) >= 0.1 { progress(state);lastReport = Date() }
|
|
}
|
|
report("Preparing import…",force:true)
|
|
// Index each directory once: a large folder must not be searched for every song.
|
|
var siblingCache = [URL:[String:[String:URL]]]()
|
|
func companions(of url: URL) throws -> [String:URL] {
|
|
let directory = url.deletingLastPathComponent()
|
|
if siblingCache[directory] == nil {
|
|
let entries = try FileManager.default.contentsOfDirectory(at:directory,includingPropertiesForKeys:nil)
|
|
var index = [String:[String:URL]]()
|
|
for entry in entries {
|
|
if cancelled() { throw CancellationError() }
|
|
index[entry.deletingPathExtension().lastPathComponent.lowercased(),default:[:]][entry.pathExtension.lowercased()] = entry
|
|
}
|
|
siblingCache[directory] = index
|
|
}
|
|
return siblingCache[directory]?[url.deletingPathExtension().lastPathComponent.lowercased()] ?? [:]
|
|
}
|
|
defer { state.total = max(state.total,state.completed);progress(state) }
|
|
// Discover before copying: overlapping selections must not import twice,
|
|
// and a destination inside the selected tree must not feed itself back in.
|
|
let keys: Set<URLResourceKey> = [.isDirectoryKey,.isRegularFileKey,.isSymbolicLinkKey,.isPackageKey]
|
|
let supported = audioExtensions.union(includeGraphics ? ["cdg","zip"] : ["zip"])
|
|
let managedRoot = media.resolvingSymlinksInPath().standardizedFileURL
|
|
var visited = Set<URL>(),discovered = Set<URL>()
|
|
func discover(_ input:URL) throws {
|
|
if cancelled() { throw CancellationError() }
|
|
let url = input.standardizedFileURL
|
|
guard visited.insert(url).inserted else { return }
|
|
do {
|
|
let values = try url.resourceValues(forKeys:keys)
|
|
// Do not follow directory symlinks (cycles or files outside the selection).
|
|
guard values.isSymbolicLink != true else { return }
|
|
if values.isDirectory == true {
|
|
guard values.isPackage != true,url.resolvingSymlinksInPath().standardizedFileURL.path != managedRoot.path else { return }
|
|
report("Scanning "+url.path)
|
|
for child in try FileManager.default.contentsOfDirectory(at:url,includingPropertiesForKeys:Array(keys),options:[.skipsHiddenFiles]) {
|
|
try discover(child)
|
|
}
|
|
} else if values.isRegularFile == true,supported.contains(url.pathExtension.lowercased()) {
|
|
discovered.insert(url)
|
|
report("Scanning · \(discovered.count.formatted()) supported files found · \(url.lastPathComponent)")
|
|
}
|
|
} catch is CancellationError { throw CancellationError() }
|
|
catch { throw LibraryError.message("Could not scan \(url.path): \(error.localizedDescription)") }
|
|
}
|
|
report("Scanning selected files and folders…",force:true)
|
|
for url in urls { try discover(url) }
|
|
var files = discovered.sorted { $0.path < $1.path }
|
|
var temporary = [URL]()
|
|
defer { for url in temporary { try? FileManager.default.removeItem(at: url) } }
|
|
// Older libraries may contain indexed tracks without stored fingerprints.
|
|
// Compare only same-size candidates, and hash each legacy file at most once.
|
|
var legacyBySize = [UInt64:[Song]]()
|
|
var existingChecked = 0
|
|
for song in try library.songs() where song.fingerprint.isEmpty {
|
|
if cancelled() { throw CancellationError() }
|
|
existingChecked += 1;report("Checking existing library · \(existingChecked.formatted()) tracks · \(song.title)")
|
|
if let info = try? FileManager.default.attributesOfItem(atPath:song.audio),let size = info[.size] as? NSNumber,size.uint64Value > 0 {
|
|
legacyBySize[size.uint64Value,default:[]].append(song)
|
|
}
|
|
}
|
|
var legacyHashes = [Int64:String]()
|
|
func duplicateWithFingerprint(_ hash: String,audio: URL) throws -> Song? {
|
|
if let song = try library.songWithFingerprint(hash) { return song }
|
|
let size = (try FileManager.default.attributesOfItem(atPath:audio.path)[.size] as? NSNumber)?.uint64Value ?? 0
|
|
for song in legacyBySize[size] ?? [] {
|
|
if cancelled() { throw CancellationError() }
|
|
let candidate: String
|
|
if let known = legacyHashes[song.id] { candidate = known }
|
|
else {
|
|
do { candidate = try audioFingerprint(URL(fileURLWithPath:song.audio),cancelled:cancelled) }
|
|
catch is CancellationError { throw CancellationError() }
|
|
catch { continue } // An offline legacy entry must not prevent a new import.
|
|
legacyHashes[song.id] = candidate
|
|
try library.execute("UPDATE songs SET fingerprint=? WHERE id=? AND fingerprint=''",[candidate,String(song.id)])
|
|
}
|
|
if candidate == hash { return song }
|
|
}
|
|
return nil
|
|
}
|
|
var songIDs = Set<Int64>()
|
|
var archives = [(url:URL,signature:String,folder:URL)]()
|
|
var archiveForFile = [String:URL]()
|
|
var archiveSongIDs = [URL:[Int64]]()
|
|
var archiveOrigins = [URL:URL]()
|
|
func importSignature(_ audio: URL,cdg: URL? = nil) throws -> String {
|
|
func signature(_ url: URL) throws -> String {
|
|
let attrs = try FileManager.default.attributesOfItem(atPath:url.path)
|
|
return "\(url.standardizedFileURL.path)|\(attrs[.size] ?? 0)|\((attrs[.modificationDate] as? Date)?.timeIntervalSince1970 ?? 0)|\((attrs[.creationDate] as? Date)?.timeIntervalSince1970 ?? 0)|\(attrs[.systemFileNumber] ?? 0)"
|
|
}
|
|
return try signature(audio)+"|"+(cdg.map { try signature($0) } ?? "no-cdg")+"|copy=\(copyMedia)"
|
|
}
|
|
func sourcePath(_ audio: URL) -> String {
|
|
if let folder = archiveForFile[audio.resolvingSymlinksInPath().path],let zip = archiveOrigins[folder] {
|
|
return zip.path+"/"+String(audio.resolvingSymlinksInPath().path.dropFirst(folder.resolvingSymlinksInPath().path.count+1))
|
|
}
|
|
return audio.path
|
|
}
|
|
func remember(_ audio:URL,_ signature:String,_ id:Int64) throws {
|
|
try library.recordOriginalName(song:id,name:audio.deletingPathExtension().lastPathComponent,path:sourcePath(audio))
|
|
if let archive = archiveForFile[audio.resolvingSymlinksInPath().path] { archiveSongIDs[archive,default:[]].append(id) }
|
|
else { try library.rememberImport(path:audio.standardizedFileURL.path,signature:signature,songs:[id]) }
|
|
}
|
|
for zip in files where zip.pathExtension.lowercased() == "zip" {
|
|
if cancelled() { throw CancellationError() }
|
|
let signature = try importSignature(zip)
|
|
if let existing = try library.importedSongs(path:zip.standardizedFileURL.path,signature:signature),try library.originalNamesRecorded(existing) {
|
|
songIDs.formUnion(existing);state.completed += existing.count;state.skipped += existing.count
|
|
for id in existing {
|
|
state.logLines.append("[SKIPPED] "+zip.path+" → "+((try library.song(id))?.title ?? String(id))+" (already imported)")
|
|
}
|
|
report("Already imported: "+zip.lastPathComponent,force:state.completed == existing.count);continue
|
|
}
|
|
report("Extracting "+zip.lastPathComponent,force:archives.isEmpty)
|
|
let folder = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
|
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
|
|
temporary.append(folder)
|
|
archives.append((zip,signature,folder));archiveOrigins[folder] = zip
|
|
// Validate names before extraction; reject path traversal and symlinks afterwards.
|
|
let details = try process(["-Z", "-l", zip.path])
|
|
guard !details.split(separator: "\n").contains(where: { $0.hasPrefix("l") }) else { throw LibraryError.message("ZIP symlinks are unsupported") }
|
|
let list = try processData(["-Z1", zip.path])
|
|
let rawNames = list.split(separator:0x0A).map { Data($0) }
|
|
let names = rawNames.map { String(decoding:$0,as:UTF8.self) }
|
|
guard !names.isEmpty, names.allSatisfy({ !$0.hasPrefix("/") && !$0.split(separator: "/").contains("..") && !$0.contains("\\") }) else { throw LibraryError.message("Unsafe or empty ZIP") }
|
|
// ditto maps legacy ZIP filename bytes to valid macOS names. unzip can
|
|
// fail with exit 50 ("disk full") on those names even when CRCs pass.
|
|
_ = try process(["-x", "-k", "--noqtn", zip.path, folder.path],executable:"/usr/bin/ditto",operation:"extract ZIP archive")
|
|
// Unflagged older Windows archives commonly use CP1252. ditto makes
|
|
// their names filesystem-safe using Mac Roman; rename those components
|
|
// to Unicode CP1252 spellings before matching MP3/CDG companions.
|
|
var renamedComponents = [String:String]()
|
|
for rawName in try archiveNames(zip) {
|
|
var original = [String]()
|
|
for rawComponent in rawName.split(separator:0x2F) {
|
|
let bytes = Data(rawComponent)
|
|
if let unicode = String(data:bytes,encoding:.utf8) { original.append(unicode);continue }
|
|
guard let native = String(data:bytes,encoding:.macOSRoman),
|
|
let unicode = String(data:bytes,encoding:.windowsCP1252) ?? String(data:bytes,encoding:.isoLatin1) else { continue }
|
|
original.append(native)
|
|
renamedComponents[original.joined(separator:"/")] = unicode.precomposedStringWithCanonicalMapping
|
|
}
|
|
}
|
|
if !renamedComponents.isEmpty {
|
|
let rootPath = folder.resolvingSymlinksInPath().path
|
|
let entries = FileManager.default.enumerator(at:folder,includingPropertiesForKeys:[.isSymbolicLinkKey])!
|
|
var pending = [(URL,String)]()
|
|
for case let entry as URL in entries {
|
|
guard try entry.resourceValues(forKeys:[.isSymbolicLinkKey]).isSymbolicLink != true else { throw LibraryError.message("ZIP symlinks are unsupported") }
|
|
let path = entry.resolvingSymlinksInPath().path
|
|
guard path.hasPrefix(rootPath+"/") else { throw LibraryError.message("Unsafe ZIP path") }
|
|
let relative = String(path.dropFirst(rootPath.count+1))
|
|
if let corrected = renamedComponents[relative] { pending.append((entry,corrected)) }
|
|
}
|
|
// Children move before their parents, keeping nested paths valid.
|
|
for (entry,corrected) in pending.sorted(by:{$0.0.pathComponents.count > $1.0.pathComponents.count}) {
|
|
if cancelled() { throw CancellationError() }
|
|
let target = entry.deletingLastPathComponent().appendingPathComponent(corrected)
|
|
if entry.lastPathComponent != corrected { try FileManager.default.moveItem(at:entry,to:target) }
|
|
}
|
|
}
|
|
let extracted = FileManager.default.enumerator(at: folder, includingPropertiesForKeys: [.isSymbolicLinkKey,.isRegularFileKey])!
|
|
for case let url as URL in extracted {
|
|
let values = try url.resourceValues(forKeys: [.isSymbolicLinkKey,.isRegularFileKey])
|
|
guard values.isSymbolicLink != true else { throw LibraryError.message("ZIP symlinks are unsupported") }
|
|
if values.isRegularFile == true { files.append(url);archiveForFile[url.resolvingSymlinksInPath().path] = folder }
|
|
}
|
|
}
|
|
for cdg in files.filter({ includeGraphics && $0.pathExtension.lowercased() == "cdg" }) {
|
|
if cancelled() { throw CancellationError() }
|
|
let matching = try companions(of:cdg)
|
|
if let audio = (["mp3","wav","m4a","mp4","m4v","mov"]+audioExtensions.subtracting(["mp3","wav","m4a","mp4","m4v","mov"]).sorted()).compactMap({ matching[$0] }).first { files.append(audio) }
|
|
}
|
|
var imported = 0
|
|
let audioFiles = Set(files.map { $0.standardizedFileURL }).filter { audioExtensions.contains($0.pathExtension.lowercased()) }.sorted { $0.path < $1.path }
|
|
state.total = audioFiles.count+state.completed
|
|
report("Importing tracks…",force:true)
|
|
for audio in audioFiles {
|
|
if cancelled() { throw CancellationError() }
|
|
report(audio.lastPathComponent)
|
|
let cdg: URL?
|
|
if includeGraphics { cdg = try companions(of:audio)["cdg"] } else { cdg = nil }
|
|
let archive = archiveForFile[audio.resolvingSymlinksInPath().path]
|
|
let copy = copyMedia || archive != nil
|
|
if copy { try ManagedMediaStorage.prepare(media) }
|
|
let signature = try importSignature(audio,cdg:cdg)
|
|
if let existing = try library.importedSongs(path:audio.standardizedFileURL.path,signature:signature) {
|
|
songIDs.formUnion(existing)
|
|
for id in existing { try remember(audio,signature,id) }
|
|
state.completed += 1;state.skipped += 1
|
|
state.logLines.append("[SKIPPED] "+sourcePath(audio)+" (already imported)")
|
|
report("Already imported: "+audio.lastPathComponent,force:state.completed == 1);continue
|
|
}
|
|
let hash: String
|
|
let graphicsHash: String?
|
|
do {
|
|
guard MediaByteCopy.nonempty(audio) else { throw LibraryError.message("Audio file is empty or unreadable: "+audio.path) }
|
|
report("Checking audio: "+audio.lastPathComponent)
|
|
if continueOnUnreadableAudio { _ = try AVAudioFile(forReading:audio) }
|
|
hash = try audioFingerprint(audio,cancelled:cancelled,progress:{ bytes in
|
|
report("Reading "+audio.lastPathComponent+" · "+ByteCountFormatter.string(fromByteCount:bytes,countStyle:.file))
|
|
})
|
|
graphicsHash = try cdg.map { try audioFingerprint($0,cancelled:cancelled) }
|
|
} catch is CancellationError { throw CancellationError() }
|
|
catch {
|
|
guard continueOnUnreadableAudio else { throw error }
|
|
state.completed += 1;state.skipped += 1
|
|
state.logLines.append("[SKIPPED] "+sourcePath(audio)+" (unreadable audio: "+error.localizedDescription+")")
|
|
report("Skipped unreadable file: "+audio.lastPathComponent,force:state.completed == 1)
|
|
continue
|
|
}
|
|
let duplicate = try duplicateWithFingerprint(hash,audio:audio)
|
|
var sameGraphics = cdg == nil
|
|
if let graphicsHash,let stored = duplicate?.cdg,FileManager.default.fileExists(atPath:stored) {
|
|
sameGraphics = try audioFingerprint(URL(fileURLWithPath:stored),cancelled:cancelled) == graphicsHash
|
|
}
|
|
if let existing = duplicate,sameGraphics,MediaByteCopy.nonempty(URL(fileURLWithPath:existing.audio)),
|
|
(archive == nil || !copy || URL(fileURLWithPath:existing.audio).deletingLastPathComponent().standardizedFileURL == media.standardizedFileURL) {
|
|
try remember(audio,signature,existing.id);songIDs.insert(existing.id)
|
|
state.completed += 1;state.skipped += 1
|
|
state.logLines.append("[SKIPPED] "+sourcePath(audio)+" (identical content already in library)")
|
|
report("Already in library: "+audio.lastPathComponent,force:state.completed == 1);continue
|
|
}
|
|
let dest = copy ? media.appendingPathComponent(hash+"."+audio.pathExtension.lowercased()) : audio.standardizedFileURL
|
|
if copy {
|
|
report("Copying: "+audio.lastPathComponent)
|
|
try MediaByteCopy.copy(audio,to:dest,expectedHash:hash,cancelled:cancelled)
|
|
}
|
|
var graphics: String?
|
|
if let cdg {
|
|
let target = copy ? media.appendingPathComponent(hash+"-"+(graphicsHash ?? "graphics")+".cdg") : cdg.standardizedFileURL
|
|
if copy { try MediaByteCopy.copy(cdg,to:target,expectedHash:graphicsHash!,cancelled:cancelled) }
|
|
graphics = target.path
|
|
}
|
|
imported += 1
|
|
report("Reading tags: "+audio.lastPathComponent,force:imported == 1)
|
|
let metadata = try audioMetadata(audio,cancelled:cancelled)
|
|
if metadata == nil { state.logLines.append("[WARNING] "+sourcePath(audio)+" (tag read timed out; imported using its filename)") }
|
|
func tag(_ key: AVMetadataKey) -> String { metadata?[key] ?? "" }
|
|
let taggedTitle = tag(.commonKeyTitle)
|
|
let duration = audioDuration(audio)
|
|
report("Saving track: "+audio.lastPathComponent)
|
|
try library.execute("SAVEPOINT kjmix_import_track")
|
|
let id: Int64
|
|
do {
|
|
id = try library.add(title:taggedTitle.isEmpty ? audio.deletingPathExtension().lastPathComponent : taggedTitle,audio:dest.path,cdg:graphics,artist:tag(.commonKeyArtist),album:tag(.commonKeyAlbumName),label:tag(.commonKeyPublisher),duration:duration,fingerprint:hash,preferIncomingFile:copy)
|
|
try remember(audio,signature,id)
|
|
try library.execute("RELEASE SAVEPOINT kjmix_import_track")
|
|
} catch {
|
|
try? library.execute("ROLLBACK TO SAVEPOINT kjmix_import_track")
|
|
try? library.execute("RELEASE SAVEPOINT kjmix_import_track")
|
|
throw error
|
|
}
|
|
songIDs.insert(id)
|
|
state.completed += 1
|
|
state.logLines.append("[IMPORTED] "+sourcePath(audio))
|
|
if let waveformCache {
|
|
report("Generating waveform: "+audio.lastPathComponent,force:true)
|
|
do {
|
|
if (try? WaveformCache.load(audio:dest,directory:waveformCache)) == nil {
|
|
let waveform = try AudioWaveform.analyse(url:dest,cancelled:cancelled)
|
|
try WaveformCache.save(waveform,audio:dest,directory:waveformCache)
|
|
}
|
|
} catch is CancellationError { throw CancellationError() }
|
|
catch { state.logLines.append("[WARNING] Waveform unavailable for "+sourcePath(audio)+": "+error.localizedDescription) }
|
|
}
|
|
report(audio.lastPathComponent,force:state.completed == 1)
|
|
}
|
|
for archive in archives {
|
|
let ids = archiveSongIDs[archive.folder] ?? []
|
|
try library.rememberImport(path:archive.url.standardizedFileURL.path,signature:archive.signature,songs:ids)
|
|
}
|
|
guard imported > 0 || state.skipped > 0 else { throw LibraryError.message("No audio found. Keep the MP3 and matching CDG together, or import an MP3+G ZIP.") }
|
|
state.finished = true;report("Import complete",force:true)
|
|
return Array(songIDs)
|
|
}
|
|
private func audioFingerprint(_ url: URL,cancelled: () -> Bool,progress: (Int64) -> Void = {_ in}) throws -> String {
|
|
let file = try FileHandle(forReadingFrom:url);defer { try? file.close() }
|
|
var hasher = SHA256()
|
|
var bytes: Int64 = 0
|
|
while true {
|
|
if cancelled() { throw CancellationError() }
|
|
guard let data = try file.read(upToCount:1_048_576),!data.isEmpty else { break }
|
|
hasher.update(data:data);bytes += Int64(data.count);progress(bytes)
|
|
}
|
|
return hasher.finalize().map { String(format:"%02x",$0) }.joined()
|
|
}
|
|
/// The importer runs on its worker queue. Bridge asynchronous tag loading here
|
|
/// so the existing synchronous transaction can retain its cancellation semantics.
|
|
private func audioMetadata(_ url: URL,cancelled: () -> Bool) throws -> [AVMetadataKey:String]? {
|
|
if cancelled() { throw CancellationError() }
|
|
let result = ImportMetadataResult(),ready = DispatchSemaphore(value:0)
|
|
let asset = AVURLAsset(url:url),loader = metadataLoader
|
|
let task = Task.detached(priority:.utility) {
|
|
defer { ready.signal() }
|
|
if let loader { result.set((try? await loader(url)) ?? [:]);return }
|
|
var tags = [AVMetadataKey:String]()
|
|
let items = (try? await asset.load(.commonMetadata)) ?? []
|
|
for item in items {
|
|
if Task.isCancelled { break }
|
|
guard let key = item.commonKey,[AVMetadataKey.commonKeyTitle,.commonKeyArtist,.commonKeyAlbumName,.commonKeyPublisher].contains(key),tags[key] == nil else { continue }
|
|
if let value = try? await item.load(.stringValue) { tags[key] = value }
|
|
}
|
|
result.set(tags)
|
|
}
|
|
defer { task.cancel();asset.cancelLoading() }
|
|
let deadline = ProcessInfo.processInfo.systemUptime+max(0.01,metadataTimeout)
|
|
while ready.wait(timeout:.now()+0.02) == .timedOut {
|
|
if cancelled() { throw CancellationError() }
|
|
if ProcessInfo.processInfo.systemUptime >= deadline { return nil }
|
|
}
|
|
if cancelled() { throw CancellationError() }
|
|
return result.get()
|
|
}
|
|
private func audioDuration(_ url: URL) -> Double {
|
|
guard let file = try? AVAudioFile(forReading:url) else { return 0 }
|
|
return Double(file.length)/file.processingFormat.sampleRate
|
|
}
|
|
/// Read original central-directory bytes: zipinfo translates legacy names
|
|
/// for the terminal, so its output cannot be used to rename extracted files.
|
|
private func archiveNames(_ url:URL) throws -> [Data] {
|
|
let file = try FileHandle(forReadingFrom:url);defer { try? file.close() }
|
|
let size = try file.seekToEnd(),tailSize = min(size,65_557)
|
|
try file.seek(toOffset:size-tailSize)
|
|
let tail = [UInt8](try file.readToEnd() ?? Data())
|
|
func integer(_ bytes:[UInt8],_ offset:Int,_ length:Int) -> UInt64 {
|
|
guard offset >= 0,offset+length <= bytes.count else { return 0 }
|
|
return (0..<length).reduce(0) { $0 | UInt64(bytes[offset+$1]) << (8*$1) }
|
|
}
|
|
guard tail.count >= 22,let end = stride(from:tail.count-22,through:0,by:-1).first(where:{integer(tail,$0,4) == 0x06054b50 && $0+22+Int(integer(tail,$0+20,2)) == tail.count}) else { throw LibraryError.message("Invalid ZIP directory") }
|
|
var count = integer(tail,end+10,2),offset = integer(tail,end+16,4)
|
|
if count == 0xFFFF || offset == 0xFFFFFFFF {
|
|
guard end >= 20,integer(tail,end-20,4) == 0x07064b50 else { throw LibraryError.message("Invalid ZIP64 directory") }
|
|
try file.seek(toOffset:integer(tail,end-12,8))
|
|
let record = [UInt8](try file.read(upToCount:56) ?? Data())
|
|
guard record.count == 56,integer(record,0,4) == 0x06064b50 else { throw LibraryError.message("Invalid ZIP64 directory") }
|
|
count = integer(record,32,8);offset = integer(record,48,8)
|
|
}
|
|
guard offset <= size,count <= size/46 else { throw LibraryError.message("Invalid ZIP directory bounds") }
|
|
var names = [Data]()
|
|
for _ in 0..<count {
|
|
try file.seek(toOffset:offset)
|
|
let header = [UInt8](try file.read(upToCount:46) ?? Data())
|
|
guard header.count == 46,integer(header,0,4) == 0x02014b50 else { throw LibraryError.message("Invalid ZIP entry") }
|
|
let length = integer(header,28,2),extra = integer(header,30,2),comment = integer(header,32,2)
|
|
guard offset+46+length+extra+comment <= size else { throw LibraryError.message("Invalid ZIP entry bounds") }
|
|
let name = try file.read(upToCount:Int(length)) ?? Data()
|
|
guard name.count == Int(length) else { throw LibraryError.message("Incomplete ZIP filename") }
|
|
names.append(name);offset += 46+length+extra+comment
|
|
}
|
|
return names
|
|
}
|
|
private func process(_ arguments: [String],executable: String = "/usr/bin/unzip",operation: String = "read ZIP archive") throws -> String {
|
|
String(decoding:try processData(arguments,executable:executable,operation:operation),as:UTF8.self)
|
|
}
|
|
private func processData(_ arguments: [String],executable: String = "/usr/bin/unzip",operation: String = "read ZIP archive") throws -> Data {
|
|
let task = Process(); task.executableURL = URL(fileURLWithPath:executable); task.arguments = arguments
|
|
let out = Pipe(); task.standardOutput = out
|
|
task.standardError = out
|
|
try task.run()
|
|
let bytes = out.fileHandleForReading.readDataToEndOfFile(); task.waitUntilExit()
|
|
guard task.terminationStatus == 0 else {
|
|
let detail = String(decoding:bytes,as:UTF8.self).trimmingCharacters(in:.whitespacesAndNewlines)
|
|
throw LibraryError.message("Could not \(operation) (exit \(task.terminationStatus))."+(detail.isEmpty ? "" : " "+String(detail.prefix(600))))
|
|
}
|
|
return bytes
|
|
}
|
|
}
|
|
|
|
struct MediaStorageLocation: Codable, Sendable {
|
|
var path: String
|
|
var bookmark: Data?
|
|
var roots: [String]
|
|
static let key = "media.storage.v1"
|
|
}
|
|
|
|
enum ManagedMediaStorage {
|
|
static func prepare(_ directory:URL) throws {
|
|
if directory.path.hasPrefix("/Volumes/") {
|
|
let components = directory.pathComponents
|
|
guard components.count >= 3 else { throw LibraryError.message("Choose a folder on the external drive") }
|
|
let mount = URL(fileURLWithPath:"/Volumes/"+components[2]).resolvingSymlinksInPath()
|
|
let mounted = FileManager.default.mountedVolumeURLs(includingResourceValuesForKeys:nil,options:[]) ?? []
|
|
guard mounted.contains(where:{$0.resolvingSymlinksInPath().path == mount.path}) else { throw LibraryError.message("Reconnect the media drive at \(mount.path). Imports will wait for that drive.") }
|
|
}
|
|
try FileManager.default.createDirectory(at:directory,withIntermediateDirectories:true)
|
|
guard FileManager.default.isWritableFile(atPath:directory.path) else { throw LibraryError.message("The media folder is not writable: \(directory.path)") }
|
|
}
|
|
static func location(_ directory:URL,roots:[String]) -> MediaStorageLocation {
|
|
let bookmark = (try? directory.bookmarkData(options:.withSecurityScope,includingResourceValuesForKeys:nil,relativeTo:nil)) ?? (try? directory.bookmarkData(options:.minimalBookmark,includingResourceValuesForKeys:nil,relativeTo:nil))
|
|
return .init(path:directory.path,bookmark:bookmark,roots:Array(Set(roots+[directory.path])).sorted())
|
|
}
|
|
static func save(_ location:MediaStorageLocation,to library:Library) throws {
|
|
try library.setSetting(MediaStorageLocation.key,String(decoding:try JSONEncoder().encode(location),as:UTF8.self))
|
|
}
|
|
static func hash(_ url:URL,cancelled:()->Bool,bytesRead:(Int64)->Void = {_ in}) throws -> SHA256.Digest {
|
|
let file = try FileHandle(forReadingFrom:url);defer { try? file.close() }
|
|
var result = SHA256(),total: Int64 = 0
|
|
while let data = try file.read(upToCount:1_048_576),!data.isEmpty { if cancelled() { throw CancellationError() };result.update(data:data);total += Int64(data.count);bytesRead(total) }
|
|
return result.finalize()
|
|
}
|
|
static func move(database:URL,roots:[URL],destination:URL,cancelled:()->Bool = {false},progress:(ImportProgress)->Void = {_ in}) throws -> ImportProgress {
|
|
var state = ImportProgress(current:"Checking media locations…"),lastReport = Date.distantPast
|
|
func scan(_ text: String) throws {
|
|
if cancelled() { throw CancellationError() }
|
|
if Date().timeIntervalSince(lastReport) >= 0.1 { state.current = text;progress(state);lastReport = Date() }
|
|
if cancelled() { throw CancellationError() }
|
|
}
|
|
try scan("Checking media locations…")
|
|
try prepare(destination)
|
|
let targetRoot = destination.resolvingSymlinksInPath()
|
|
let sources = Set(roots.map{$0.resolvingSymlinksInPath()}).filter{$0 != targetRoot}
|
|
guard !sources.contains(where:{targetRoot.path.hasPrefix($0.path+"/") || $0.path.hasPrefix(targetRoot.path+"/")}) else { throw LibraryError.message("Choose a separate media folder, outside the existing media folders") }
|
|
let db = try Library(url:database),songs = try db.songs()
|
|
var files = Set<URL>()
|
|
for song in songs {
|
|
try scan("Finding media: "+song.title)
|
|
for path in [song.audio,song.cdg].compactMap({$0}) {
|
|
let file = URL(fileURLWithPath:path).resolvingSymlinksInPath()
|
|
if sources.contains(file.deletingLastPathComponent()) { files.insert(file) }
|
|
}
|
|
}
|
|
// Include managed hash-named remnants of previous cancelled imports.
|
|
for root in sources where FileManager.default.fileExists(atPath:root.path) {
|
|
try scan("Checking folder: "+root.path)
|
|
for file in try FileManager.default.contentsOfDirectory(at:root,includingPropertiesForKeys:[.isRegularFileKey,.isSymbolicLinkKey]) {
|
|
try scan("Checking media: "+file.lastPathComponent)
|
|
let stem = file.deletingPathExtension().lastPathComponent
|
|
if [64,129].contains(stem.count),stem.allSatisfy({$0.isHexDigit || $0 == "-"}),["mp3","wav","m4a","mp4","m4v","mov","cdg"].contains(file.pathExtension.lowercased()) {
|
|
let values = try file.resourceValues(forKeys:[.isRegularFileKey,.isSymbolicLinkKey])
|
|
if values.isRegularFile == true,values.isSymbolicLink != true { files.insert(file.resolvingSymlinksInPath()) }
|
|
}
|
|
}
|
|
}
|
|
state.total = files.count
|
|
var verified = [(URL,URL)]()
|
|
defer { progress(state) }
|
|
var sourceStamps = [URL:String]()
|
|
func stamp(_ url:URL) throws -> String { let info = try FileManager.default.attributesOfItem(atPath:url.path);return "\(info[.size] ?? 0)|\(info[.modificationDate] ?? "")|\(info[.systemFileNumber] ?? 0)" }
|
|
progress(state)
|
|
for source in files.sorted(by:{$0.path < $1.path}) {
|
|
if cancelled() { throw CancellationError() }
|
|
state.current = "Copying "+source.lastPathComponent
|
|
if Date().timeIntervalSince(lastReport) >= 0.1 { progress(state);lastReport = Date() }
|
|
let target = targetRoot.appendingPathComponent(source.lastPathComponent)
|
|
sourceStamps[source] = try stamp(source)
|
|
let expected = try hash(source,cancelled:cancelled,bytesRead:{ bytes in
|
|
if Date().timeIntervalSince(lastReport) >= 0.1 { state.current = "Verifying "+source.lastPathComponent+" · "+ByteCountFormatter.string(fromByteCount:bytes,countStyle:.file);progress(state);lastReport = Date() }
|
|
})
|
|
if FileManager.default.fileExists(atPath:target.path) {
|
|
guard try hash(target,cancelled:cancelled) == expected else { throw LibraryError.message("A different file already exists at \(target.path). Choose an empty folder or resolve the conflict first.") }
|
|
state.skipped += 1
|
|
} else {
|
|
try MediaByteCopy.copy(source,to:target,expectedHash:expected.map{String(format:"%02x",$0)}.joined(),cancelled:cancelled,bytesCopied:{ bytes in
|
|
if Date().timeIntervalSince(lastReport) >= 0.1 { state.current = "Copying "+source.lastPathComponent+" · "+ByteCountFormatter.string(fromByteCount:bytes,countStyle:.file);progress(state);lastReport = Date() }
|
|
})
|
|
}
|
|
verified.append((source,target));state.completed += 1;state.logLines.append("[COPIED] "+source.path)
|
|
if Date().timeIntervalSince(lastReport) >= 0.1 || state.completed == 1 || state.completed == state.total { progress(state);lastReport = Date() }
|
|
}
|
|
if cancelled() { throw CancellationError() }
|
|
state.current = "Updating library links…";progress(state)
|
|
let targets = Dictionary(uniqueKeysWithValues:verified.map { ($0.0.path,$0.1.path) })
|
|
let studioURL = database.deletingLastPathComponent().appendingPathComponent("StudioSettings.json")
|
|
if let data = try? Data(contentsOf:studioURL),var settings = try JSONSerialization.jsonObject(with:data) as? [String:Any],let background = settings["backgroundPath"] as? String,let target = targets[URL(fileURLWithPath:background).resolvingSymlinksInPath().path] {
|
|
settings["backgroundPath"] = target;settings.removeValue(forKey:"backgroundBookmark")
|
|
try JSONSerialization.data(withJSONObject:settings,options:[.sortedKeys]).write(to:studioURL,options:.atomic)
|
|
}
|
|
try db.execute("BEGIN IMMEDIATE")
|
|
do {
|
|
for song in try db.songs() {
|
|
if let target = targets[URL(fileURLWithPath:song.audio).resolvingSymlinksInPath().path] { try db.execute("UPDATE songs SET audio=? WHERE id=?",[target,String(song.id)]) }
|
|
if let graphics = song.cdg,let target = targets[URL(fileURLWithPath:graphics).resolvingSymlinksInPath().path] { try db.execute("UPDATE songs SET cdg=? WHERE id=?",[target,String(song.id)]) }
|
|
}
|
|
try save(location(targetRoot,roots:roots.map(\.path)),to:db)
|
|
try db.execute("COMMIT")
|
|
} catch { try? db.execute("ROLLBACK");throw error }
|
|
// Every destination is verified and database links are committed before
|
|
// any original is removed. A failed/cancelled copy keeps originals intact.
|
|
var retained = 0
|
|
for (source,_) in verified {
|
|
do {
|
|
guard try stamp(source) == sourceStamps[source] else { retained += 1;continue }
|
|
try FileManager.default.removeItem(at:source)
|
|
} catch { retained += 1 }
|
|
}
|
|
state.finished = true;state.current = "Media moved to "+targetRoot.path
|
|
if retained > 0 { state.failure = "Media links were updated, but \(retained) original files could not be removed. You can retry the move to clean them up." }
|
|
progress(state);return state
|
|
}
|
|
}
|
|
|
|
/// Copy media content, not source ACLs, flags or private security-scope attributes.
|
|
/// A failed/cancelled transfer never publishes a partial destination.
|
|
enum MediaByteCopy {
|
|
static func nonempty(_ url: URL) -> Bool {
|
|
guard let values = try? url.resourceValues(forKeys:[.fileSizeKey,.isRegularFileKey]),values.isRegularFile == true else { return false }
|
|
return (values.fileSize ?? 0) > 0
|
|
}
|
|
static func digest(_ url: URL,cancelled: () -> Bool = { false }) throws -> String {
|
|
let input = try FileHandle(forReadingFrom:url);defer { try? input.close() }
|
|
var hash = SHA256()
|
|
while true {
|
|
if cancelled() { throw CancellationError() }
|
|
guard let bytes = try input.read(upToCount:1_048_576),!bytes.isEmpty else { break }
|
|
hash.update(data:bytes)
|
|
}
|
|
return hash.finalize().map{String(format:"%02x",$0)}.joined()
|
|
}
|
|
static func copy(_ source: URL,to destination: URL,expectedHash: String,cancelled: () -> Bool = { false },bytesCopied: (Int64) -> Void = {_ in}) throws {
|
|
if cancelled() { throw CancellationError() }
|
|
if nonempty(destination),try digest(destination,cancelled:cancelled) == expectedHash { return }
|
|
let temporary = destination.deletingLastPathComponent().appendingPathComponent(".kjmix-import-"+UUID().uuidString)
|
|
defer { try? FileManager.default.removeItem(at:temporary) }
|
|
do {
|
|
let input = try FileHandle(forReadingFrom:source);defer { try? input.close() }
|
|
guard FileManager.default.createFile(atPath:temporary.path,contents:nil,attributes:[.posixPermissions:0o644]) else { throw LibraryError.message("Cannot create a file in "+destination.deletingLastPathComponent().path) }
|
|
let output = try FileHandle(forWritingTo:temporary);defer { try? output.close() }
|
|
var bytesWritten = 0
|
|
while true {
|
|
if cancelled() { throw CancellationError() }
|
|
guard let data = try input.read(upToCount:1_048_576),!data.isEmpty else { break }
|
|
try output.write(contentsOf:data);bytesWritten += data.count;bytesCopied(Int64(bytesWritten))
|
|
}
|
|
if cancelled() { throw CancellationError() }
|
|
try output.synchronize();try output.close()
|
|
guard bytesWritten > 0,try digest(temporary,cancelled:cancelled) == expectedHash else { throw LibraryError.message("Audio data was empty or changed during copying. Retry the import.") }
|
|
if cancelled() { throw CancellationError() }
|
|
// Atomic same-directory replacement also repairs a previous zero-byte copy.
|
|
let result = temporary.withUnsafeFileSystemRepresentation { from in destination.withUnsafeFileSystemRepresentation { to in Darwin.rename(from!,to!) } }
|
|
guard result == 0 else { throw NSError(domain:NSPOSIXErrorDomain,code:Int(errno)) }
|
|
} catch is CancellationError { throw CancellationError() }
|
|
catch { throw LibraryError.message("Could not import \(source.lastPathComponent) from \(source.deletingLastPathComponent().path) into \(destination.deletingLastPathComponent().path): \(error.localizedDescription)") }
|
|
}
|
|
}
|
|
|
|
/// DJ media and waveform cache relocation; the library database stays on this Mac.
|
|
enum DJStorageMove {
|
|
static let key = "dj.storage.v1"
|
|
static func location(in library: Library) throws -> MediaStorageLocation? {
|
|
guard let text = try library.setting(key) else { return nil }
|
|
return try JSONDecoder().decode(MediaStorageLocation.self,from:Data(text.utf8))
|
|
}
|
|
static func run(database: URL,defaultRoot: URL,destination: URL,cancelled: () -> Bool = {false},progress: (ImportProgress) -> Void = {_ in}) throws -> ImportProgress {
|
|
var discovery = ImportProgress(current:"Checking DJ storage…"),lastDiscovery = Date.distantPast
|
|
func reportDiscovery(_ text: String,force: Bool = false) throws {
|
|
if cancelled() { throw CancellationError() }
|
|
if force || Date().timeIntervalSince(lastDiscovery) >= 0.1 { discovery.current = text;progress(discovery);lastDiscovery = Date() }
|
|
if cancelled() { throw CancellationError() }
|
|
}
|
|
try reportDiscovery("Checking DJ storage…",force:true)
|
|
let db = try Library(url:database),old = try location(in:db)
|
|
let oldRoots = Array(Set((old?.roots ?? [])+[defaultRoot.path,old?.path ?? defaultRoot.path])).map{URL(fileURLWithPath:$0)}
|
|
let target = destination.resolvingSymlinksInPath(),media = target.appendingPathComponent("Media"),waves = target.appendingPathComponent("Waveforms")
|
|
try ManagedMediaStorage.prepare(media);try ManagedMediaStorage.prepare(waves)
|
|
let oldSongs = try db.songs()
|
|
var cacheSources = [Int64:URL](),allCaches = Set<URL>()
|
|
var cacheFolders = Set<URL>()
|
|
for folder in Set(oldRoots.map({$0.appendingPathComponent("Waveforms").resolvingSymlinksInPath()})) {
|
|
try reportDiscovery("Scanning saved waveforms: "+folder.path)
|
|
if let files = try? FileManager.default.contentsOfDirectory(at:folder,includingPropertiesForKeys:[.isRegularFileKey,.isSymbolicLinkKey]) {
|
|
for file in files where file.pathExtension == "waveform" {
|
|
try reportDiscovery("Checking saved waveforms: \(allCaches.count.formatted()) found")
|
|
let attributes = try file.resourceValues(forKeys:[.isRegularFileKey,.isSymbolicLinkKey])
|
|
if attributes.isRegularFile == true,attributes.isSymbolicLink != true { allCaches.insert(file.resolvingSymlinksInPath());cacheFolders.insert(folder) }
|
|
}
|
|
}
|
|
}
|
|
if !cacheFolders.isEmpty {
|
|
let filesByName = Dictionary(grouping:allCaches,by: \.lastPathComponent)
|
|
for (index,song) in oldSongs.enumerated() {
|
|
try reportDiscovery("Matching waveform \(index+1) of \(oldSongs.count): "+song.title)
|
|
// The key filename does not depend on its cache directory.
|
|
if let key = try? WaveformCache.location(audio:URL(fileURLWithPath:song.audio),directory:waves).lastPathComponent,
|
|
let file = filesByName[key]?.first { cacheSources[song.id] = file }
|
|
}
|
|
}
|
|
let manifestKey = "dj.storage.waveform.sources.v1"
|
|
if let json = try db.setting(manifestKey),let saved = try? JSONDecoder().decode([String:String].self,from:Data(json.utf8)) {
|
|
let folders = Set(oldRoots.map{$0.appendingPathComponent("Waveforms").resolvingSymlinksInPath()})
|
|
for song in oldSongs where cacheSources[song.id] == nil {
|
|
try reportDiscovery("Checking interrupted transfers: "+song.title)
|
|
if let path = saved[String(song.id)] {
|
|
let file = URL(fileURLWithPath:path).resolvingSymlinksInPath()
|
|
if folders.contains(file.deletingLastPathComponent()),file.pathExtension == "waveform",FileManager.default.fileExists(atPath:file.path) { cacheSources[song.id] = file;allCaches.insert(file) }
|
|
}
|
|
}
|
|
}
|
|
if cancelled() { throw CancellationError() }
|
|
try db.setSetting(manifestKey,String(decoding:try JSONEncoder().encode(Dictionary(uniqueKeysWithValues:cacheSources.map{(String($0.key),$0.value.path)})),as:UTF8.self))
|
|
var mediaRoots = oldRoots.map{$0.appendingPathComponent("Media")}
|
|
if let json = try db.setting(MediaStorageLocation.key),let location = try? JSONDecoder().decode(MediaStorageLocation.self,from:Data(json.utf8)) { mediaRoots += (location.roots+[location.path]).map{URL(fileURLWithPath:$0)} }
|
|
var state = try ManagedMediaStorage.move(database:database,roots:mediaRoots,destination:media,cancelled:cancelled,progress:{ value in var update = value;update.finished = false;progress(update) })
|
|
state.finished = false
|
|
// Commit the new location before the cache phase so a cancelled cache copy
|
|
// still reopens the successfully relocated media on the next launch.
|
|
let saved = ManagedMediaStorage.location(target,roots:oldRoots.map(\.path))
|
|
try db.setSetting(key,String(decoding:try JSONEncoder().encode(saved),as:UTF8.self))
|
|
let songs = try db.songs()
|
|
var transfers = [URL:Set<URL>]()
|
|
let referencedCaches = Set(cacheSources.values)
|
|
for source in allCaches where !referencedCaches.contains(source) { transfers[source,default:[]].insert(waves.appendingPathComponent(source.lastPathComponent)) }
|
|
for song in songs {
|
|
if let source = cacheSources[song.id] { transfers[source,default:[]].insert(try WaveformCache.location(audio:URL(fileURLWithPath:song.audio),directory:waves)) }
|
|
}
|
|
state.total += transfers.values.reduce(0){$0+$1.count}
|
|
var lastCacheReport = Date.distantPast
|
|
progress(state)
|
|
for source in transfers.keys.sorted(by:{$0.path < $1.path}) {
|
|
if cancelled() { throw CancellationError() }
|
|
let hash = try ManagedMediaStorage.hash(source,cancelled:cancelled).map{String(format:"%02x",$0)}.joined()
|
|
for target in transfers[source]!.sorted(by:{$0.path < $1.path}) {
|
|
if cancelled() { throw CancellationError() }
|
|
state.current = "Moving waveform "+source.lastPathComponent
|
|
if source.standardizedFileURL != target.standardizedFileURL { try MediaByteCopy.copy(source,to:target,expectedHash:hash,cancelled:cancelled) }
|
|
state.completed += 1;state.logLines.append("[COPIED] "+target.path)
|
|
if Date().timeIntervalSince(lastCacheReport) >= 0.12 || state.completed == state.total { progress(state);lastCacheReport = Date() }
|
|
}
|
|
// All copies of this cache are verified before its original is removed.
|
|
if source.deletingLastPathComponent().standardizedFileURL != waves.standardizedFileURL {
|
|
let unchanged = try ManagedMediaStorage.hash(source,cancelled:cancelled).map{String(format:"%02x",$0)}.joined() == hash
|
|
if unchanged { try FileManager.default.removeItem(at:source) }
|
|
}
|
|
}
|
|
try db.setSetting(manifestKey,"")
|
|
state.finished = true;state.current = "DJ media and waveforms moved";progress(state);return state
|
|
}
|
|
}
|
|
|
|
struct DJCleanupCandidate: Identifiable, Sendable {
|
|
var id: String { url.path }
|
|
let url: URL
|
|
let size: Int64
|
|
let modified: Date?
|
|
}
|
|
struct DJCleanupPlan: Identifiable, Sendable {
|
|
let id = UUID()
|
|
var files: [DJCleanupCandidate]
|
|
var bytes: Int64 { files.reduce(0){$0+$1.size} }
|
|
}
|
|
enum DJLibraryMaintenance {
|
|
static func managedRoots(database: URL,root: URL) throws -> [URL] {
|
|
let db = try Library(url:database),location = try DJStorageMove.location(in:db)
|
|
return Array(Set(((location?.roots ?? [])+[location?.path ?? root.path,root.path]).map{URL(fileURLWithPath:$0).resolvingSymlinksInPath()}))
|
|
}
|
|
static func cleanupPlan(database: URL,root: URL,cancelled: () -> Bool = {false},progress: (ImportProgress) -> Void = {_ in}) throws -> DJCleanupPlan {
|
|
let db = try Library(url:database),songs = try db.songs(),roots = try managedRoots(database:database,root:root)
|
|
let referenced = Set(songs.flatMap{[$0.audio,$0.cdg].compactMap{$0}.map{URL(fileURLWithPath:$0).resolvingSymlinksInPath()}})
|
|
var canValidateCaches = true
|
|
var validCaches = Set<String>(),state = ImportProgress(current:"Checking referenced tracks…"),last = Date.distantPast
|
|
progress(state)
|
|
for (index,song) in songs.enumerated() {
|
|
if cancelled() { throw CancellationError() }
|
|
if let name = try? WaveformCache.location(audio:URL(fileURLWithPath:song.audio),directory:root).lastPathComponent { validCaches.insert(name) }
|
|
else { canValidateCaches = false }
|
|
if Date().timeIntervalSince(last) >= 0.12 { state.current = "Checking reference \(index+1) of \(songs.count)";progress(state);last = Date() }
|
|
}
|
|
var files = [DJCleanupCandidate]()
|
|
for folder in roots.flatMap({[$0.appendingPathComponent("Media"),$0.appendingPathComponent("Waveforms")]}) {
|
|
if cancelled() { throw CancellationError() }
|
|
guard let items = FileManager.default.enumerator(at:folder,includingPropertiesForKeys:[.isRegularFileKey,.isSymbolicLinkKey,.fileSizeKey,.contentModificationDateKey],options:[.skipsHiddenFiles,.skipsSubdirectoryDescendants]) else { continue }
|
|
for case let url as URL in items {
|
|
if cancelled() { throw CancellationError() }
|
|
let stem = url.deletingPathExtension().lastPathComponent
|
|
guard [64,129].contains(stem.count),stem.allSatisfy({$0.isHexDigit || $0 == "-"}) else { continue }
|
|
let isWaveform = folder.lastPathComponent == "Waveforms" && url.pathExtension == "waveform"
|
|
let isMedia = folder.lastPathComponent == "Media" && ["mp3","wav","m4a","mp4","m4v","mov","aiff","aif","caf","flac","aac","cdg"].contains(url.pathExtension.lowercased())
|
|
guard isWaveform || isMedia else { continue }
|
|
if isWaveform,!canValidateCaches { continue }
|
|
let values = try url.resourceValues(forKeys:[.isRegularFileKey,.isSymbolicLinkKey,.fileSizeKey,.contentModificationDateKey])
|
|
guard values.isRegularFile == true,values.isSymbolicLink != true,!referenced.contains(url.resolvingSymlinksInPath()),!isWaveform || !validCaches.contains(url.lastPathComponent) else { continue }
|
|
files.append(.init(url:url,size:Int64(values.fileSize ?? 0),modified:values.contentModificationDate))
|
|
if Date().timeIntervalSince(last) >= 0.12 { state.current = "Found \(files.count) unused managed files";progress(state);last = Date() }
|
|
}
|
|
}
|
|
return .init(files:files.sorted{$0.url.path < $1.url.path})
|
|
}
|
|
static func clean(_ plan: DJCleanupPlan,database: URL,root: URL,cancelled: () -> Bool = {false},progress: (ImportProgress) -> Void = {_ in},trash: (URL) throws -> Void = { try FileManager.default.trashItem(at:$0,resultingItemURL:nil) }) throws -> ImportProgress {
|
|
let allowed = Set(try cleanupPlan(database:database,root:root,cancelled:cancelled).files.map(\.id))
|
|
var state = ImportProgress(total:plan.files.count,current:"Moving unused files to Trash…")
|
|
for file in plan.files {
|
|
if cancelled() { throw CancellationError() }
|
|
let values = try? file.url.resourceValues(forKeys:[.fileSizeKey,.contentModificationDateKey,.isSymbolicLinkKey])
|
|
if allowed.contains(file.id),values?.isSymbolicLink != true,Int64(values?.fileSize ?? -1) == file.size,values?.contentModificationDate == file.modified {
|
|
try trash(file.url);state.logLines.append("[REMOVED] "+file.url.path)
|
|
} else { state.skipped += 1;state.logLines.append("[SKIPPED] "+file.url.path+" (changed or now referenced)") }
|
|
state.completed += 1;state.current = file.url.lastPathComponent;progress(state)
|
|
}
|
|
state.finished = true;state.current = "Cleanup complete";return state
|
|
}
|
|
static func removeTracks(_ ids: Set<Int64>,database: URL,cancelled: () -> Bool = {false}) throws {
|
|
let db = try Library(url:database)
|
|
try db.execute("BEGIN IMMEDIATE")
|
|
do {
|
|
for id in ids {
|
|
if cancelled() { throw CancellationError() }
|
|
try db.execute("DELETE FROM reinterpret_undo WHERE song=?",[String(id)])
|
|
try db.execute("DELETE FROM songs WHERE id=?",[String(id)])
|
|
}
|
|
try db.execute("COMMIT")
|
|
} catch { try? db.execute("ROLLBACK");throw error }
|
|
}
|
|
}
|
|
|
|
private final class ImportMetadataResult: @unchecked Sendable {
|
|
private let lock = NSLock()
|
|
private var tags = [AVMetadataKey:String]()
|
|
func set(_ value: [AVMetadataKey:String]) { lock.lock();tags = value;lock.unlock() }
|
|
func get() -> [AVMetadataKey:String] { lock.lock();defer { lock.unlock() };return tags }
|
|
}
|