Photochk 1.0.0 — 類似写真を探して整理する macOS アプリ
フォルダ内の写真から、中身が同じものと見た目が似ているものを検出して グループにまとめ、残す1枚を選んでほかをゴミ箱へ送れるようにする。 検出は安い判定から順に4段: - ファイルサイズが同じものだけ SHA-256 でバイト一致を確定 - dHash（知覚ハッシュ）のハミング距離で候補ペアを抽出 - 候補に出てくる画像だけ Vision の feature print で精査 - Union-Find で連結成分をグループ化 判定のしきい値は実写画像で距離を実測して決めた（docs/calibration.md）。 tools/release-update.sh で Gitea のリリースを更新する。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
yfoxjp
@@ -0,0 +1,42 @@
|
||||
import Foundation
|
||||
|
||||
/// 同時実行数を抑えつつ並列に変換し、入力と同じ順序で結果を返す。
|
||||
///
|
||||
/// 画像デコードや Vision の推論は CPU を使い切るので、無制限に投げるより
|
||||
/// コア数程度に抑えたほうが速く、メモリも安定する。
|
||||
func mapConcurrently<Item: Sendable, Output: Sendable>(
|
||||
_ items: [Item],
|
||||
limit: Int = max(2, ProcessInfo.processInfo.activeProcessorCount),
|
||||
onProgress: (@Sendable (Int) -> Void)? = nil,
|
||||
_ transform: @escaping @Sendable (Item) async -> Output
|
||||
) async throws -> [Output] {
|
||||
guard !items.isEmpty else { return [] }
|
||||
|
||||
var results = [Output?](repeating: nil, count: items.count)
|
||||
|
||||
try await withThrowingTaskGroup(of: (Int, Output).self) { group in
|
||||
var next = 0
|
||||
let initial = min(limit, items.count)
|
||||
while next < initial {
|
||||
let index = next
|
||||
group.addTask { (index, await transform(items[index])) }
|
||||
next += 1
|
||||
}
|
||||
|
||||
var finished = 0
|
||||
while let (index, value) = try await group.next() {
|
||||
results[index] = value
|
||||
finished += 1
|
||||
onProgress?(finished)
|
||||
try Task.checkCancellation()
|
||||
|
||||
if next < items.count {
|
||||
let index = next
|
||||
group.addTask { (index, await transform(items[index])) }
|
||||
next += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results.compactMap { $0 }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import CryptoKit
|
||||
import Foundation
|
||||
|
||||
enum ContentHasher {
|
||||
private static let chunkSize = 1 << 20 // 1 MB
|
||||
|
||||
/// ファイル全体を読み込まずに SHA-256 を求める。
|
||||
static func sha256(of url: URL) throws -> String {
|
||||
let handle = try FileHandle(forReadingFrom: url)
|
||||
defer { try? handle.close() }
|
||||
|
||||
var hasher = SHA256()
|
||||
while let chunk = try handle.read(upToCount: chunkSize), !chunk.isEmpty {
|
||||
hasher.update(data: chunk)
|
||||
}
|
||||
return hasher.finalize().map { String(format: "%02x", $0) }.joined()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import ImageIO
|
||||
|
||||
/// 画像を1回デコードして得られる基本情報。dHash と表示用メタデータをまとめて取る。
|
||||
struct ImageBasics: Codable, Hashable, Sendable {
|
||||
let dHash: UInt64
|
||||
let pixelWidth: Int
|
||||
let pixelHeight: Int
|
||||
let captureDate: Date?
|
||||
}
|
||||
|
||||
/// 隣接ピクセルの明暗差から 64bit の知覚ハッシュ（dHash）を作る。
|
||||
///
|
||||
/// リサイズや再圧縮では値がほとんど変わらないので、feature print にかける候補ペアを
|
||||
/// 安く絞り込むための一次フィルタとして使う。
|
||||
enum DHash {
|
||||
/// 9×8 に落として横隣接を比較するので 8×8 = 64bit になる。
|
||||
private static let hashWidth = 9
|
||||
private static let hashHeight = 8
|
||||
|
||||
static func compute(for url: URL) -> ImageBasics? {
|
||||
guard let source = CGImageSourceCreateWithURL(url as CFURL, nil),
|
||||
CGImageSourceGetCount(source) > 0
|
||||
else { return nil }
|
||||
|
||||
let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any]
|
||||
let (width, height) = pixelSize(from: properties)
|
||||
let captureDate = captureDate(from: properties)
|
||||
|
||||
// フルデコードを避けるため、まず小さなサムネイルを取り出す。
|
||||
// WithTransform で EXIF の回転を適用しておく。Vision の feature print も同じく EXIF を
|
||||
// 反映するので、これで「表示したときの見た目」どうしを比較する形に揃う。
|
||||
// 結果として、回転が焼き込まれた版と EXIF で回している版は同じ写真として検出できる一方、
|
||||
// 実際に向きが違う写真どうしは別物として扱われる。
|
||||
let thumbnailOptions: [CFString: Any] = [
|
||||
kCGImageSourceCreateThumbnailFromImageAlways: true,
|
||||
kCGImageSourceCreateThumbnailWithTransform: true,
|
||||
kCGImageSourceShouldCacheImmediately: true,
|
||||
kCGImageSourceThumbnailMaxPixelSize: 64,
|
||||
]
|
||||
guard let thumbnail = CGImageSourceCreateThumbnailAtIndex(
|
||||
source, 0, thumbnailOptions as CFDictionary
|
||||
) else { return nil }
|
||||
|
||||
guard let hash = hash(of: thumbnail) else { return nil }
|
||||
|
||||
return ImageBasics(
|
||||
dHash: hash,
|
||||
pixelWidth: width,
|
||||
pixelHeight: height,
|
||||
captureDate: captureDate
|
||||
)
|
||||
}
|
||||
|
||||
/// 2つのハッシュがどれだけ違うか（異なるビット数）。
|
||||
static func hammingDistance(_ a: UInt64, _ b: UInt64) -> Int {
|
||||
(a ^ b).nonzeroBitCount
|
||||
}
|
||||
|
||||
// MARK: - 内部
|
||||
|
||||
private static func hash(of image: CGImage) -> UInt64? {
|
||||
let bytesPerRow = 16 // hashWidth(9) 以上で 16 バイト境界に揃える
|
||||
var buffer = [UInt8](repeating: 0, count: bytesPerRow * hashHeight)
|
||||
|
||||
let ok = buffer.withUnsafeMutableBytes { raw -> Bool in
|
||||
guard let context = CGContext(
|
||||
data: raw.baseAddress,
|
||||
width: hashWidth,
|
||||
height: hashHeight,
|
||||
bitsPerComponent: 8,
|
||||
bytesPerRow: bytesPerRow,
|
||||
space: CGColorSpaceCreateDeviceGray(),
|
||||
bitmapInfo: CGImageAlphaInfo.none.rawValue
|
||||
) else { return false }
|
||||
|
||||
context.interpolationQuality = .medium
|
||||
context.draw(
|
||||
image,
|
||||
in: CGRect(x: 0, y: 0, width: hashWidth, height: hashHeight)
|
||||
)
|
||||
return true
|
||||
}
|
||||
guard ok else { return nil }
|
||||
|
||||
var hash: UInt64 = 0
|
||||
var bit = 0
|
||||
for y in 0..<hashHeight {
|
||||
let row = y * bytesPerRow
|
||||
for x in 0..<(hashWidth - 1) {
|
||||
if buffer[row + x] > buffer[row + x + 1] {
|
||||
hash |= (1 << UInt64(bit))
|
||||
}
|
||||
bit += 1
|
||||
}
|
||||
}
|
||||
return hash
|
||||
}
|
||||
|
||||
private static func pixelSize(from properties: [CFString: Any]?) -> (Int, Int) {
|
||||
let width = properties?[kCGImagePropertyPixelWidth] as? Int ?? 0
|
||||
let height = properties?[kCGImagePropertyPixelHeight] as? Int ?? 0
|
||||
|
||||
// EXIF の回転を反映した「見た目の」サイズを返す。
|
||||
let orientation = properties?[kCGImagePropertyOrientation] as? Int ?? 1
|
||||
return (5...8).contains(orientation) ? (height, width) : (width, height)
|
||||
}
|
||||
|
||||
private static func captureDate(from properties: [CFString: Any]?) -> Date? {
|
||||
guard let exif = properties?[kCGImagePropertyExifDictionary] as? [CFString: Any] else {
|
||||
return nil
|
||||
}
|
||||
let raw = exif[kCGImagePropertyExifDateTimeOriginal] as? String
|
||||
?? exif[kCGImagePropertyExifDateTimeDigitized] as? String
|
||||
guard let raw else { return nil }
|
||||
return exifDateFormatter.date(from: raw)
|
||||
}
|
||||
|
||||
/// EXIF の日時は "2024:05:01 12:34:56" 形式。
|
||||
private static let exifDateFormatter: DateFormatter = {
|
||||
let f = DateFormatter()
|
||||
f.locale = Locale(identifier: "en_US_POSIX")
|
||||
f.dateFormat = "yyyy:MM:dd HH:mm:ss"
|
||||
return f
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import Foundation
|
||||
import Vision
|
||||
|
||||
/// Vision の feature print で「見た目の近さ」を測る。
|
||||
///
|
||||
/// `perform(on: URL)` は Vision 側が ImageIO でデコードするため、HEIC や RAW もそのまま渡せる。
|
||||
enum FeaturePrinter {
|
||||
static func compute(for url: URL) async throws -> FeaturePrintObservation {
|
||||
var request = GenerateImageFeaturePrintRequest()
|
||||
// 縦横比が違う版（トリミング・リサイズ）でも比較できるよう、切り取らずに収める。
|
||||
request.cropAndScaleAction = .scaleToFit
|
||||
return try await request.perform(on: url)
|
||||
}
|
||||
|
||||
/// 距離は 0 が同一。比較できない組み合わせでは nil を返す。
|
||||
static func distance(
|
||||
_ a: FeaturePrintObservation?,
|
||||
_ b: FeaturePrintObservation?
|
||||
) -> Double? {
|
||||
guard let a, let b else { return nil }
|
||||
return try? a.distance(to: b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import Foundation
|
||||
import Vision
|
||||
|
||||
/// 1ファイル分のキャッシュ。ファイルサイズと更新日時が一致するときだけ再利用する。
|
||||
struct FingerprintRecord: Codable, Sendable {
|
||||
var fileSize: Int64
|
||||
var modificationDate: Date
|
||||
var basics: ImageBasics?
|
||||
var featurePrint: FeaturePrintObservation?
|
||||
|
||||
func matches(_ file: ScannedFile) -> Bool {
|
||||
fileSize == file.fileSize
|
||||
&& abs(modificationDate.timeIntervalSince(file.modificationDate)) < 1
|
||||
}
|
||||
}
|
||||
|
||||
/// dHash と feature print をディスクに保存し、同じフォルダの再スキャンを速くする。
|
||||
///
|
||||
/// feature print は1枚あたり数 KB あるので、JSON ではなくバイナリ plist で持つ。
|
||||
actor FingerprintCache {
|
||||
private var records: [String: FingerprintRecord] = [:]
|
||||
private var isDirty = false
|
||||
private let fileURL: URL?
|
||||
|
||||
init(fileURL: URL? = FingerprintCache.defaultFileURL()) {
|
||||
self.fileURL = fileURL
|
||||
self.records = Self.loadRecords(from: fileURL)
|
||||
}
|
||||
|
||||
static func defaultFileURL() -> URL? {
|
||||
guard let caches = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first
|
||||
else { return nil }
|
||||
let directory = caches.appendingPathComponent("Photochk", isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
return directory.appendingPathComponent("fingerprints.plist")
|
||||
}
|
||||
|
||||
// MARK: - 読み出し
|
||||
|
||||
func basics(for files: [ScannedFile]) -> [URL: ImageBasics] {
|
||||
var found: [URL: ImageBasics] = [:]
|
||||
for file in files {
|
||||
if let record = records[file.url.path], record.matches(file), let basics = record.basics {
|
||||
found[file.url] = basics
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
func featurePrints(for files: [ScannedFile]) -> [URL: FeaturePrintObservation] {
|
||||
var found: [URL: FeaturePrintObservation] = [:]
|
||||
for file in files {
|
||||
if let record = records[file.url.path], record.matches(file),
|
||||
let print = record.featurePrint {
|
||||
found[file.url] = print
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// MARK: - 書き込み
|
||||
|
||||
func store(basics: [URL: ImageBasics], for files: [ScannedFile]) {
|
||||
for file in files {
|
||||
guard let value = basics[file.url] else { continue }
|
||||
update(file) { $0.basics = value }
|
||||
}
|
||||
}
|
||||
|
||||
func store(featurePrints: [URL: FeaturePrintObservation], for files: [ScannedFile]) {
|
||||
for file in files {
|
||||
guard let value = featurePrints[file.url] else { continue }
|
||||
update(file) { $0.featurePrint = value }
|
||||
}
|
||||
}
|
||||
|
||||
private func update(_ file: ScannedFile, _ body: (inout FingerprintRecord) -> Void) {
|
||||
var record = records[file.url.path] ?? FingerprintRecord(
|
||||
fileSize: file.fileSize,
|
||||
modificationDate: file.modificationDate
|
||||
)
|
||||
if !record.matches(file) {
|
||||
// ファイルが変わっていたら作り直す
|
||||
record = FingerprintRecord(
|
||||
fileSize: file.fileSize,
|
||||
modificationDate: file.modificationDate
|
||||
)
|
||||
}
|
||||
body(&record)
|
||||
records[file.url.path] = record
|
||||
isDirty = true
|
||||
}
|
||||
|
||||
/// 消えたファイルのエントリを捨てる。スキャンのたびに肥大化するのを防ぐ。
|
||||
func prune(keeping files: [ScannedFile]) {
|
||||
let alive = Set(files.map(\.url.path))
|
||||
let before = records.count
|
||||
records = records.filter { alive.contains($0.key) }
|
||||
if records.count != before { isDirty = true }
|
||||
}
|
||||
|
||||
// MARK: - 永続化
|
||||
|
||||
func save() {
|
||||
guard isDirty, let fileURL else { return }
|
||||
let encoder = PropertyListEncoder()
|
||||
encoder.outputFormat = .binary
|
||||
guard let data = try? encoder.encode(records) else { return }
|
||||
try? data.write(to: fileURL, options: .atomic)
|
||||
isDirty = false
|
||||
}
|
||||
|
||||
private static func loadRecords(from fileURL: URL?) -> [String: FingerprintRecord] {
|
||||
guard let fileURL, let data = try? Data(contentsOf: fileURL) else { return [:] }
|
||||
// 形式が変わっていたら黙って捨てる。キャッシュなので作り直せばよい。
|
||||
return (try? PropertyListDecoder().decode([String: FingerprintRecord].self, from: data)) ?? [:]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import Foundation
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
/// 走査で見つかった画像ファイル1件。中身はまだ読んでいない。
|
||||
struct ScannedFile: Hashable, Sendable {
|
||||
let url: URL
|
||||
let fileSize: Int64
|
||||
let modificationDate: Date
|
||||
let creationDate: Date?
|
||||
}
|
||||
|
||||
enum PhotoScanner {
|
||||
/// フォルダ配下の画像ファイルを列挙する。
|
||||
///
|
||||
/// UTType が `.image` に conform するものだけを拾うので、JPEG / PNG / HEIC / TIFF に加えて
|
||||
/// 各社の RAW も自動的に対象になる。写真ライブラリなどのパッケージは中に入らない。
|
||||
static func scan(root: URL, includesSubfolders: Bool) throws -> [ScannedFile] {
|
||||
let keys: [URLResourceKey] = [
|
||||
.isRegularFileKey,
|
||||
.contentTypeKey,
|
||||
.fileSizeKey,
|
||||
.contentModificationDateKey,
|
||||
.creationDateKey,
|
||||
]
|
||||
|
||||
var options: FileManager.DirectoryEnumerationOptions = [
|
||||
.skipsHiddenFiles,
|
||||
.skipsPackageDescendants,
|
||||
]
|
||||
if !includesSubfolders {
|
||||
options.insert(.skipsSubdirectoryDescendants)
|
||||
}
|
||||
|
||||
guard let enumerator = FileManager.default.enumerator(
|
||||
at: root,
|
||||
includingPropertiesForKeys: keys,
|
||||
options: options
|
||||
) else {
|
||||
throw ScanError.cannotEnumerate(root)
|
||||
}
|
||||
|
||||
var results: [ScannedFile] = []
|
||||
for case let url as URL in enumerator {
|
||||
try Task.checkCancellation()
|
||||
guard let values = try? url.resourceValues(forKeys: Set(keys)),
|
||||
values.isRegularFile == true,
|
||||
let type = values.contentType,
|
||||
type.conforms(to: .image),
|
||||
let size = values.fileSize, size > 0
|
||||
else { continue }
|
||||
|
||||
results.append(
|
||||
ScannedFile(
|
||||
url: url,
|
||||
fileSize: Int64(size),
|
||||
modificationDate: values.contentModificationDate ?? .distantPast,
|
||||
creationDate: values.creationDate
|
||||
)
|
||||
)
|
||||
}
|
||||
return results
|
||||
}
|
||||
}
|
||||
|
||||
enum ScanError: LocalizedError {
|
||||
case cannotEnumerate(URL)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .cannotEnumerate(let url):
|
||||
return "「\(url.lastPathComponent)」の中身を読み取れませんでした。"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
import Foundation
|
||||
import Vision
|
||||
|
||||
/// 走査からグループ生成までの一連の処理をまとめる。
|
||||
///
|
||||
/// 安いフィルタから順にかけて候補を絞る:
|
||||
/// 1. ファイルサイズが同じものだけ SHA-256 → 完全同一を確定
|
||||
/// 2. dHash（知覚ハッシュ）で「見た目が近そうな」ペアを抽出
|
||||
/// 3. そのペアに登場する画像だけ Vision の feature print で精査
|
||||
/// 4. Union-Find で連結成分をグループ化
|
||||
enum SimilarityGrouper {
|
||||
private struct Pair: Sendable {
|
||||
let a: Int
|
||||
let b: Int
|
||||
}
|
||||
|
||||
private struct HashEntry: Sendable {
|
||||
let index: Int
|
||||
let hash: UInt64
|
||||
}
|
||||
|
||||
/// feature print を計算できなかったときの保険。dHash がここまで一致していれば同一とみなす。
|
||||
private static let fallbackHammingThreshold = 4
|
||||
|
||||
static func run(
|
||||
root: URL,
|
||||
settings: ScanSettings,
|
||||
cache: FingerprintCache,
|
||||
progress: @escaping @Sendable (ScanProgress) -> Void
|
||||
) async throws -> [SimilarityGroup] {
|
||||
progress(ScanProgress(phase: .scanning))
|
||||
let files = try PhotoScanner.scan(root: root, includesSubfolders: settings.includesSubfolders)
|
||||
try Task.checkCancellation()
|
||||
guard files.count > 1 else { return [] }
|
||||
|
||||
// --- Stage 1: 完全同一をまとめ、以降は各セットの代表1枚だけを扱う ---
|
||||
let leaderOf = try await exactLeaders(files: files, progress: progress)
|
||||
let leaderIndices = files.indices.filter { leaderOf[$0] == $0 }
|
||||
let leaderFiles = leaderIndices.map { files[$0] }
|
||||
|
||||
// --- Stage 2: dHash ---
|
||||
let basicsByURL = try await loadBasics(for: leaderFiles, cache: cache, progress: progress)
|
||||
|
||||
// --- Stage 3: 候補ペア → feature print ---
|
||||
let entries = leaderIndices.compactMap { index -> HashEntry? in
|
||||
guard let basics = basicsByURL[files[index].url] else { return nil }
|
||||
return HashEntry(index: index, hash: basics.dHash)
|
||||
}
|
||||
progress(ScanProgress(phase: .comparing))
|
||||
let pairs = try await candidatePairs(
|
||||
entries,
|
||||
threshold: settings.sensitivity.hammingThreshold
|
||||
)
|
||||
|
||||
var involved = Set<Int>()
|
||||
for pair in pairs {
|
||||
involved.insert(pair.a)
|
||||
involved.insert(pair.b)
|
||||
}
|
||||
let involvedFiles = involved.sorted().map { files[$0] }
|
||||
let printsByURL = try await loadFeaturePrints(
|
||||
for: involvedFiles, cache: cache, progress: progress
|
||||
)
|
||||
|
||||
// --- Stage 4: グループ化 ---
|
||||
progress(ScanProgress(phase: .grouping))
|
||||
var unionFind = UnionFind(count: files.count)
|
||||
for index in files.indices where leaderOf[index] != index {
|
||||
unionFind.union(index, leaderOf[index])
|
||||
}
|
||||
|
||||
let threshold = settings.sensitivity.featurePrintThreshold
|
||||
for pair in pairs {
|
||||
let printA = printsByURL[files[pair.a].url]
|
||||
let printB = printsByURL[files[pair.b].url]
|
||||
|
||||
if let distance = FeaturePrinter.distance(printA, printB) {
|
||||
if distance < threshold { unionFind.union(pair.a, pair.b) }
|
||||
} else if let basicsA = basicsByURL[files[pair.a].url],
|
||||
let basicsB = basicsByURL[files[pair.b].url],
|
||||
DHash.hammingDistance(basicsA.dHash, basicsB.dHash) <= fallbackHammingThreshold {
|
||||
unionFind.union(pair.a, pair.b)
|
||||
}
|
||||
}
|
||||
try Task.checkCancellation()
|
||||
|
||||
let groups = buildGroups(
|
||||
components: unionFind.components(),
|
||||
files: files,
|
||||
leaderOf: leaderOf,
|
||||
basicsByURL: basicsByURL,
|
||||
printsByURL: printsByURL
|
||||
)
|
||||
|
||||
await cache.prune(keeping: files)
|
||||
await cache.save()
|
||||
return groups
|
||||
}
|
||||
|
||||
// MARK: - Stage 1
|
||||
|
||||
/// 各インデックスに「完全同一セットの代表インデックス」を割り当てる。単独なら自分自身。
|
||||
private static func exactLeaders(
|
||||
files: [ScannedFile],
|
||||
progress: @escaping @Sendable (ScanProgress) -> Void
|
||||
) async throws -> [Int] {
|
||||
var leaderOf = Array(files.indices)
|
||||
|
||||
// サイズが違えば中身も違う。同サイズのものだけハッシュすればよい。
|
||||
var bySize: [Int64: [Int]] = [:]
|
||||
for (index, file) in files.enumerated() {
|
||||
bySize[file.fileSize, default: []].append(index)
|
||||
}
|
||||
let candidates = bySize.values.filter { $0.count > 1 }.flatMap { $0 }.sorted()
|
||||
guard !candidates.isEmpty else { return leaderOf }
|
||||
|
||||
progress(ScanProgress(phase: .hashing, completed: 0, total: candidates.count))
|
||||
let hashes = try await mapConcurrently(
|
||||
candidates,
|
||||
onProgress: { done in
|
||||
progress(ScanProgress(phase: .hashing, completed: done, total: candidates.count))
|
||||
}
|
||||
) { index -> String? in
|
||||
try? ContentHasher.sha256(of: files[index].url)
|
||||
}
|
||||
|
||||
var byDigest: [String: [Int]] = [:]
|
||||
for (index, hash) in zip(candidates, hashes) {
|
||||
guard let hash else { continue }
|
||||
byDigest["\(files[index].fileSize):\(hash)", default: []].append(index)
|
||||
}
|
||||
for indices in byDigest.values where indices.count > 1 {
|
||||
guard let leader = indices.min(by: { files[$0].url.path < files[$1].url.path }) else {
|
||||
continue
|
||||
}
|
||||
for index in indices { leaderOf[index] = leader }
|
||||
}
|
||||
return leaderOf
|
||||
}
|
||||
|
||||
// MARK: - Stage 2 / 3
|
||||
|
||||
private static func loadBasics(
|
||||
for files: [ScannedFile],
|
||||
cache: FingerprintCache,
|
||||
progress: @escaping @Sendable (ScanProgress) -> Void
|
||||
) async throws -> [URL: ImageBasics] {
|
||||
var result = await cache.basics(for: files)
|
||||
let missing = files.filter { result[$0.url] == nil }
|
||||
guard !missing.isEmpty else { return result }
|
||||
|
||||
let cached = files.count - missing.count
|
||||
progress(ScanProgress(phase: .fingerprinting, completed: cached, total: files.count))
|
||||
let computed = try await mapConcurrently(
|
||||
missing,
|
||||
onProgress: { done in
|
||||
progress(
|
||||
ScanProgress(phase: .fingerprinting, completed: cached + done, total: files.count)
|
||||
)
|
||||
}
|
||||
) { file -> ImageBasics? in
|
||||
DHash.compute(for: file.url)
|
||||
}
|
||||
|
||||
var fresh: [URL: ImageBasics] = [:]
|
||||
for (file, basics) in zip(missing, computed) {
|
||||
guard let basics else { continue }
|
||||
fresh[file.url] = basics
|
||||
result[file.url] = basics
|
||||
}
|
||||
await cache.store(basics: fresh, for: missing)
|
||||
return result
|
||||
}
|
||||
|
||||
private static func loadFeaturePrints(
|
||||
for files: [ScannedFile],
|
||||
cache: FingerprintCache,
|
||||
progress: @escaping @Sendable (ScanProgress) -> Void
|
||||
) async throws -> [URL: FeaturePrintObservation] {
|
||||
guard !files.isEmpty else { return [:] }
|
||||
var result = await cache.featurePrints(for: files)
|
||||
let missing = files.filter { result[$0.url] == nil }
|
||||
guard !missing.isEmpty else { return result }
|
||||
|
||||
let cached = files.count - missing.count
|
||||
progress(ScanProgress(phase: .comparing, completed: cached, total: files.count))
|
||||
let computed = try await mapConcurrently(
|
||||
missing,
|
||||
onProgress: { done in
|
||||
progress(
|
||||
ScanProgress(phase: .comparing, completed: cached + done, total: files.count)
|
||||
)
|
||||
}
|
||||
) { file -> FeaturePrintObservation? in
|
||||
try? await FeaturePrinter.compute(for: file.url)
|
||||
}
|
||||
|
||||
var fresh: [URL: FeaturePrintObservation] = [:]
|
||||
for (file, print) in zip(missing, computed) {
|
||||
guard let print else { continue }
|
||||
fresh[file.url] = print
|
||||
result[file.url] = print
|
||||
}
|
||||
await cache.store(featurePrints: fresh, for: missing)
|
||||
return result
|
||||
}
|
||||
|
||||
/// dHash のハミング距離がしきい値以下のペアを総当たりで探す。
|
||||
///
|
||||
/// 64bit の XOR + popcount なので1ペアあたりの実費はごく小さい。LSH などで
|
||||
/// 絞ると取りこぼしが出るため、区間をコアに分けて総当たりする方針にしている。
|
||||
private static func candidatePairs(
|
||||
_ entries: [HashEntry],
|
||||
threshold: Int
|
||||
) async throws -> [Pair] {
|
||||
guard entries.count > 1 else { return [] }
|
||||
|
||||
let cores = max(2, ProcessInfo.processInfo.activeProcessorCount)
|
||||
let chunkSize = max(64, entries.count / (cores * 4))
|
||||
let starts = stride(from: 0, to: entries.count, by: chunkSize).map { $0 }
|
||||
|
||||
let chunks = try await mapConcurrently(starts) { start -> [Pair] in
|
||||
var found: [Pair] = []
|
||||
let end = min(start + chunkSize, entries.count)
|
||||
for i in start..<end {
|
||||
let hash = entries[i].hash
|
||||
for j in (i + 1)..<entries.count
|
||||
where DHash.hammingDistance(hash, entries[j].hash) <= threshold {
|
||||
found.append(Pair(a: entries[i].index, b: entries[j].index))
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
return chunks.flatMap { $0 }
|
||||
}
|
||||
|
||||
// MARK: - Stage 4
|
||||
|
||||
/// グループの中に「代表とは別だが、互いにバイト単位で同じ」ファイルの組があれば印を付ける。
|
||||
///
|
||||
/// 代表と同一のものは相手が「完全同一」と表示されるので印は要らない。ここで拾うのは
|
||||
/// たとえば縮小版が2つあってその2つだけが同一、といったケース。
|
||||
private static func exactSetTags(
|
||||
in ranked: [Int],
|
||||
leaderOf: [Int],
|
||||
excluding representativeLeader: Int
|
||||
) -> [Int: String] {
|
||||
var membersPerLeader: [Int: Int] = [:]
|
||||
for index in ranked where leaderOf[index] != representativeLeader {
|
||||
membersPerLeader[leaderOf[index], default: 0] += 1
|
||||
}
|
||||
|
||||
let labels = Array("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
||||
var tags: [Int: String] = [:]
|
||||
// ranked の順に印を振るので、同じ結果からは常に同じ印になる。
|
||||
for index in ranked {
|
||||
let leader = leaderOf[index]
|
||||
guard membersPerLeader[leader, default: 0] > 1, tags[leader] == nil else { continue }
|
||||
tags[leader] = tags.count < labels.count
|
||||
? String(labels[tags.count])
|
||||
: "\(tags.count + 1)"
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
private static func buildGroups(
|
||||
components: [[Int]],
|
||||
files: [ScannedFile],
|
||||
leaderOf: [Int],
|
||||
basicsByURL: [URL: ImageBasics],
|
||||
printsByURL: [URL: FeaturePrintObservation]
|
||||
) -> [SimilarityGroup] {
|
||||
func makeAsset(_ index: Int) -> PhotoAsset {
|
||||
let file = files[index]
|
||||
// 完全同一なら中身は同じなので、代表の解像度・撮影日をそのまま使える。
|
||||
let basics = basicsByURL[files[leaderOf[index]].url]
|
||||
return PhotoAsset(
|
||||
url: file.url,
|
||||
fileSize: file.fileSize,
|
||||
modificationDate: file.modificationDate,
|
||||
captureDate: basics?.captureDate ?? file.creationDate,
|
||||
pixelWidth: basics?.pixelWidth ?? 0,
|
||||
pixelHeight: basics?.pixelHeight ?? 0
|
||||
)
|
||||
}
|
||||
|
||||
var groups: [SimilarityGroup] = []
|
||||
for component in components where component.count > 1 {
|
||||
let assets = Dictionary(uniqueKeysWithValues: component.map { ($0, makeAsset($0)) })
|
||||
|
||||
// 代表は「解像度が最大 → ファイルサイズが最大 → 撮影が最古」で選ぶ。
|
||||
let ranked = component.sorted { lhs, rhs in
|
||||
guard let a = assets[lhs], let b = assets[rhs] else { return lhs < rhs }
|
||||
if a.pixelCount != b.pixelCount { return a.pixelCount > b.pixelCount }
|
||||
if a.fileSize != b.fileSize { return a.fileSize > b.fileSize }
|
||||
let dateA = a.captureDate ?? .distantFuture
|
||||
let dateB = b.captureDate ?? .distantFuture
|
||||
if dateA != dateB { return dateA < dateB }
|
||||
return a.url.path < b.url.path
|
||||
}
|
||||
guard let representativeIndex = ranked.first,
|
||||
let representative = assets[representativeIndex]
|
||||
else { continue }
|
||||
let representativeLeader = leaderOf[representativeIndex]
|
||||
let exactSetTags = exactSetTags(
|
||||
in: ranked, leaderOf: leaderOf, excluding: representativeLeader
|
||||
)
|
||||
|
||||
let members = component.compactMap { index -> SimilarityGroup.Member? in
|
||||
guard let asset = assets[index] else { return nil }
|
||||
let relation: SimilarityGroup.Relation
|
||||
if leaderOf[index] == representativeLeader {
|
||||
relation = .exact
|
||||
} else {
|
||||
relation = .similar(
|
||||
distance: FeaturePrinter.distance(
|
||||
printsByURL[files[leaderOf[index]].url],
|
||||
printsByURL[files[representativeLeader].url]
|
||||
)
|
||||
)
|
||||
}
|
||||
return SimilarityGroup.Member(
|
||||
asset: asset,
|
||||
relation: relation,
|
||||
exactSetTag: exactSetTags[leaderOf[index]]
|
||||
)
|
||||
}
|
||||
|
||||
let ordered = members.sorted { lhs, rhs in
|
||||
if lhs.id == representative.url { return true }
|
||||
if rhs.id == representative.url { return false }
|
||||
if lhs.relation.sortKey != rhs.relation.sortKey {
|
||||
return lhs.relation.sortKey < rhs.relation.sortKey
|
||||
}
|
||||
return lhs.asset.url.path < rhs.asset.url.path
|
||||
}
|
||||
|
||||
groups.append(
|
||||
SimilarityGroup(members: ordered, representativeID: representative.url)
|
||||
)
|
||||
}
|
||||
|
||||
// 減らせる容量が大きいグループから見せる。
|
||||
return groups.sorted { lhs, rhs in
|
||||
if lhs.reclaimableBytes != rhs.reclaimableBytes {
|
||||
return lhs.reclaimableBytes > rhs.reclaimableBytes
|
||||
}
|
||||
return lhs.representativeID.path < rhs.representativeID.path
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import Foundation
|
||||
|
||||
/// 経路圧縮 + ランク併合の Union-Find。類似ペアから連結成分（グループ）を作るのに使う。
|
||||
struct UnionFind {
|
||||
private var parent: [Int]
|
||||
private var rank: [Int]
|
||||
|
||||
init(count: Int) {
|
||||
parent = Array(0..<count)
|
||||
rank = Array(repeating: 0, count: count)
|
||||
}
|
||||
|
||||
mutating func find(_ x: Int) -> Int {
|
||||
var root = x
|
||||
while parent[root] != root { root = parent[root] }
|
||||
// 経路圧縮
|
||||
var node = x
|
||||
while parent[node] != root {
|
||||
let next = parent[node]
|
||||
parent[node] = root
|
||||
node = next
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
mutating func union(_ a: Int, _ b: Int) -> Bool {
|
||||
let ra = find(a)
|
||||
let rb = find(b)
|
||||
guard ra != rb else { return false }
|
||||
if rank[ra] < rank[rb] {
|
||||
parent[ra] = rb
|
||||
} else if rank[ra] > rank[rb] {
|
||||
parent[rb] = ra
|
||||
} else {
|
||||
parent[rb] = ra
|
||||
rank[ra] += 1
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/// 連結成分を、根のインデックスをキーにしてまとめて返す。
|
||||
mutating func components() -> [[Int]] {
|
||||
var buckets: [Int: [Int]] = [:]
|
||||
for i in parent.indices {
|
||||
buckets[find(i), default: []].append(i)
|
||||
}
|
||||
return Array(buckets.values)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user