zstrike/PhotosImporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add project files.

Browse Source
This commit is contained in:
ZStrike
2025-03-12 13:45:28 -04:00
parent 98de37bac9
commit ad585bd716
4 changed files with 366 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
PhotosImporter.csproj Normal file
View File

@@ -0,0 +1,30 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ManyConsole" Version="2.0.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.PowerShell.Commands.Diagnostics" Version="7.4.7" />
<PackageReference Include="Microsoft.PowerShell.Commands.Management" Version="7.4.7" />
<PackageReference Include="Microsoft.PowerShell.Commands.Utility" Version="7.4.7" />
<PackageReference Include="Microsoft.PowerShell.ConsoleHost" Version="7.4.7" />
<PackageReference Include="Microsoft.PowerShell.SDK" Version="7.6.0-preview.3" />
<PackageReference Include="Microsoft.WSMan.Management" Version="7.4.7" />
<PackageReference Include="Microsoft.WSMan.Runtime" Version="7.4.7" />
<PackageReference Include="MySql.Data" Version="9.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="NExifTool" Version="0.12.0" />
<PackageReference Include="Nito.AsyncEx" Version="5.1.2" />
<PackageReference Include="System.Data.Odbc" Version="8.0.0" />
<PackageReference Include="System.Diagnostics.PerformanceCounter" Version="8.0.1" />
<PackageReference Include="System.Management" Version="10.0.0-preview.1.25080.5" />
<PackageReference Include="System.Management.Automation" Version="7.4.7" />
</ItemGroup>
</Project>

25
PhotosImporter.sln Normal file
View File

@@ -0,0 +1,25 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.11.35312.102
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PhotosImporter", "PhotosImporter.csproj", "{0D052AAA-0255-4410-96C2-597DCC6A5236}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{0D052AAA-0255-4410-96C2-597DCC6A5236}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0D052AAA-0255-4410-96C2-597DCC6A5236}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0D052AAA-0255-4410-96C2-597DCC6A5236}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0D052AAA-0255-4410-96C2-597DCC6A5236}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {5D37C1A6-5006-443E-8B70-FE14194E14A8}
EndGlobalSection
EndGlobal

303
Program.cs Normal file
View File

@@ -0,0 +1,303 @@
using ManyConsole;
using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using System.Data.Odbc;
using NExifTool;
using Nito.AsyncEx.Synchronous;
using Nito.AsyncEx;
using System.Diagnostics;
using System.Text;
using System.Text.RegularExpressions;
using Newtonsoft.Json.Linq;
using System.Management.Automation;
using Microsoft.PowerShell;
using System.IO;
using Microsoft.PowerShell.Commands;
namespace PhotosImporter
{
public class Program
{
public static int Main(string[] args)
{
var commands = GetCommands();
return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out);
}
public static IEnumerable<ConsoleCommand> GetCommands()
{
return ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program));
}
}
public class ImportCommand : ConsoleCommand
{
private const int Success = 0;
private const int Failure = 2;
public string PhotosPath { get; set; }
public string PhotosTable { get; set; }
private static bool HasNonASCIIChars(string str)
{
return (System.Text.Encoding.UTF8.GetByteCount(str) != str.Length);
}
private static string[] GetBannedExtensions(string path)
{
string json = File.ReadAllText(path);
return JsonConvert.DeserializeObject<string[]>(json);
}
private static string[] GetVideoExtensions(string path)
{
string json = File.ReadAllText(path);
return JsonConvert.DeserializeObject<string[]>(json);
}
private static string Escape(string input)
{
return input.Replace("\"", "\\\"").Replace("'", "\\'");
}
private static bool IsBadFilename(string filename)
{
bool prq1 = filename.Contains("'") || filename.Contains('"') || filename.Contains(',') || filename.Contains('`') || filename.Contains('’') || filename.Contains('—');
bool prq2 = HasNonASCIIChars(filename);
return (prq1 || prq2);
}
private static void FixBadFilenameAndUpdateRecord(FileInfo file, string[] videoExtensions, string table)
{
char[] invalidChars = Path.GetInvalidFileNameChars();
//invalidChars.Append('\'');
string newFileName = string.Join("_", Encoding.ASCII.GetString(ASCIIEncoding.Convert(Encoding.Unicode, Encoding.ASCII, Encoding.Unicode.GetBytes(file.Name))).Split(invalidChars));
string newPath = Path.Combine(file.Directory.FullName,newFileName);
//string newFilename = filename.Replace("'", "");//.Replace('"', '').Replace(",", "").Replace("`", "").Replace("’", "").Replace("—", "-");
//File.Move(table + filename, table + newFilename);
file.MoveTo(newPath);
UpdateRecord(new FileInfo(newPath), videoExtensions, table);
}
private static bool IsBannedExtension(string filename, string[] bannedExtensions)
{
string extension = Path.GetExtension(filename).ToLower();
return bannedExtensions.Contains(extension);
}
private static void UpdateRecord(FileInfo file, string[] videoExtensions, string table)
{
string name = Path.GetFileNameWithoutExtension(file.FullName);
string hash = GetFileHash(file);
string extension = file.Extension;
string tags = GetFileTags(file);
int isVideo = 0;
if (videoExtensions.Contains(extension))
{
isVideo = 1;
}
double fileSizeMb = GetFileSizeMB(file);
using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
{
connection.Open();
//string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (@name, @hash, @ext, @tags, @video, @size)";
string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (\"{name}\", \"{Escape(hash)}\", \"{extension.Substring(1)}\", \'{tags}\', {isVideo.ToString()}, {fileSizeMb.ToString()})";
//string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (?, ?, ?, ?, ?, ?)";
//Console.WriteLine(query);
OdbcCommand command = new(query, connection);
//command.Parameters.AddWithValue("@name", name);
//command.Parameters.AddWithValue("@hash", hash);
//command.Parameters.AddWithValue("@ext", extension);
//command.Parameters.AddWithValue("@tags", tags);
//command.Parameters.AddWithValue("@video", isVideo);
//command.Parameters.AddWithValue("@size", fileSizeMb);
Console.WriteLine(command.CommandText);
command.ExecuteNonQuery();
}
}
private static string GetFileHash(FileInfo file)
{
string res = "";
using (var md5 = System.Security.Cryptography.MD5.Create())
{
if (file.Length < (500 * 1024 * 1024))
{
using (var stream = File.OpenRead(file.FullName))
{
var hash = md5.ComputeHash(stream);
res = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
} else
{
PowerShell ps = PowerShell.Create();
ps.AddCommand("Get-FileHash").AddParameter("Path", file.FullName).AddParameter("Algorithm", "MD5");
dynamic results = ps.Invoke();
foreach (var item in results)
{
//Type t = item.GetType();
//if (t is FileHashInfo)
//{
res = item.Hash.ToString().ToLowerInvariant();
}
}
return res;
}
}
private static string GetFileTags(FileInfo file)
{
// Start the child process.
Process p = new();
// Redirect the output stream of the child process.
p.StartInfo.UseShellExecute = false;
p.StartInfo.RedirectStandardOutput = true;
p.StartInfo.FileName = "C:\\Users\\ZStrike\\.aftman\\bin\\exiftool.exe";
p.StartInfo.Arguments = $"\"{(file.FullName)}\" -TagsList -json";
// FileName = "C:\\Deps\\getopt.bat";
p.Start();
// Do not wait for the child process to exit before
// reading to the end of its redirected stream.
// p.WaitForExit();
// Read the output stream first and then wait.
string output = p.StandardOutput.ReadToEnd();
Console.WriteLine(output);
dynamic tmp = JsonConvert.DeserializeObject(output);
p.WaitForExit();
if (tmp[0].ContainsKey("TagsList")) {
return JsonConvert.SerializeObject(tmp[0]["TagsList"]);
} else
{
return "{}";
}
//Console.WriteLine("Line 100: Tags (none)");
// This method is not implemented as it requires a third-party library to read EXIF tags.
// You can use a library like ExifLib or MetadataExtractor.
//ExifToolOptions opts = new ExifToolOptions { ExifToolPath = "C:\\Users\\ZStrike\\.aftman\\bin\\exiftool.exe" };
//var tagresult = new ExifTool(opts).GetTagsAsync(filePath).WaitAndUnwrapException();
//Console.WriteLine(tagresult);
}
private static double GetFileSizeMB(FileInfo file)
{
return (double)file.Length / (1024 * 1024);
}
private static void WriteBadFileNotifications(List<string> badFileNotifications)
{
using (StreamWriter writer = new StreamWriter("invalidfiles.txt"))
{
foreach (string notification in badFileNotifications)
{
writer.WriteLine(notification);
}
}
}
private static void DeleteRecords(string table)
{
using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
{
connection.Open();
string query = $"DELETE FROM {table}";
OdbcCommand command = new OdbcCommand(query, connection);
command.ExecuteNonQuery();
Console.WriteLine("Deleted!");
}
}
private static void ReIndexRecords(string table)
{
using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
{
connection.Open();
string query = $"ALTER TABLE {table} AUTO_INCREMENT = 1;";
OdbcCommand command = new OdbcCommand(query, connection);
command.ExecuteNonQuery();
}
}
public ImportCommand() {
// Register the actual command with a simple (optional) description.
IsCommand("Import", "Import Files");
// Add a longer description for the help on that specific command.
HasLongDescription("This can be used to quickly read a file's contents " +
"while optionally stripping out the ',' character.");
// Required options/flags, append '=' to obtain the required value.
HasRequiredOption("p|path=", "Path", p => PhotosPath = p);
HasRequiredOption("t|table=", "Table", t => PhotosTable = t);
// Optional options/flags, append ':' to obtain an optional value, or null if not specified.
//HasOption("s|strip:", "Strips ',' from the file before writing to output.",
// t => StripCommaCharacter = t == null ? true : Convert.ToBoolean(t));
}
public override int Run(string[] remainingArguments)
{
string sqlFilePath = Path.Combine(System.Reflection.Assembly.GetExecutingAssembly().Location, "output.sql");
System.IO.FileStream sqlFile = File.OpenWrite(sqlFilePath); // base class not allowed???
string bannedExtensionsPath = "C:/Deps/bannedExtensions.json";
string videoExtensionsPath = "C:/Deps/videoExtensions.json";
string[] bannedExtensions = GetBannedExtensions(bannedExtensionsPath);
string[] videoExtensions = GetVideoExtensions(videoExtensionsPath);
List<string> badFileNotifications = new List<string>();
Console.WriteLine("Deleting and re-indexing in 10 seconds...");
// System.Threading.Thread.Sleep(10000);
DeleteRecords(PhotosTable);
ReIndexRecords(PhotosTable);
foreach (string filename in Directory.GetFiles(PhotosPath))
{
FileInfo file = new(filename);
if (!file.Name.StartsWith('.'))
{
Console.WriteLine($"{file.Name}");
//Path.GetFileName(filename);
if (IsBadFilename(file.Name) && !IsBannedExtension(file.Name, bannedExtensions))
{
badFileNotifications.Add($"Bad Filename: {file}");
Console.WriteLine($"Bad Filename: {file.Name}");
FixBadFilenameAndUpdateRecord(file, videoExtensions, PhotosTable);
}
if (!IsBannedExtension(file.Name, bannedExtensions))
{
UpdateRecord(file, videoExtensions, PhotosTable);
}
Console.WriteLine("Processed: " + file.Name);
}
WriteBadFileNotifications(badFileNotifications);
}
foreach (var item in badFileNotifications)
{
Console.WriteLine($"{item}");
}
return Success;
}
}
}

8
Properties/launchSettings.json Normal file
View File

@@ -0,0 +1,8 @@
{
"profiles": {
"PhotosImporter": {
"commandName": "Project",
"commandLineArgs": "Import -t test -p C:\\Users\\ZStrike\\Downloads\\DPRK\\"
}
}
}