382 lines
16 KiB
C#
382 lines
16 KiB
C#
using ManyConsole;
|
||
using Microsoft.PowerShell.Commands;
|
||
using MySql.Data.MySqlClient;
|
||
using Mysqlx.Crud;
|
||
using Newtonsoft.Json;
|
||
using Spectre.Console;
|
||
using System.Data.Odbc;
|
||
using System.Diagnostics;
|
||
using System.IO;
|
||
using System;
|
||
using System.Management;
|
||
using System.Management.Automation;
|
||
using System.Text;
|
||
namespace PhotosImporter
|
||
{
|
||
public class Program
|
||
{
|
||
public static int Main(string[] args)
|
||
{
|
||
var commands = GetCommands();
|
||
|
||
return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out);
|
||
}
|
||
|
||
public static IEnumerable<ConsoleCommand> GetCommands()
|
||
{
|
||
return ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program));
|
||
}
|
||
}
|
||
public class ImportCommand : ConsoleCommand // deez nuts
|
||
{
|
||
private const int Success = 0;
|
||
private const int Failure = 2;
|
||
|
||
public string PhotosPath { get; set; }
|
||
public string PhotosTable { get; set; }
|
||
public bool LiveUpdate { get; set; }
|
||
public bool Recursive { get; set; }
|
||
|
||
private static bool HasNonASCIIChars(string str)
|
||
{
|
||
return (System.Text.Encoding.UTF8.GetByteCount(str) != str.Length);
|
||
}
|
||
|
||
private static string[] GetBannedExtensions(string path)
|
||
{
|
||
string json = File.ReadAllText(path);
|
||
return JsonConvert.DeserializeObject<string[]>(json);
|
||
}
|
||
|
||
private static string[] GetVideoExtensions(string path)
|
||
{
|
||
string json = File.ReadAllText(path);
|
||
return JsonConvert.DeserializeObject<string[]>(json);
|
||
}
|
||
|
||
private static string Escape(string input)
|
||
{
|
||
return input.Replace("\"", "\\\"").Replace("'", "\\'");
|
||
}
|
||
private static bool IsBadFilename(string filename)
|
||
{
|
||
bool prq1 = filename.Contains("'") || filename.Contains('"') || filename.Contains(',') || filename.Contains('`') || filename.Contains('’') || filename.Contains('—');
|
||
bool prq2 = HasNonASCIIChars(filename);
|
||
|
||
return (prq1 || prq2);
|
||
}
|
||
|
||
private static string FixBadFilenameAndUpdateRecord(FileInfo file, string[] videoExtensions, string table, string path)
|
||
{
|
||
string newFileName = string.Join("_", Encoding.ASCII.GetString(ASCIIEncoding.Convert(Encoding.Unicode, Encoding.ASCII, Encoding.Unicode.GetBytes(file.Name))).Split(Path.GetInvalidFileNameChars()));
|
||
string newPath = Path.Combine(file.Directory.FullName,newFileName);
|
||
file.MoveTo(newPath);
|
||
|
||
return UpdateRecord(new FileInfo(newPath), videoExtensions, table, path);
|
||
|
||
}
|
||
|
||
private static bool IsBannedExtension(string filename, string[] bannedExtensions)
|
||
{
|
||
string extension = Path.GetExtension(filename).ToLower();
|
||
return bannedExtensions.Contains(extension);
|
||
}
|
||
|
||
private static string UpdateRecord(FileInfo file, string[] videoExtensions, string table, string path)
|
||
{
|
||
//string name = Path.GetFileNameWithoutExtension(file.FullName);
|
||
string hash = GetFileHash(file);
|
||
string extension = file.Extension;
|
||
string tags = GetFileTags(file);
|
||
bool isVideo = videoExtensions.Contains(extension);
|
||
double fileSizeMb = GetFileSizeMB(file);
|
||
|
||
//using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
|
||
//{
|
||
// connection.Open();
|
||
// //string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (@name, @hash, @ext, @tags, @video, @size)";
|
||
// string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (\"{name}\", \"{Escape(hash)}\", \"{extension.Substring(1)}\", \'{tags}\', {isVideo.ToString()}, {fileSizeMb.ToString()})";
|
||
// //string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (?, ?, ?, ?, ?, ?)";
|
||
// //Console.WriteLine(query);
|
||
// OdbcCommand command = new(query, connection);
|
||
|
||
// //command.Parameters.AddWithValue("@name", name);
|
||
// //command.Parameters.AddWithValue("@hash", hash);
|
||
// //command.Parameters.AddWithValue("@ext", extension);
|
||
// //command.Parameters.AddWithValue("@tags", tags);
|
||
// //command.Parameters.AddWithValue("@video", isVideo);
|
||
// //command.Parameters.AddWithValue("@size", fileSizeMb);
|
||
|
||
// Console.WriteLine(command.CommandText);
|
||
// command.ExecuteNonQuery();
|
||
//}
|
||
string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (\"{file.ToString().Replace(path+"\\", "").Replace(extension,"").Replace("\\","/")}\", \"{Escape(hash)}\", \"{extension.Substring(1)}\", \'{tags}\', {Convert.ToInt16(isVideo).ToString()}, {fileSizeMb.ToString()});";
|
||
return query;
|
||
}
|
||
|
||
private static string GetFileHash(FileInfo file)
|
||
{
|
||
string res = "";
|
||
|
||
using (var md5 = System.Security.Cryptography.MD5.Create())
|
||
{
|
||
if (file.Length < (500 * 1024 * 1024))
|
||
{
|
||
using (var stream = File.OpenRead(file.FullName))
|
||
{
|
||
var hash = md5.ComputeHash(stream);
|
||
res = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
|
||
}
|
||
} else
|
||
{
|
||
PowerShell ps = PowerShell.Create();
|
||
ps.AddCommand("Get-FileHash").AddParameter("Path", file.FullName).AddParameter("Algorithm", "MD5");
|
||
dynamic results = ps.Invoke();
|
||
foreach (var item in results)
|
||
{
|
||
res = item.Hash.ToString().ToLowerInvariant();
|
||
}
|
||
}
|
||
return res;
|
||
}
|
||
}
|
||
|
||
private static string GetFileTags(FileInfo file)
|
||
{
|
||
// Start the child process.
|
||
Process p = new();
|
||
// Redirect the output stream of the child process.
|
||
p.StartInfo.UseShellExecute = false;
|
||
p.StartInfo.RedirectStandardOutput = true;
|
||
p.StartInfo.FileName = "C:\\Users\\ZStrike\\.aftman\\bin\\exiftool.exe";
|
||
p.StartInfo.Arguments = $"\"{(file.FullName)}\" -TagsList -json";
|
||
// FileName = "C:\\Deps\\getopt.bat";
|
||
p.Start();
|
||
// Do not wait for the child process to exit before
|
||
// reading to the end of its redirected stream.
|
||
// p.WaitForExit();
|
||
// Read the output stream first and then wait.
|
||
string output = p.StandardOutput.ReadToEnd();
|
||
//Console.WriteLine(output); //dep 5/3/25
|
||
dynamic tmp = JsonConvert.DeserializeObject(output);
|
||
|
||
p.WaitForExit();
|
||
if (tmp[0].ContainsKey("TagsList")) {
|
||
return string.Join(":", tmp[0]["TagsList"]);
|
||
} else
|
||
{
|
||
return "{}";
|
||
}
|
||
|
||
//Console.WriteLine("Line 100: Tags (none)");
|
||
// This method is not implemented as it requires a third-party library to read EXIF tags.
|
||
// You can use a library like ExifLib or MetadataExtractor.
|
||
//ExifToolOptions opts = new ExifToolOptions { ExifToolPath = "C:\\Users\\ZStrike\\.aftman\\bin\\exiftool.exe" };
|
||
|
||
//var tagresult = new ExifTool(opts).GetTagsAsync(filePath).WaitAndUnwrapException();
|
||
//Console.WriteLine(tagresult);
|
||
|
||
}
|
||
|
||
private static double GetFileSizeMB(FileInfo file)
|
||
{
|
||
return (double)file.Length / (1024 * 1024);
|
||
}
|
||
|
||
private static void WriteBadFileNotifications(List<string> badFileNotifications)
|
||
{
|
||
using (StreamWriter writer = new StreamWriter("invalidfiles.txt"))
|
||
{
|
||
foreach (string notification in badFileNotifications)
|
||
{
|
||
writer.WriteLine(notification);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static void DeleteRecords(string table)
|
||
{
|
||
using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
|
||
{
|
||
connection.Open();
|
||
|
||
string query = $"DELETE FROM {table}";
|
||
OdbcCommand command = new OdbcCommand(query, connection);
|
||
|
||
command.ExecuteNonQuery();
|
||
Console.WriteLine("Deleted!");
|
||
}
|
||
}
|
||
|
||
private static void ReIndexRecords(string table)
|
||
{
|
||
using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
|
||
{
|
||
connection.Open();
|
||
|
||
string query = $"ALTER TABLE {table} AUTO_INCREMENT = 1;";
|
||
OdbcCommand command = new OdbcCommand(query, connection);
|
||
|
||
command.ExecuteNonQuery();
|
||
}
|
||
}
|
||
|
||
public ImportCommand() {
|
||
// Register the actual command with a simple (optional) description.
|
||
IsCommand("Import", "Import Files");
|
||
|
||
// Add a longer description for the help on that specific command.
|
||
HasLongDescription("This can be used to quickly read a file's contents " +
|
||
"while optionally stripping out the ',' character.");
|
||
|
||
// Required options/flags, append '=' to obtain the required value.
|
||
HasRequiredOption("p|path=", "Path", p => PhotosPath = p);
|
||
HasRequiredOption("t|table=", "Table", t => PhotosTable = t);
|
||
HasRequiredOption<bool>("l|live=", "Live Update", l => LiveUpdate = l);
|
||
HasRequiredOption<bool>("r|recursive=", "Recursive Search", r => Recursive = r);
|
||
|
||
// Optional options/flags, append ':' to obtain an optional value, or null if not specified.
|
||
//HasOption("s|strip:", "Strips ',' from the file before writing to output.",
|
||
// t => StripCommaCharacter = t == null ? true : Convert.ToBoolean(t));
|
||
}
|
||
|
||
public override int Run(string[] remainingArguments)
|
||
{
|
||
string sqlFilePath = Path.Combine(System.AppContext.BaseDirectory, "output.sql");
|
||
StreamWriter sqlFile = File.CreateText(sqlFilePath); // base class not allowed??? // null;
|
||
sqlFile.AutoFlush = true;
|
||
sqlFile.WriteLine($"DELETE FROM {PhotosTable};");
|
||
sqlFile.WriteLine($"ALTER TABLE {PhotosTable} AUTO_INCREMENT = 1;");
|
||
|
||
string bannedExtensionsPath = "C:/Deps/bannedExtensions.json";
|
||
string videoExtensionsPath = "C:/Deps/videoExtensions.json";
|
||
|
||
string[] bannedExtensions = GetBannedExtensions(bannedExtensionsPath);
|
||
string[] videoExtensions = GetVideoExtensions(videoExtensionsPath);
|
||
|
||
List<string> badFileNotifications = new List<string>();
|
||
// Console.WriteLine("Deleting and re-indexing in 10 seconds...");
|
||
// System.Threading.Thread.Sleep(10000);
|
||
if (LiveUpdate) {
|
||
DeleteRecords(PhotosTable);
|
||
ReIndexRecords(PhotosTable);
|
||
}
|
||
//new ProgressBarColumn { IndeterminateStyle}.
|
||
AnsiConsole.Progress()
|
||
.AutoRefresh(true) // Turn off auto refresh
|
||
.AutoClear(false) // Do not remove the task list when done
|
||
.HideCompleted(false) // Hide tasks as they are completed
|
||
.Columns(new ProgressColumn[]
|
||
{
|
||
new SpinnerColumn(Spinner.Known.BoxBounce), // Task description
|
||
new ProgressBarColumn(), // Progress bar
|
||
new PercentageColumn(), // Percentage
|
||
new RemainingTimeColumn(), // Remaining time
|
||
}).Start(ctx =>
|
||
{
|
||
|
||
var tsk = new ProgressTaskSettings { MaxValue = Directory.GetFiles(PhotosPath).Length, AutoStart = true };
|
||
var task1 = ctx.AddTask("Photos Processed", tsk);
|
||
System.IO.EnumerationOptions Eopt = new()
|
||
{
|
||
RecurseSubdirectories = true
|
||
};
|
||
foreach (string filename in Directory.GetFiles(PhotosPath,"*",Eopt))
|
||
{
|
||
FileInfo file = new(filename);
|
||
//"".Replace(PhotosPath+"\\","")
|
||
string entryName = file.ToString().Replace(PhotosPath, "");
|
||
if (!file.Name.StartsWith('.') && !IsBannedExtension(file.Name, bannedExtensions))
|
||
{
|
||
//Console.WriteLine($"{file.Name}");
|
||
//Path.GetFileName(filename);
|
||
if (IsBadFilename(file.Name))
|
||
{
|
||
badFileNotifications.Add($"Bad Filename: {file}, added anyways.");
|
||
//Console.WriteLine($"Bad Filename: {file.Name}");
|
||
UpdateRecord(file, videoExtensions, PhotosTable, PhotosPath);
|
||
//sqlFile.WriteLine(FixBadFilenameAndUpdateRecord(file, videoExtensions, PhotosTable));
|
||
}
|
||
else
|
||
{
|
||
|
||
sqlFile.WriteLine(UpdateRecord(file, videoExtensions, PhotosTable, PhotosPath));
|
||
}
|
||
//Console.WriteLine("Processed: " + file.Name);
|
||
}
|
||
task1.Increment(1.0);
|
||
}
|
||
});
|
||
|
||
foreach (var item in badFileNotifications)
|
||
{
|
||
Console.WriteLine($"{item}");
|
||
}
|
||
|
||
|
||
return Success;
|
||
|
||
}
|
||
|
||
}
|
||
public class CreateLibraryCommand: ConsoleCommand
|
||
{
|
||
private const int Success = 0;
|
||
private const int Failure = 2;
|
||
|
||
public string PhotosPath { get; set; }
|
||
public string PhotosTable { get; set; }
|
||
public bool Force { get; set; }
|
||
public string WebPath { get; set; }
|
||
//public string LocalWebPath { get; set; }
|
||
public CreateLibraryCommand()
|
||
{
|
||
// Register the actual command with a simple (optional) description.
|
||
IsCommand("create", "Create Library");
|
||
|
||
// Add a longer description for the help on that specific command.
|
||
HasLongDescription("This can be used to quickly read a file's contents " +
|
||
"while optionally stripping out the ',' character.");
|
||
|
||
// Required options/flags, append '=' to obtain the required value.
|
||
HasRequiredOption("p|path=", "Path", p => PhotosPath = p);
|
||
HasRequiredOption("t|table=", "Table", t => PhotosTable = t);
|
||
|
||
HasRequiredOption("f|force=", "Force", f => Force = false);
|
||
HasRequiredOption("w|wp=", "Web Path", wp => WebPath = wp);
|
||
//HasRequiredOption("l|lwp=", "Local Web Path", lwp => LocalWebPath = lwp);
|
||
|
||
|
||
// Optional options/flags, append ':' to obtain an optional value, or null if not specified.
|
||
//HasOption("s|strip:", "Strips ',' from the file before writing to output.",
|
||
// t => StripCommaCharacter = t == null ? true : Convert.ToBoolean(t));
|
||
}
|
||
|
||
public override int Run(string[] remainingArguments)
|
||
{
|
||
using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
|
||
{
|
||
connection.Open();
|
||
|
||
string query = $"SHOW TABLES like \"{PhotosTable}\";";
|
||
OdbcCommand command = new OdbcCommand(query, connection);
|
||
|
||
int exists = command.ExecuteNonQuery();
|
||
|
||
if ( exists > 0)
|
||
{
|
||
Console.WriteLine("Library already exists!");
|
||
return Failure;
|
||
} else
|
||
{
|
||
var ggg = new DirectoryInfo(PhotosPath).FullName;
|
||
|
||
new OdbcCommand($"CREATE TABLE {PhotosTable} LIKE template;", connection).ExecuteNonQuery();
|
||
new OdbcCommand($"INSERT INTO meta (db, path, webpath, localwebpath) VALUES (\"{PhotosTable}\", \"{PhotosPath}\", \"https://cdn2.zstrike.dev/{WebPath}\", \"http://192.168.1.155:8445/{WebPath}\");",connection).ExecuteNonQuery();
|
||
Console.WriteLine($"Created Library \"{PhotosTable}\"");
|
||
return Success;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} |