Add project files.

2025-03-12 13:45:28 -04:00
parent 98de37bac9
commit ad585bd716
4 changed files with 366 additions and 0 deletions
--- a/PhotosImporter.csproj
+++ b/PhotosImporter.csproj
@@ -0,0 +1,30 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="ManyConsole" Version="2.0.1" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+    <PackageReference Include="Microsoft.PowerShell.Commands.Diagnostics" Version="7.4.7" />
+    <PackageReference Include="Microsoft.PowerShell.Commands.Management" Version="7.4.7" />
+    <PackageReference Include="Microsoft.PowerShell.Commands.Utility" Version="7.4.7" />
+    <PackageReference Include="Microsoft.PowerShell.ConsoleHost" Version="7.4.7" />
+    <PackageReference Include="Microsoft.PowerShell.SDK" Version="7.6.0-preview.3" />
+    <PackageReference Include="Microsoft.WSMan.Management" Version="7.4.7" />
+    <PackageReference Include="Microsoft.WSMan.Runtime" Version="7.4.7" />
+    <PackageReference Include="MySql.Data" Version="9.0.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="NExifTool" Version="0.12.0" />
+    <PackageReference Include="Nito.AsyncEx" Version="5.1.2" />
+    <PackageReference Include="System.Data.Odbc" Version="8.0.0" />
+    <PackageReference Include="System.Diagnostics.PerformanceCounter" Version="8.0.1" />
+    <PackageReference Include="System.Management" Version="10.0.0-preview.1.25080.5" />
+    <PackageReference Include="System.Management.Automation" Version="7.4.7" />
+  </ItemGroup>
+
+</Project>
--- a/PhotosImporter.sln
+++ b/PhotosImporter.sln
@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.11.35312.102
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PhotosImporter", "PhotosImporter.csproj", "{0D052AAA-0255-4410-96C2-597DCC6A5236}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{0D052AAA-0255-4410-96C2-597DCC6A5236}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0D052AAA-0255-4410-96C2-597DCC6A5236}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0D052AAA-0255-4410-96C2-597DCC6A5236}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0D052AAA-0255-4410-96C2-597DCC6A5236}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {5D37C1A6-5006-443E-8B70-FE14194E14A8}
+	EndGlobalSection
+EndGlobal
--- a/Program.cs
+++ b/Program.cs
@@ -0,0 +1,303 @@
+using ManyConsole;
+using MySql.Data.MySqlClient;
+using Newtonsoft.Json;
+using System.Data.Odbc;
+using NExifTool;
+using Nito.AsyncEx.Synchronous;
+using Nito.AsyncEx;
+using System.Diagnostics;
+using System.Text;
+using System.Text.RegularExpressions;
+using Newtonsoft.Json.Linq;
+using System.Management.Automation;
+using Microsoft.PowerShell;
+using System.IO;
+using Microsoft.PowerShell.Commands;
+namespace PhotosImporter
+{
+    public class Program
+    {
+        public static int Main(string[] args)
+        {
+            var commands = GetCommands();
+
+            return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out);
+        }
+
+        public static IEnumerable<ConsoleCommand> GetCommands()
+        {
+            return ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program));
+        }
+    }
+    public class ImportCommand : ConsoleCommand
+    {
+        private const int Success = 0;
+        private const int Failure = 2;
+
+        public string PhotosPath { get; set; }
+        public string PhotosTable { get; set; }
+
+        private static bool HasNonASCIIChars(string str)
+        {
+            return (System.Text.Encoding.UTF8.GetByteCount(str) != str.Length);
+        }
+
+        private static string[] GetBannedExtensions(string path)
+        {
+            string json = File.ReadAllText(path);
+            return JsonConvert.DeserializeObject<string[]>(json);
+        }
+
+        private static string[] GetVideoExtensions(string path)
+        {
+            string json = File.ReadAllText(path);
+            return JsonConvert.DeserializeObject<string[]>(json);
+        }
+
+        private static string Escape(string input)
+        {
+            return input.Replace("\"", "\\\"").Replace("'", "\\'");
+        }
+        private static bool IsBadFilename(string filename)
+        {
+            bool prq1 = filename.Contains("'") || filename.Contains('"') || filename.Contains(',') || filename.Contains('`') || filename.Contains('’') || filename.Contains('—');
+            bool prq2 = HasNonASCIIChars(filename);
+
+            return (prq1 || prq2);
+        }
+
+        private static void FixBadFilenameAndUpdateRecord(FileInfo file, string[] videoExtensions, string table)
+        {
+            char[] invalidChars = Path.GetInvalidFileNameChars();
+            //invalidChars.Append('\'');
+            string newFileName = string.Join("_", Encoding.ASCII.GetString(ASCIIEncoding.Convert(Encoding.Unicode, Encoding.ASCII, Encoding.Unicode.GetBytes(file.Name))).Split(invalidChars));
+            string newPath = Path.Combine(file.Directory.FullName,newFileName);
+            //string newFilename = filename.Replace("'", "");//.Replace('"', '').Replace(",", "").Replace("`", "").Replace("’", "").Replace("—", "-");
+            //File.Move(table + filename, table + newFilename);
+            file.MoveTo(newPath);
+            UpdateRecord(new FileInfo(newPath), videoExtensions, table);
+            
+        }
+
+        private static bool IsBannedExtension(string filename, string[] bannedExtensions)
+        {
+            string extension = Path.GetExtension(filename).ToLower();
+            return bannedExtensions.Contains(extension);
+        }
+
+        private static void UpdateRecord(FileInfo file, string[] videoExtensions, string table)
+        {
+            string name = Path.GetFileNameWithoutExtension(file.FullName);
+            string hash = GetFileHash(file);
+            string extension = file.Extension;
+            string tags = GetFileTags(file);
+            int isVideo = 0;
+            if (videoExtensions.Contains(extension))
+            {
+                isVideo = 1;
+            }
+            double fileSizeMb = GetFileSizeMB(file);
+
+            using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
+            {
+                connection.Open();
+                //string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (@name, @hash, @ext, @tags, @video, @size)";
+                string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (\"{name}\", \"{Escape(hash)}\", \"{extension.Substring(1)}\", \'{tags}\', {isVideo.ToString()}, {fileSizeMb.ToString()})";
+                //string query = $"INSERT INTO {table} (name, hash, ext, tags, video, size) VALUES (?, ?, ?, ?, ?, ?)";
+                //Console.WriteLine(query);
+                OdbcCommand command = new(query, connection);
+
+                //command.Parameters.AddWithValue("@name", name);
+                //command.Parameters.AddWithValue("@hash", hash);
+                //command.Parameters.AddWithValue("@ext", extension);
+                //command.Parameters.AddWithValue("@tags", tags);
+                //command.Parameters.AddWithValue("@video", isVideo);
+                //command.Parameters.AddWithValue("@size", fileSizeMb);
+                
+                Console.WriteLine(command.CommandText);
+                command.ExecuteNonQuery();
+            }
+        }
+
+        private static string GetFileHash(FileInfo file)
+        {
+            string res = "";
+            using (var md5 = System.Security.Cryptography.MD5.Create())
+            {
+                if (file.Length < (500 * 1024 * 1024))
+                {
+                    using (var stream = File.OpenRead(file.FullName))
+                    {
+                        var hash = md5.ComputeHash(stream);
+                        res = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
+                    }
+                } else
+                {
+                    PowerShell ps = PowerShell.Create();
+                    ps.AddCommand("Get-FileHash").AddParameter("Path", file.FullName).AddParameter("Algorithm", "MD5");
+                    dynamic results = ps.Invoke();
+                    foreach (var item in results)
+                    {
+                        //Type t = item.GetType();
+                        //if (t is FileHashInfo)
+                        //{
+                        res = item.Hash.ToString().ToLowerInvariant();
+                            
+                    }
+                }
+                return res;
+            }
+        }
+
+        private static string GetFileTags(FileInfo file)
+        {
+            // Start the child process.
+            Process p = new();
+            // Redirect the output stream of the child process.
+            p.StartInfo.UseShellExecute = false;
+            p.StartInfo.RedirectStandardOutput = true;
+            p.StartInfo.FileName = "C:\\Users\\ZStrike\\.aftman\\bin\\exiftool.exe";
+            p.StartInfo.Arguments = $"\"{(file.FullName)}\" -TagsList -json";
+            // FileName = "C:\\Deps\\getopt.bat";
+            p.Start();
+            // Do not wait for the child process to exit before
+            // reading to the end of its redirected stream.
+            // p.WaitForExit();
+            // Read the output stream first and then wait.
+            string output = p.StandardOutput.ReadToEnd();
+            Console.WriteLine(output);
+            dynamic tmp = JsonConvert.DeserializeObject(output);
+            
+            p.WaitForExit();
+            if (tmp[0].ContainsKey("TagsList")) {
+               return JsonConvert.SerializeObject(tmp[0]["TagsList"]);
+            } else
+            {
+                return "{}";
+            }
+
+            //Console.WriteLine("Line 100: Tags (none)");
+            // This method is not implemented as it requires a third-party library to read EXIF tags.
+            // You can use a library like ExifLib or MetadataExtractor.
+            //ExifToolOptions opts = new ExifToolOptions { ExifToolPath = "C:\\Users\\ZStrike\\.aftman\\bin\\exiftool.exe" };
+
+            //var tagresult = new ExifTool(opts).GetTagsAsync(filePath).WaitAndUnwrapException();
+            //Console.WriteLine(tagresult);
+
+        }
+
+        private static double GetFileSizeMB(FileInfo file)
+        {
+
+            return (double)file.Length / (1024 * 1024);
+        }
+
+        private static void WriteBadFileNotifications(List<string> badFileNotifications)
+        {
+            using (StreamWriter writer = new StreamWriter("invalidfiles.txt"))
+            {
+                foreach (string notification in badFileNotifications)
+                {
+                    writer.WriteLine(notification);
+                }
+            }
+        }
+
+        private static void DeleteRecords(string table)
+        {
+            using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
+            {
+                connection.Open();
+
+                string query = $"DELETE FROM {table}";
+                OdbcCommand command = new OdbcCommand(query, connection);
+
+                command.ExecuteNonQuery();
+                Console.WriteLine("Deleted!");
+            }
+        }
+
+        private static void ReIndexRecords(string table)
+        {
+            using (OdbcConnection connection = new OdbcConnection("DSN=photos"))
+            {
+                connection.Open();
+
+                string query = $"ALTER TABLE {table} AUTO_INCREMENT = 1;";
+                OdbcCommand command = new OdbcCommand(query, connection);
+
+                command.ExecuteNonQuery();
+            }
+        }
+
+        public ImportCommand() {
+            // Register the actual command with a simple (optional) description.
+            IsCommand("Import", "Import Files");
+
+            // Add a longer description for the help on that specific command.
+            HasLongDescription("This can be used to quickly read a file's contents " +
+            "while optionally stripping out the ',' character.");
+
+            // Required options/flags, append '=' to obtain the required value.
+            HasRequiredOption("p|path=", "Path", p => PhotosPath = p);
+            HasRequiredOption("t|table=", "Table", t => PhotosTable = t);
+
+            // Optional options/flags, append ':' to obtain an optional value, or null if not specified.
+            //HasOption("s|strip:", "Strips ',' from the file before writing to output.",
+            //    t => StripCommaCharacter = t == null ? true : Convert.ToBoolean(t));
+        }
+
+        public override int Run(string[] remainingArguments)
+        {
+            string sqlFilePath = Path.Combine(System.Reflection.Assembly.GetExecutingAssembly().Location, "output.sql");
+            System.IO.FileStream sqlFile = File.OpenWrite(sqlFilePath); // base class not allowed???
+
+            
+                string bannedExtensionsPath = "C:/Deps/bannedExtensions.json";
+                string videoExtensionsPath = "C:/Deps/videoExtensions.json";
+
+                string[] bannedExtensions = GetBannedExtensions(bannedExtensionsPath);
+                string[] videoExtensions = GetVideoExtensions(videoExtensionsPath);
+
+                List<string> badFileNotifications = new List<string>();
+                Console.WriteLine("Deleting and re-indexing in 10 seconds...");
+               // System.Threading.Thread.Sleep(10000);
+
+                DeleteRecords(PhotosTable);
+                ReIndexRecords(PhotosTable);
+                foreach (string filename in Directory.GetFiles(PhotosPath))
+                {
+                    FileInfo file = new(filename);
+                if (!file.Name.StartsWith('.'))
+                {
+                    Console.WriteLine($"{file.Name}");
+                    //Path.GetFileName(filename);
+
+                    if (IsBadFilename(file.Name) && !IsBannedExtension(file.Name, bannedExtensions))
+                    {
+                        badFileNotifications.Add($"Bad Filename: {file}");
+                        Console.WriteLine($"Bad Filename: {file.Name}");
+                        FixBadFilenameAndUpdateRecord(file, videoExtensions, PhotosTable);
+                    }
+
+                    if (!IsBannedExtension(file.Name, bannedExtensions))
+                    {
+                        UpdateRecord(file, videoExtensions, PhotosTable);
+                    }
+                    Console.WriteLine("Processed: " + file.Name);
+                }
+
+                WriteBadFileNotifications(badFileNotifications);
+                }
+            foreach (var item in badFileNotifications)
+            {
+                Console.WriteLine($"{item}");
+            }
+
+
+            return Success;
+
+        }
+        
+    }
+}
--- a/Properties/launchSettings.json
+++ b/Properties/launchSettings.json
@@ -0,0 +1,8 @@
+{
+  "profiles": {
+    "PhotosImporter": {
+      "commandName": "Project",
+      "commandLineArgs": "Import -t test -p C:\\Users\\ZStrike\\Downloads\\DPRK\\"
+    }
+  }
+}