From bb1fbc66d9aaa4591ca35f77b0fa9b0da10e5c13 Mon Sep 17 00:00:00 2001
From: inga-lovinde <52715130+inga-lovinde@users.noreply.github.com>
Date: Sun, 4 Jul 2010 04:14:40 +0000
Subject: [PATCH] RegistryCleaner initial commit; ShallerDBProcessor optimised

---
 Builder/IISMainHandler/build.txt    |  2 +-
 Builder/IISUploadHandler/build.txt  |  2 +-
 Core/Core.csproj                    |  1 +
 Core/Registry.cs                    |  6 +++++
 Core/RegistryCleaner.cs             | 14 ++++++++++++
 ImportConsole/ShallerDBProcessor.cs | 35 ++++++++++++++++++++++-------
 6 files changed, 50 insertions(+), 10 deletions(-)
 create mode 100644 Core/RegistryCleaner.cs

diff --git a/Builder/IISMainHandler/build.txt b/Builder/IISMainHandler/build.txt
index c97fea1..64ae959 100644
--- a/Builder/IISMainHandler/build.txt
+++ b/Builder/IISMainHandler/build.txt
@@ -1 +1 @@
-533
\ No newline at end of file
+546
\ No newline at end of file
diff --git a/Builder/IISUploadHandler/build.txt b/Builder/IISUploadHandler/build.txt
index 02225a5..f3a5e81 100644
--- a/Builder/IISUploadHandler/build.txt
+++ b/Builder/IISUploadHandler/build.txt
@@ -1 +1 @@
-268
\ No newline at end of file
+281
\ No newline at end of file
diff --git a/Core/Core.csproj b/Core/Core.csproj
index 48ba9f2..1b0c237 100644
--- a/Core/Core.csproj
+++ b/Core/Core.csproj
@@ -80,6 +80,7 @@
     <Compile Include="IDataObject.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Registry.cs" />
+    <Compile Include="RegistryCleaner.cs" />
     <Compile Include="Switch.cs" />
     <Compile Include="Util.cs" />
   </ItemGroup>
diff --git a/Core/Registry.cs b/Core/Registry.cs
index 948e53c..4a401a7 100644
--- a/Core/Registry.cs
+++ b/Core/Registry.cs
@@ -51,6 +51,12 @@ namespace FLocal.Core {
             return this.storage.ContainsKey(id);
         }
 
+		internal void Clear() {
+			lock(this.locks) {
+				this.storage.Clear();
+			}
+		}
+
 		internal void Delete(TKey[] idsToDelete) {
 			foreach(TKey id in idsToDelete) {
 				lock(this.locks[id]) {
diff --git a/Core/RegistryCleaner.cs b/Core/RegistryCleaner.cs
new file mode 100644
index 0000000..ed51622
--- /dev/null
+++ b/Core/RegistryCleaner.cs
@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace FLocal.Core {
+	public static class RegistryCleaner {
+
+		public static void CleanRegistry<TKey, TData>() where TKey : struct where TData : IDataObject<TKey, TData>, new() {
+			Registry<TKey, TData>.instance.Clear();
+		}
+
+	}
+}
diff --git a/ImportConsole/ShallerDBProcessor.cs b/ImportConsole/ShallerDBProcessor.cs
index be24b8a..460487d 100644
--- a/ImportConsole/ShallerDBProcessor.cs
+++ b/ImportConsole/ShallerDBProcessor.cs
@@ -5,19 +5,30 @@ using System.Text;
 using System.IO;
 using FLocal.Core;
 using FLocal.Importer;
+using FLocal.Common;
 using FLocal.Common.actions;
 using FLocal.Common.dataobjects;
+using FLocal.Core.DB;
+using FLocal.Core.DB.conditions;
 
 namespace FLocal.ImportConsole {
 	static class ShallerDBProcessor {
 
 		private readonly static Dictionary<int, string> discussions = new Dictionary<int, string> {
 			{ 384486, "Common.Photos" },
+			{ 2665162, "sport" }, //Обсуждение игроков 
+			{ 2099333, "hobby" }, //Клуб загадывателей ников 
+			{ 2189773, "automoto" }, //ПРедлагаю ТОПИК! ФОТОГРАФИИ АВТО ФОРУМЧАН! И АВТО МИРА! (originally from common)
+			{ 1961373, "sport" }, //Результаты европейских чемпионатов
+			{ 2334188, "common" }, //Some foto about shanghai.
+			{ 2467452, "hobby" }, //Пентагон, ЧГК, десяточка и т.д. 
+
 		};
 
 		private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0);
 
 		public static void processDB(string filename) {
+			HashSet<int> discussionsIds = new HashSet<int>();
 			using(StreamReader reader = new StreamReader(filename)) {
 				int i=0;
 				while(!reader.EndOfStream) {
@@ -31,13 +42,7 @@ namespace FLocal.ImportConsole {
 					Dictionary<string, string> data = DictionaryConverter.FromDump(line);
 					int postId = int.Parse(data["Number"]);
 					try {
-						bool exists = false;
-						try {
-							Post post = Post.LoadById(postId);
-							exists = true;
-						} catch(NotFoundInDBException) {
-						}
-						if(exists) {
+						if(Config.instance.mainConnection.GetCountByConditions(Post.TableSpec.instance, new ComparisonCondition(Post.TableSpec.instance.getIdSpec(), ComparisonType.EQUAL, postId.ToString())) > 0) {
 							Console.Write("-");
 						} else {
 							int localMain = int.Parse(data["Local_Main"]);
@@ -48,7 +53,7 @@ namespace FLocal.ImportConsole {
 							try {
 								user = User.LoadByName(username);
 							} catch(NotFoundInDBException) {
-								Console.WriteLine("Cannot find user " + username);
+								Console.Error.WriteLine("Cannot find user '" + username + "'; creating one...");
 								ChangeSetUtil.ApplyChanges(
 									new InsertChange(
 										User.TableSpec.instance,
@@ -77,6 +82,7 @@ namespace FLocal.ImportConsole {
 								//first post in the thread
 								string legacyBoardName;
 								if(localMain != 0) {
+									discussionsIds.Add(main);
 									legacyBoardName = discussions[main];
 								} else {
 									legacyBoardName = data["Board"];
@@ -113,9 +119,22 @@ namespace FLocal.ImportConsole {
 //						Console.ReadLine();
 					} finally {
 						i++;
+						if((i%50000)==0) {
+							Core.RegistryCleaner.CleanRegistry<int, Post>();
+							Core.RegistryCleaner.CleanRegistry<int, Thread>();
+							GC.Collect();
+							Console.Error.WriteLine();
+							Console.Error.WriteLine("Registry cleaned; garbage collected");
+							Console.Error.WriteLine();
+						}
 					}
 				}
 			}
+
+			Console.WriteLine("Not found discussions:");
+			foreach(int discussionId in discussionsIds.OrderBy(id => id)) {
+				Console.WriteLine(discussionId);
+			}
 		}
 
 	}