RegistryCleaner initial commit; ShallerDBProcessor optimised

Builder/IISMainHandler/build.txt

@@ -1 +1 @@
-533
+546

Builder/IISUploadHandler/build.txt

@@ -1 +1 @@
-268
+281

Core/Core.csproj

@@ -80,6 +80,7 @@
     <Compile Include="IDataObject.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Registry.cs" />
+    <Compile Include="RegistryCleaner.cs" />
     <Compile Include="Switch.cs" />
     <Compile Include="Util.cs" />
   </ItemGroup>

Core/Registry.cs

@@ -51,6 +51,12 @@ namespace FLocal.Core {
             return this.storage.ContainsKey(id);
         }
 
+		internal void Clear() {
+			lock(this.locks) {
+				this.storage.Clear();
+			}
+		}
+
 		internal void Delete(TKey[] idsToDelete) {
 			foreach(TKey id in idsToDelete) {
 				lock(this.locks[id]) {

Core/RegistryCleaner.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace FLocal.Core {
+	public static class RegistryCleaner {
+
+		public static void CleanRegistry<TKey, TData>() where TKey : struct where TData : IDataObject<TKey, TData>, new() {
+			Registry<TKey, TData>.instance.Clear();
+		}
+
+	}
+}

ImportConsole/ShallerDBProcessor.cs

@@ -5,19 +5,30 @@ using System.Text;
 using System.IO;
 using FLocal.Core;
 using FLocal.Importer;
+using FLocal.Common;
 using FLocal.Common.actions;
 using FLocal.Common.dataobjects;
+using FLocal.Core.DB;
+using FLocal.Core.DB.conditions;
 
 namespace FLocal.ImportConsole {
 	static class ShallerDBProcessor {
 
 		private readonly static Dictionary<int, string> discussions = new Dictionary<int, string> {
 			{ 384486, "Common.Photos" },
+			{ 2665162, "sport" }, //Обсуждение игроков 
+			{ 2099333, "hobby" }, //Клуб загадывателей ников 
+			{ 2189773, "automoto" }, //ПРедлагаю ТОПИК! ФОТОГРАФИИ АВТО ФОРУМЧАН! И АВТО МИРА! (originally from common)
+			{ 1961373, "sport" }, //Результаты европейских чемпионатов
+			{ 2334188, "common" }, //Some foto about shanghai.
+			{ 2467452, "hobby" }, //Пентагон, ЧГК, десяточка и т.д. 
+
 		};
 
 		private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0);
 
 		public static void processDB(string filename) {
+			HashSet<int> discussionsIds = new HashSet<int>();
 			using(StreamReader reader = new StreamReader(filename)) {
 				int i=0;
 				while(!reader.EndOfStream) {
@@ -31,13 +42,7 @@ namespace FLocal.ImportConsole {
 					Dictionary<string, string> data = DictionaryConverter.FromDump(line);
 					int postId = int.Parse(data["Number"]);
 					try {
-						bool exists = false;
-						try {
-							Post post = Post.LoadById(postId);
-							exists = true;
-						} catch(NotFoundInDBException) {
-						}
-						if(exists) {
+						if(Config.instance.mainConnection.GetCountByConditions(Post.TableSpec.instance, new ComparisonCondition(Post.TableSpec.instance.getIdSpec(), ComparisonType.EQUAL, postId.ToString())) > 0) {
 							Console.Write("-");
 						} else {
 							int localMain = int.Parse(data["Local_Main"]);
@@ -48,7 +53,7 @@ namespace FLocal.ImportConsole {
 							try {
 								user = User.LoadByName(username);
 							} catch(NotFoundInDBException) {
-								Console.WriteLine("Cannot find user " + username);
+								Console.Error.WriteLine("Cannot find user '" + username + "'; creating one...");
 								ChangeSetUtil.ApplyChanges(
 									new InsertChange(
 										User.TableSpec.instance,
@@ -77,6 +82,7 @@ namespace FLocal.ImportConsole {
 								//first post in the thread
 								string legacyBoardName;
 								if(localMain != 0) {
+									discussionsIds.Add(main);
 									legacyBoardName = discussions[main];
 								} else {
 									legacyBoardName = data["Board"];
@@ -113,9 +119,22 @@ namespace FLocal.ImportConsole {
 //						Console.ReadLine();
 					} finally {
 						i++;
+						if((i%50000)==0) {
+							Core.RegistryCleaner.CleanRegistry<int, Post>();
+							Core.RegistryCleaner.CleanRegistry<int, Thread>();
+							GC.Collect();
+							Console.Error.WriteLine();
+							Console.Error.WriteLine("Registry cleaned; garbage collected");
+							Console.Error.WriteLine();
+						}
 					}
 				}
 			}
+
+			Console.WriteLine("Not found discussions:");
+			foreach(int discussionId in discussionsIds.OrderBy(id => id)) {
+				Console.WriteLine(discussionId);
+			}
 		}
 
 	}