An alternative to UBB.threads
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
FLocal/ImportConsole/ShallerDBProcessor.cs

141 lines
5.3 KiB

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using FLocal.Core;
using FLocal.Importer;
using FLocal.Common;
using FLocal.Common.actions;
using FLocal.Common.dataobjects;
using FLocal.Core.DB;
using FLocal.Core.DB.conditions;
namespace FLocal.ImportConsole {
static class ShallerDBProcessor {
private readonly static Dictionary<int, string> discussions = new Dictionary<int, string> {
{ 384486, "Common.Photos" },
{ 2665162, "sport" }, //Обсуждение игроков
{ 2099333, "hobby" }, //Клуб загадывателей ников
{ 2189773, "automoto" }, //ПРедлагаю ТОПИК! ФОТОГРАФИИ АВТО ФОРУМЧАН! И АВТО МИРА! (originally from common)
{ 1961373, "sport" }, //Результаты европейских чемпионатов
{ 2334188, "common" }, //Some foto about shanghai.
{ 2467452, "hobby" }, //Пентагон, ЧГК, десяточка и т.д.
};
private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0);
public static void processDB(string filename) {
HashSet<int> discussionsIds = new HashSet<int>();
using(StreamReader reader = new StreamReader(filename)) {
int i=0;
while(!reader.EndOfStream) {
string line = reader.ReadLine().Trim();
if(line == "") {
continue;
}
if(i%1000 == 0) {
Console.Write("[" + (int)(i/1000) + "]");
}
Dictionary<string, string> data = DictionaryConverter.FromDump(line);
int postId = int.Parse(data["Number"]);
try {
if(Config.instance.mainConnection.GetCountByConditions(Post.TableSpec.instance, new ComparisonCondition(Post.TableSpec.instance.getIdSpec(), ComparisonType.EQUAL, postId.ToString())) > 0) {
Console.Write("-");
} else {
int localMain = int.Parse(data["Local_Main"]);
int main = int.Parse(data["Main"]);
DateTime date = UNIX.AddSeconds(int.Parse(data["UnixTime"])).ToLocalTime();
User user;
string username = data["Username"];
try {
user = User.LoadByName(username);
} catch(NotFoundInDBException) {
Console.Error.WriteLine("Cannot find user '" + username + "'; creating one...");
ChangeSetUtil.ApplyChanges(
new InsertChange(
User.TableSpec.instance,
new Dictionary<string, AbstractFieldValue> {
{ User.TableSpec.FIELD_NAME, new ScalarFieldValue(username) },
{ User.TableSpec.FIELD_REGDATE, new ScalarFieldValue(date.ToUTCString()) },
{ User.TableSpec.FIELD_SHOWPOSTSTOUSERS, new ScalarFieldValue(User.ENUM_SHOWPOSTSTOUSERS_ALL) },
{ User.TableSpec.FIELD_BIOGRAPHY, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_LOCATION, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_SIGNATURE, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_TITLE, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_TOTALPOSTS, new ScalarFieldValue("0") },
{ User.TableSpec.FIELD_USERGROUPID, new ScalarFieldValue("1") },
}
)
);
user = User.LoadByName(data["Username"]);
}
string title = data["Subject"];
string body = data["Body"];
PostLayer layer = PostLayer.LoadById(1);
if(data.ContainsKey("Layer")) {
layer = PostLayer.LoadById(int.Parse(data["Layer"]));
}
if(postId == main || postId == localMain) {
//first post in the thread
string legacyBoardName;
if(localMain != 0) {
discussionsIds.Add(main);
legacyBoardName = discussions[main];
} else {
legacyBoardName = data["Board"];
}
Board board;
try {
board = Board.LoadByLegacyName(legacyBoardName);
} catch(NotFoundInDBException) {
throw new ApplicationException("Cannot find board '" + legacyBoardName + "'");
}
board.CreateThread(user, title, body, layer, date, postId);
} else {
int parentId = int.Parse(data["Parent"]);
if(parentId == 0) {
parentId = localMain;
if(parentId == 0) {
parentId = main;
}
}
Post post;
try {
post = Post.LoadById(parentId);
} catch(NotFoundInDBException) {
throw new ApplicationException("Cannot find parent post #" + parentId);
}
post.Reply(user, title, body, layer, date, postId);
}
Console.Write("+");
}
} catch(Exception e) {
Console.Error.WriteLine("Cannot process post #" + postId + ": " + e.GetType().FullName + ": " + e.Message);
Console.Error.WriteLine(e.StackTrace);
Console.Write("!");
// Console.ReadLine();
} finally {
i++;
if((i%50000)==0) {
Core.RegistryCleaner.CleanRegistry<int, Post>();
Core.RegistryCleaner.CleanRegistry<int, Thread>();
GC.Collect();
Console.Error.WriteLine();
Console.Error.WriteLine("Registry cleaned; garbage collected");
Console.Error.WriteLine();
}
}
}
}
Console.WriteLine("Not found discussions:");
foreach(int discussionId in discussionsIds.OrderBy(id => id)) {
Console.WriteLine(discussionId);
}
}
}
}